diff --git a/api/video/resolution.h b/api/video/resolution.h index 99cb62299b..11ffef0b03 100644 --- a/api/video/resolution.h +++ b/api/video/resolution.h @@ -11,12 +11,18 @@ #ifndef API_VIDEO_RESOLUTION_H_ #define API_VIDEO_RESOLUTION_H_ +#include + namespace webrtc { // A struct representing a video resolution in pixels. struct Resolution { int width = 0; int height = 0; + + // Helper methods. + int PixelCount() const { return width * height; } + std::pair ToPair() const { return std::make_pair(width, height); } }; inline bool operator==(const Resolution& lhs, const Resolution& rhs) { diff --git a/call/call_perf_tests.cc b/call/call_perf_tests.cc index 402d69df0c..9379dce833 100644 --- a/call/call_perf_tests.cc +++ b/call/call_perf_tests.cc @@ -789,11 +789,11 @@ TEST_F(CallPerfTest, MAYBE_KeepsHighBitrateWhenReconfiguringSender) { private: std::vector CreateEncoderStreams( - int width, - int height, - const VideoEncoderConfig& encoder_config) override { + int frame_width, + int frame_height, + const webrtc::VideoEncoderConfig& encoder_config) override { std::vector streams = - test::CreateVideoStreams(width, height, encoder_config); + test::CreateVideoStreams(frame_width, frame_height, encoder_config); streams[0].min_bitrate_bps = 50000; streams[0].target_bitrate_bps = streams[0].max_bitrate_bps = 2000000; return streams; diff --git a/call/rampup_tests.cc b/call/rampup_tests.cc index eea54ed37b..dd4fe573df 100644 --- a/call/rampup_tests.cc +++ b/call/rampup_tests.cc @@ -146,11 +146,11 @@ class RampUpTester::VideoStreamFactory private: std::vector CreateEncoderStreams( - int width, - int height, + int frame_width, + int frame_height, const VideoEncoderConfig& encoder_config) override { std::vector streams = - test::CreateVideoStreams(width, height, encoder_config); + test::CreateVideoStreams(frame_width, frame_height, encoder_config); if (encoder_config.number_of_streams == 1) { streams[0].target_bitrate_bps = streams[0].max_bitrate_bps = 2000000; } diff --git a/media/engine/fake_webrtc_call.cc b/media/engine/fake_webrtc_call.cc index 9fe5cf8337..48a8b12092 100644 --- a/media/engine/fake_webrtc_call.cc +++ b/media/engine/fake_webrtc_call.cc @@ -248,13 +248,14 @@ void FakeVideoSendStream::OnFrame(const webrtc::VideoFrame& frame) { encoder_config_.video_stream_factory->CreateEncoderStreams( frame.width(), frame.height(), encoder_config_); } else { + webrtc::VideoEncoder::EncoderInfo encoder_info; rtc::scoped_refptr< webrtc::VideoEncoderConfig::VideoStreamFactoryInterface> factory = rtc::make_ref_counted( encoder_config_.video_format.name, encoder_config_.max_qp, encoder_config_.content_type == webrtc::VideoEncoderConfig::ContentType::kScreen, - encoder_config_.legacy_conference_mode); + encoder_config_.legacy_conference_mode, encoder_info); video_streams_ = factory->CreateEncoderStreams( frame.width(), frame.height(), encoder_config_); @@ -286,12 +287,13 @@ void FakeVideoSendStream::ReconfigureVideoEncoder( video_streams_ = config.video_stream_factory->CreateEncoderStreams( width, height, config); } else { + webrtc::VideoEncoder::EncoderInfo encoder_info; rtc::scoped_refptr factory = rtc::make_ref_counted( config.video_format.name, config.max_qp, config.content_type == webrtc::VideoEncoderConfig::ContentType::kScreen, - config.legacy_conference_mode); + config.legacy_conference_mode, encoder_info); video_streams_ = factory->CreateEncoderStreams(width, height, config); } diff --git a/media/engine/webrtc_video_engine_unittest.cc b/media/engine/webrtc_video_engine_unittest.cc index d72403b599..c544128571 100644 --- a/media/engine/webrtc_video_engine_unittest.cc +++ b/media/engine/webrtc_video_engine_unittest.cc @@ -239,6 +239,18 @@ class MockVideoSource : public rtc::VideoSourceInterface { (override)); }; +std::vector GetStreamResolutions( + const std::vector& streams) { + std::vector res; + for (const auto& s : streams) { + if (s.active) { + res.push_back( + {rtc::checked_cast(s.width), rtc::checked_cast(s.height)}); + } + } + return res; +} + } // namespace #define EXPECT_FRAME_WAIT(c, w, h, t) \ @@ -8995,4 +9007,199 @@ TEST_F(WebRtcVideoChannelBaseTest, EncoderSelectorSwitchCodec) { channel_->SetEncoderSelector(kSsrc, nullptr); } +TEST_F(WebRtcVideoChannelTest, RequestedResolutionSinglecast) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = AddSendStream(); + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(1280, 720, + rtc::kNumMicrosecsPerSec / 30); + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder)); + + { // TEST requested_resolution < frame size + webrtc::RtpParameters rtp_parameters = + channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1UL, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].requested_resolution = {.width = 640, + .height = 360}; + channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + auto streams = stream->GetVideoStreams(); + ASSERT_EQ(streams.size(), 1u); + EXPECT_EQ(rtc::checked_cast(640), streams[0].width); + EXPECT_EQ(rtc::checked_cast(360), streams[0].height); + } + + { // TEST requested_resolution == frame size + auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1UL, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].requested_resolution = {.width = 1280, + .height = 720}; + channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + auto streams = stream->GetVideoStreams(); + ASSERT_EQ(streams.size(), 1u); + EXPECT_EQ(rtc::checked_cast(1280), streams[0].width); + EXPECT_EQ(rtc::checked_cast(720), streams[0].height); + } + + { // TEST requested_resolution > frame size + auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1UL, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].requested_resolution = {.width = 2 * 1280, + .height = 2 * 720}; + channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + auto streams = stream->GetVideoStreams(); + ASSERT_EQ(streams.size(), 1u); + EXPECT_EQ(rtc::checked_cast(1280), streams[0].width); + EXPECT_EQ(rtc::checked_cast(720), streams[0].height); + } + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, RequestedResolutionSinglecastCropping) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = AddSendStream(); + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(1280, 720, + rtc::kNumMicrosecsPerSec / 30); + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder)); + + { + auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1UL, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].requested_resolution = {.width = 720, + .height = 720}; + channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + auto streams = stream->GetVideoStreams(); + ASSERT_EQ(streams.size(), 1u); + EXPECT_EQ(rtc::checked_cast(720), streams[0].width); + EXPECT_EQ(rtc::checked_cast(720), streams[0].height); + } + + { + auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1UL, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].requested_resolution = {.width = 1280, + .height = 1280}; + channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + auto streams = stream->GetVideoStreams(); + ASSERT_EQ(streams.size(), 1u); + EXPECT_EQ(rtc::checked_cast(720), streams[0].width); + EXPECT_EQ(rtc::checked_cast(720), streams[0].height); + } + + { + auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(1UL, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].requested_resolution = {.width = 650, + .height = 650}; + channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + + auto streams = stream->GetVideoStreams(); + ASSERT_EQ(streams.size(), 1u); + EXPECT_EQ(rtc::checked_cast(480), streams[0].width); + EXPECT_EQ(rtc::checked_cast(480), streams[0].height); + } + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + +TEST_F(WebRtcVideoChannelTest, RequestedResolutionSimulcast) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + ASSERT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = SetUpSimulcast(true, false); + webrtc::test::FrameForwarder frame_forwarder; + cricket::FakeFrameSource frame_source(1280, 720, + rtc::kNumMicrosecsPerSec / 30); + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, &frame_forwarder)); + + { + webrtc::RtpParameters rtp_parameters = + channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(3UL, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].requested_resolution = {.width = 320, + .height = 180}; + rtp_parameters.encodings[1].requested_resolution = {.width = 640, + .height = 360}; + rtp_parameters.encodings[2].requested_resolution = {.width = 1280, + .height = 720}; + channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + EXPECT_EQ(GetStreamResolutions(stream->GetVideoStreams()), + (std::vector{ + {.width = 320, .height = 180}, + {.width = 640, .height = 360}, + {.width = 1280, .height = 720}, + })); + } + + { + webrtc::RtpParameters rtp_parameters = + channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(3UL, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].requested_resolution = {.width = 320, + .height = 180}; + rtp_parameters.encodings[1].active = false; + + rtp_parameters.encodings[2].requested_resolution = {.width = 1280, + .height = 720}; + channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + EXPECT_EQ(GetStreamResolutions(stream->GetVideoStreams()), + (std::vector{ + {.width = 320, .height = 180}, + {.width = 1280, .height = 720}, + })); + } + + { + webrtc::RtpParameters rtp_parameters = + channel_->GetRtpSendParameters(last_ssrc_); + EXPECT_EQ(3UL, rtp_parameters.encodings.size()); + rtp_parameters.encodings[0].requested_resolution = {.width = 320, + .height = 180}; + rtp_parameters.encodings[1].active = true; + rtp_parameters.encodings[1].requested_resolution = {.width = 640, + .height = 360}; + rtp_parameters.encodings[2].requested_resolution = {.width = 960, + .height = 540}; + channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters); + + frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame()); + + EXPECT_EQ(GetStreamResolutions(stream->GetVideoStreams()), + (std::vector{ + {.width = 320, .height = 180}, + {.width = 640, .height = 360}, + {.width = 960, .height = 540}, + })); + } + + EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr)); +} + } // namespace cricket diff --git a/rtc_tools/rtp_generator/rtp_generator.cc b/rtc_tools/rtp_generator/rtp_generator.cc index 51f3c35796..e1a2cb30da 100644 --- a/rtc_tools/rtp_generator/rtp_generator.cc +++ b/rtc_tools/rtp_generator/rtp_generator.cc @@ -175,6 +175,7 @@ RtpGenerator::RtpGenerator(const RtpGeneratorOptions& options) constexpr int kMaxBitrateBps = 2500000; // 2.5 Mbps int stream_count = 0; + webrtc::VideoEncoder::EncoderInfo encoder_info; for (const auto& send_config : options.video_streams) { webrtc::VideoSendStream::Config video_config(this); video_config.encoder_settings.encoder_factory = @@ -225,7 +226,7 @@ RtpGenerator::RtpGenerator(const RtpGeneratorOptions& options) encoder_config.video_stream_factory = rtc::make_ref_counted( video_config.rtp.payload_name, /*max qp*/ 56, /*screencast*/ false, - /*screenshare enabled*/ false); + /*screenshare enabled*/ false, encoder_info); // Setup the fake video stream for this. std::unique_ptr frame_generator = diff --git a/test/encoder_settings.cc b/test/encoder_settings.cc index 7398272874..f5b298b107 100644 --- a/test/encoder_settings.cc +++ b/test/encoder_settings.cc @@ -106,10 +106,10 @@ std::vector CreateVideoStreams( DefaultVideoStreamFactory::DefaultVideoStreamFactory() {} std::vector DefaultVideoStreamFactory::CreateEncoderStreams( - int width, - int height, + int frame_width, + int frame_height, const webrtc::VideoEncoderConfig& encoder_config) { - return CreateVideoStreams(width, height, encoder_config); + return CreateVideoStreams(frame_width, frame_height, encoder_config); } void FillEncoderConfiguration(VideoCodecType codec_type, diff --git a/test/encoder_settings.h b/test/encoder_settings.h index 01eb4fa773..6dbad0fee2 100644 --- a/test/encoder_settings.h +++ b/test/encoder_settings.h @@ -35,9 +35,9 @@ class DefaultVideoStreamFactory private: std::vector CreateEncoderStreams( - int width, - int height, - const VideoEncoderConfig& encoder_config) override; + int frame_width, + int frame_height, + const webrtc::VideoEncoderConfig& encoder_config) override; }; // Creates `encoder_config.number_of_streams` VideoStreams where index diff --git a/test/scenario/video_stream.cc b/test/scenario/video_stream.cc index 60bf5ef27a..ad352f9ab9 100644 --- a/test/scenario/video_stream.cc +++ b/test/scenario/video_stream.cc @@ -224,6 +224,7 @@ CreateEncoderSpecificSettings(VideoStreamConfig config) { } VideoEncoderConfig CreateVideoEncoderConfig(VideoStreamConfig config) { + webrtc::VideoEncoder::EncoderInfo encoder_info; VideoEncoderConfig encoder_config; encoder_config.codec_type = config.encoder.codec; encoder_config.content_type = ConvertContentType(config.encoder.content_type); @@ -241,7 +242,8 @@ VideoEncoderConfig CreateVideoEncoderConfig(VideoStreamConfig config) { VideoStreamConfig::Encoder::ContentType::kScreen; encoder_config.video_stream_factory = rtc::make_ref_counted( - cricket_codec, kDefaultMaxQp, screenshare, screenshare); + cricket_codec, kDefaultMaxQp, screenshare, screenshare, + encoder_info); } else { encoder_config.video_stream_factory = rtc::make_ref_counted(); diff --git a/video/config/encoder_stream_factory.cc b/video/config/encoder_stream_factory.cc index f6156106fd..c43922da17 100644 --- a/video/config/encoder_stream_factory.cc +++ b/video/config/encoder_stream_factory.cc @@ -18,9 +18,11 @@ #include "absl/strings/match.h" #include "api/video/video_codec_constants.h" #include "media/base/media_constants.h" +#include "media/base/video_adapter.h" #include "modules/video_coding/codecs/vp9/svc_config.h" #include "rtc_base/experiments/min_video_bitrate_experiment.h" #include "rtc_base/experiments/normalize_simulcast_size_experiment.h" +#include "rtc_base/logging.h" #include "video/config/simulcast.h" namespace cricket { @@ -96,22 +98,35 @@ static int GetMaxDefaultVideoBitrateKbps(int width, // TODO(bugs.webrtc.org/8785): Consider removing max_qp as member of // EncoderStreamFactory and instead set this value individually for each stream // in the VideoEncoderConfig.simulcast_layers. +EncoderStreamFactory::EncoderStreamFactory(std::string codec_name, + int max_qp, + bool is_screenshare, + bool conference_mode) + : codec_name_(codec_name), + max_qp_(max_qp), + is_screenshare_(is_screenshare), + conference_mode_(conference_mode), + trials_(fallback_trials_), + encoder_info_requested_resolution_alignment_(1) {} + EncoderStreamFactory::EncoderStreamFactory( std::string codec_name, int max_qp, bool is_screenshare, bool conference_mode, + const webrtc::VideoEncoder::EncoderInfo& encoder_info, const webrtc::FieldTrialsView* trials) - : codec_name_(codec_name), max_qp_(max_qp), is_screenshare_(is_screenshare), conference_mode_(conference_mode), - trials_(trials ? *trials : fallback_trials_) {} + trials_(trials ? *trials : fallback_trials_), + encoder_info_requested_resolution_alignment_( + encoder_info.requested_resolution_alignment) {} std::vector EncoderStreamFactory::CreateEncoderStreams( - int width, - int height, + int frame_width, + int frame_height, const webrtc::VideoEncoderConfig& encoder_config) { RTC_DCHECK_GT(encoder_config.number_of_streams, 0); RTC_DCHECK_GE(encoder_config.simulcast_layers.size(), @@ -125,10 +140,10 @@ std::vector EncoderStreamFactory::CreateEncoderStreams( absl::EqualsIgnoreCase(codec_name_, kH264CodecName)) && is_screenshare_ && conference_mode_)) { return CreateSimulcastOrConferenceModeScreenshareStreams( - width, height, encoder_config, experimental_min_bitrate); + frame_width, frame_height, encoder_config, experimental_min_bitrate); } - return CreateDefaultVideoStreams(width, height, encoder_config, + return CreateDefaultVideoStreams(frame_width, frame_height, encoder_config, experimental_min_bitrate); } @@ -172,7 +187,13 @@ EncoderStreamFactory::CreateDefaultVideoStreams( layer.active = absl::c_any_of(encoder_config.simulcast_layers, [](const auto& layer) { return layer.active; }); - if (encoder_config.simulcast_layers[0].scale_resolution_down_by > 1.) { + if (encoder_config.simulcast_layers[0].requested_resolution) { + auto res = GetLayerResolutionFromRequestedResolution( + width, height, + *encoder_config.simulcast_layers[0].requested_resolution); + layer.width = res.width; + layer.height = res.height; + } else if (encoder_config.simulcast_layers[0].scale_resolution_down_by > 1.) { layer.width = ScaleDownResolution( layer.width, encoder_config.simulcast_layers[0].scale_resolution_down_by, @@ -307,7 +328,13 @@ EncoderStreamFactory::CreateSimulcastOrConferenceModeScreenshareStreams( layers[i].max_framerate = encoder_config.simulcast_layers[i].max_framerate; } - if (has_scale_resolution_down_by) { + if (encoder_config.simulcast_layers[i].requested_resolution.has_value()) { + auto res = GetLayerResolutionFromRequestedResolution( + normalized_width, normalized_height, + *encoder_config.simulcast_layers[i].requested_resolution); + layers[i].width = res.width; + layers[i].height = res.height; + } else if (has_scale_resolution_down_by) { const double scale_resolution_down_by = std::max( encoder_config.simulcast_layers[i].scale_resolution_down_by, 1.0); layers[i].width = ScaleDownResolution( @@ -398,4 +425,23 @@ EncoderStreamFactory::CreateSimulcastOrConferenceModeScreenshareStreams( return layers; } +webrtc::Resolution +EncoderStreamFactory::GetLayerResolutionFromRequestedResolution( + int frame_width, + int frame_height, + webrtc::Resolution requested_resolution) const { + VideoAdapter adapter(encoder_info_requested_resolution_alignment_); + adapter.OnOutputFormatRequest(requested_resolution.ToPair(), + requested_resolution.PixelCount(), + absl::nullopt); + int cropped_width, cropped_height; + int out_width = 0, out_height = 0; + if (!adapter.AdaptFrameResolution(frame_width, frame_height, 0, + &cropped_width, &cropped_height, &out_width, + &out_height)) { + RTC_LOG(LS_ERROR) << "AdaptFrameResolution returned false!"; + } + return {.width = out_width, .height = out_height}; +} + } // namespace cricket diff --git a/video/config/encoder_stream_factory.h b/video/config/encoder_stream_factory.h index 7da4050116..11910eb1fe 100644 --- a/video/config/encoder_stream_factory.h +++ b/video/config/encoder_stream_factory.h @@ -15,6 +15,7 @@ #include "api/transport/field_trial_based_config.h" #include "api/units/data_rate.h" +#include "api/video_codecs/video_encoder.h" #include "video/config/video_encoder_config.h" namespace cricket { @@ -22,21 +23,18 @@ namespace cricket { class EncoderStreamFactory : public webrtc::VideoEncoderConfig::VideoStreamFactoryInterface { public: + // Note: this constructor is used by testcase in downstream. EncoderStreamFactory(std::string codec_name, int max_qp, bool is_screenshare, - bool conference_mode) - : EncoderStreamFactory(codec_name, - max_qp, - is_screenshare, - conference_mode, - nullptr) {} + bool conference_mode); EncoderStreamFactory(std::string codec_name, int max_qp, bool is_screenshare, bool conference_mode, - const webrtc::FieldTrialsView* trials); + const webrtc::VideoEncoder::EncoderInfo& encoder_info, + const webrtc::FieldTrialsView* trials = nullptr); private: std::vector CreateEncoderStreams( @@ -57,6 +55,11 @@ class EncoderStreamFactory const webrtc::VideoEncoderConfig& encoder_config, const absl::optional& experimental_min_bitrate) const; + webrtc::Resolution GetLayerResolutionFromRequestedResolution( + int in_frame_width, + int in_frame_height, + webrtc::Resolution requested_resolution) const; + const std::string codec_name_; const int max_qp_; const bool is_screenshare_; @@ -65,6 +68,7 @@ class EncoderStreamFactory const bool conference_mode_; const webrtc::FieldTrialBasedConfig fallback_trials_; const webrtc::FieldTrialsView& trials_; + const int encoder_info_requested_resolution_alignment_; }; } // namespace cricket diff --git a/video/config/video_encoder_config.h b/video/config/video_encoder_config.h index 41bdefad83..5a79d58cbf 100644 --- a/video/config/video_encoder_config.h +++ b/video/config/video_encoder_config.h @@ -135,8 +135,8 @@ class VideoEncoderConfig { // The size of the vector may not be larger than // `encoder_config.number_of_streams`. virtual std::vector CreateEncoderStreams( - int width, - int height, + int frame_width, + int frame_height, const VideoEncoderConfig& encoder_config) = 0; protected: diff --git a/video/end_to_end_tests/resolution_bitrate_limits_tests.cc b/video/end_to_end_tests/resolution_bitrate_limits_tests.cc index 5d905b850c..8455832885 100644 --- a/video/end_to_end_tests/resolution_bitrate_limits_tests.cc +++ b/video/end_to_end_tests/resolution_bitrate_limits_tests.cc @@ -129,6 +129,7 @@ class InitEncodeTest : public test::EndToEndTest, VideoSendStream::Config* send_config, std::vector* receive_configs, VideoEncoderConfig* encoder_config) override { + webrtc::VideoEncoder::EncoderInfo encoder_info; send_config->encoder_settings.encoder_factory = &encoder_factory_; send_config->rtp.payload_name = payload_name_; send_config->rtp.payload_type = test::CallTest::kVideoSendPayloadType; @@ -137,7 +138,7 @@ class InitEncodeTest : public test::EndToEndTest, encoder_config->video_stream_factory = rtc::make_ref_counted( payload_name_, /*max qp*/ 0, /*screencast*/ false, - /*screenshare enabled*/ false); + /*screenshare enabled*/ false, encoder_info); encoder_config->max_bitrate_bps = -1; if (configs_.size() == 1 && configs_[0].bitrate.max) encoder_config->max_bitrate_bps = configs_[0].bitrate.max->bps(); diff --git a/video/end_to_end_tests/rtp_rtcp_tests.cc b/video/end_to_end_tests/rtp_rtcp_tests.cc index 621bb8ad18..c7a34485ee 100644 --- a/video/end_to_end_tests/rtp_rtcp_tests.cc +++ b/video/end_to_end_tests/rtp_rtcp_tests.cc @@ -138,11 +138,11 @@ void RtpRtcpEndToEndTest::TestRtpStatePreservation( private: std::vector CreateEncoderStreams( - int width, - int height, + int frame_width, + int frame_height, const VideoEncoderConfig& encoder_config) override { std::vector streams = - test::CreateVideoStreams(width, height, encoder_config); + test::CreateVideoStreams(frame_width, frame_height, encoder_config); if (encoder_config.number_of_streams > 1) { // Lower bitrates so that all streams send initially. diff --git a/video/video_quality_test.cc b/video/video_quality_test.cc index d0554319f6..21dfaa8ae8 100644 --- a/video/video_quality_test.cc +++ b/video/video_quality_test.cc @@ -108,13 +108,13 @@ class VideoStreamFactory private: std::vector CreateEncoderStreams( - int width, - int height, + int frame_width, + int frame_height, const VideoEncoderConfig& encoder_config) override { // The highest layer must match the incoming resolution. std::vector streams = streams_; - streams[streams_.size() - 1].height = height; - streams[streams_.size() - 1].width = width; + streams[streams_.size() - 1].height = frame_height; + streams[streams_.size() - 1].width = frame_width; streams[0].bitrate_priority = encoder_config.bitrate_priority; return streams; @@ -613,6 +613,7 @@ void VideoQualityTest::FillScalabilitySettings( const std::vector& sl_descriptors) { if (params->ss[video_idx].streams.empty() && params->ss[video_idx].infer_streams) { + webrtc::VideoEncoder::EncoderInfo encoder_info; webrtc::VideoEncoderConfig encoder_config; encoder_config.codec_type = PayloadStringToCodecType(params->video[video_idx].codec); @@ -629,11 +630,11 @@ void VideoQualityTest::FillScalabilitySettings( encoder_config.video_stream_factory = rtc::make_ref_counted( params->video[video_idx].codec, kDefaultMaxQp, - params->screenshare[video_idx].enabled, true); + params->screenshare[video_idx].enabled, true, encoder_info); params->ss[video_idx].streams = encoder_config.video_stream_factory->CreateEncoderStreams( - static_cast(params->video[video_idx].width), - static_cast(params->video[video_idx].height), encoder_config); + params->video[video_idx].width, params->video[video_idx].height, + encoder_config); } else { // Read VideoStream and SpatialLayer elements from a list of comma separated // lists. To use a default value for an element, use -1 or leave empty. @@ -707,6 +708,7 @@ void VideoQualityTest::SetupVideo(Transport* send_transport, RTC_CHECK(num_video_streams_ > 0); video_encoder_configs_.resize(num_video_streams_); std::string generic_codec_name; + webrtc::VideoEncoder::EncoderInfo encoder_info; for (size_t video_idx = 0; video_idx < num_video_streams_; ++video_idx) { video_send_configs_.push_back(VideoSendStream::Config(send_transport)); video_encoder_configs_.push_back(VideoEncoderConfig()); @@ -804,7 +806,7 @@ void VideoQualityTest::SetupVideo(Transport* send_transport, rtc::make_ref_counted( params_.video[video_idx].codec, params_.ss[video_idx].streams[0].max_qp, - params_.screenshare[video_idx].enabled, true); + params_.screenshare[video_idx].enabled, true, encoder_info); video_encoder_configs_[video_idx].spatial_layers = params_.ss[video_idx].spatial_layers; diff --git a/video/video_send_stream_tests.cc b/video/video_send_stream_tests.cc index 33a607bd1a..b34450a9d8 100644 --- a/video/video_send_stream_tests.cc +++ b/video/video_send_stream_tests.cc @@ -2625,11 +2625,11 @@ TEST_F(VideoSendStreamTest, TranslatesTwoLayerScreencastToTargetBitrate) { private: std::vector CreateEncoderStreams( - int width, - int height, + int frame_width, + int frame_height, const VideoEncoderConfig& encoder_config) override { std::vector streams = - test::CreateVideoStreams(width, height, encoder_config); + test::CreateVideoStreams(frame_width, frame_height, encoder_config); RTC_CHECK_GT(streams[0].max_bitrate_bps, kScreencastMaxTargetBitrateDeltaKbps); streams[0].target_bitrate_bps = @@ -4080,6 +4080,7 @@ void VideoSendStreamTest::TestTemporalLayers( VideoSendStream::Config* send_config, std::vector* receive_configs, VideoEncoderConfig* encoder_config) override { + webrtc::VideoEncoder::EncoderInfo encoder_info; send_config->encoder_settings.encoder_factory = encoder_factory_; send_config->rtp.payload_name = payload_name_; send_config->rtp.payload_type = test::CallTest::kVideoSendPayloadType; @@ -4088,7 +4089,7 @@ void VideoSendStreamTest::TestTemporalLayers( encoder_config->video_stream_factory = rtc::make_ref_counted( payload_name_, /*max_qp=*/56, /*is_screenshare=*/false, - /*conference_mode=*/false); + /*conference_mode=*/false, encoder_info); encoder_config->max_bitrate_bps = kMaxBitrateBps; if (absl::EqualsIgnoreCase(payload_name_, "VP9")) { encoder_config->encoder_specific_settings = rtc::make_ref_counted< diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 046148e06d..deebc87fbc 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -932,6 +932,7 @@ void VideoStreamEncoder::ReconfigureEncoder() { encoder_reset_required = true; } + // TODO(webrtc:14451) : Move AlignmentAdjuster into EncoderStreamFactory // Possibly adjusts scale_resolution_down_by in `encoder_config_` to limit the // alignment value. AlignmentAdjuster::GetAlignmentAndMaybeAdjustScaleFactors( @@ -948,12 +949,13 @@ void VideoStreamEncoder::ReconfigureEncoder() { encoder_config_.video_format.name, encoder_config_.max_qp, encoder_config_.content_type == webrtc::VideoEncoderConfig::ContentType::kScreen, - encoder_config_.legacy_conference_mode); + encoder_config_.legacy_conference_mode, encoder_->GetEncoderInfo()); streams = factory->CreateEncoderStreams( last_frame_info_->width, last_frame_info_->height, encoder_config_); } + // TODO(webrtc:14451) : Move AlignmentAdjuster into EncoderStreamFactory // Get alignment when actual number of layers are known. int alignment = AlignmentAdjuster::GetAlignmentAndMaybeAdjustScaleFactors( encoder_->GetEncoderInfo(), &encoder_config_, streams.size()); diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 961f592165..5271654ac9 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -500,11 +500,12 @@ class CroppingVideoStreamFactory private: std::vector CreateEncoderStreams( - int width, - int height, + int frame_width, + int frame_height, const VideoEncoderConfig& encoder_config) override { std::vector streams = test::CreateVideoStreams( - width - width % 4, height - height % 4, encoder_config); + frame_width - frame_width % 4, frame_height - frame_height % 4, + encoder_config); return streams; } }; @@ -2190,6 +2191,7 @@ TEST_F(VideoStreamEncoderTest, // Two streams, highest stream active. VideoEncoderConfig config; + webrtc::VideoEncoder::EncoderInfo encoder_info; const int kNumStreams = 2; test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config); config.max_bitrate_bps = 0; @@ -2198,7 +2200,7 @@ TEST_F(VideoStreamEncoderTest, config.video_stream_factory = rtc::make_ref_counted( "VP8", /*max qp*/ 56, /*screencast*/ false, - /*screenshare enabled*/ false); + /*screenshare enabled*/ false, encoder_info); video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength); // The encoder bitrate limits for 270p should be used. @@ -2254,6 +2256,7 @@ TEST_F(VideoStreamEncoderTest, DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) { // Two streams, highest stream active. VideoEncoderConfig config; + webrtc::VideoEncoder::EncoderInfo encoder_info; const int kNumStreams = 2; test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config); config.max_bitrate_bps = 0; @@ -2262,7 +2265,7 @@ TEST_F(VideoStreamEncoderTest, config.video_stream_factory = rtc::make_ref_counted( "VP8", /*max qp*/ 56, /*screencast*/ false, - /*screenshare enabled*/ false); + /*screenshare enabled*/ false, encoder_info); video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength); // Default bitrate limits for 270p should be used. @@ -2327,6 +2330,7 @@ TEST_F(VideoStreamEncoderTest, // Three streams, middle stream active. VideoEncoderConfig config; + webrtc::VideoEncoder::EncoderInfo encoder_info; const int kNumStreams = 3; test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config); config.simulcast_layers[0].active = false; @@ -2335,7 +2339,7 @@ TEST_F(VideoStreamEncoderTest, config.video_stream_factory = rtc::make_ref_counted( "VP8", /*max qp*/ 56, /*screencast*/ false, - /*screenshare enabled*/ false); + /*screenshare enabled*/ false, encoder_info); video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength); // The encoder bitrate limits for 360p should be used. @@ -2371,6 +2375,7 @@ TEST_F(VideoStreamEncoderTest, // Three streams, lowest stream active. VideoEncoderConfig config; + webrtc::VideoEncoder::EncoderInfo encoder_info; const int kNumStreams = 3; test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config); config.simulcast_layers[0].active = true; @@ -2379,7 +2384,7 @@ TEST_F(VideoStreamEncoderTest, config.video_stream_factory = rtc::make_ref_counted( "VP8", /*max qp*/ 56, /*screencast*/ false, - /*screenshare enabled*/ false); + /*screenshare enabled*/ false, encoder_info); video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength); // Resolution on lowest stream lower than 270p. The encoder limits not applied @@ -2407,6 +2412,7 @@ TEST_F(VideoStreamEncoderTest, // Two streams, highest stream active. VideoEncoderConfig config; + webrtc::VideoEncoder::EncoderInfo encoder_info; const int kNumStreams = 2; test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config); config.simulcast_layers[0].active = false; @@ -2415,7 +2421,7 @@ TEST_F(VideoStreamEncoderTest, config.video_stream_factory = rtc::make_ref_counted( "VP8", /*max qp*/ 56, /*screencast*/ false, - /*screenshare enabled*/ false); + /*screenshare enabled*/ false, encoder_info); video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength); // The encoder bitrate limits for 270p should be used. @@ -2496,6 +2502,7 @@ TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) { // Fill config with the scaling factor by which to reduce encoding size. const int num_streams = scale_factors_.size(); VideoEncoderConfig config; + webrtc::VideoEncoder::EncoderInfo encoder_info; test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config); for (int i = 0; i < num_streams; ++i) { config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i]; @@ -2503,7 +2510,7 @@ TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) { config.video_stream_factory = rtc::make_ref_counted( "VP8", /*max qp*/ 56, /*screencast*/ false, - /*screenshare enabled*/ false); + /*screenshare enabled*/ false, encoder_info); video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength); video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( @@ -5647,12 +5654,13 @@ TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) { // Trigger QVGA "singlecast" // Update the config. VideoEncoderConfig video_encoder_config; + webrtc::VideoEncoder::EncoderInfo encoder_info; test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3, &video_encoder_config); video_encoder_config.video_stream_factory = rtc::make_ref_counted( "VP8", /*max qp*/ 56, /*screencast*/ false, - /*screenshare enabled*/ false); + /*screenshare enabled*/ false, encoder_info); for (auto& layer : video_encoder_config.simulcast_layers) { layer.num_temporal_layers = 1; layer.max_framerate = kDefaultFramerate; @@ -7971,6 +7979,7 @@ TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) { const int number_layers = sizeof(downscale_factors) / sizeof(downscale_factors[0]); VideoEncoderConfig config; + webrtc::VideoEncoder::EncoderInfo encoder_info; test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config); for (int i = 0; i < number_layers; ++i) { config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i]; @@ -7979,7 +7988,7 @@ TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) { config.video_stream_factory = rtc::make_ref_counted( "VP8", /*max qp*/ 56, /*screencast*/ false, - /*screenshare enabled*/ false); + /*screenshare enabled*/ false, encoder_info); video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate, 0, 0, 0); @@ -8096,6 +8105,7 @@ TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) { kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]); VideoEncoderConfig config; + webrtc::VideoEncoder::EncoderInfo encoder_info; test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config); for (size_t i = 0; i < kNumSimulcastLayers; ++i) { config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i]; @@ -8104,7 +8114,7 @@ TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) { config.video_stream_factory = rtc::make_ref_counted( "VP8", /*max qp*/ 56, /*screencast*/ false, - /*screenshare enabled*/ false); + /*screenshare enabled*/ false, encoder_info); video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate, 0, 0, 0); @@ -8772,6 +8782,7 @@ TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) { kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]); VideoEncoderConfig config; + webrtc::VideoEncoder::EncoderInfo encoder_info; if (codec_type_ == VideoCodecType::kVideoCodecVP9) { test::FillEncoderConfiguration(codec_type_, 1, &config); config.max_bitrate_bps = kSimulcastTargetBitrate.bps(); @@ -8825,7 +8836,7 @@ TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) { rtc::make_ref_counted( CodecTypeToPayloadString(codec_type_), /*max qp*/ 56, /*screencast*/ false, - /*screenshare enabled*/ false); + /*screenshare enabled*/ false, encoder_info); video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate, 0, 0, 0); @@ -8969,13 +8980,15 @@ class ReconfigureEncoderTest : public VideoStreamEncoderTest { void ConfigureEncoder(const VideoStream& stream) { VideoEncoderConfig config; + webrtc::VideoEncoder::EncoderInfo encoder_info; + test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config); config.max_bitrate_bps = stream.max_bitrate_bps; config.simulcast_layers[0] = stream; config.video_stream_factory = rtc::make_ref_counted( /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false, - /*conference_mode=*/false); + /*conference_mode=*/false, encoder_info); video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength); }