diff --git a/common_types.h b/common_types.h index c1c5b4e1eb..beb6bcaf50 100644 --- a/common_types.h +++ b/common_types.h @@ -361,6 +361,7 @@ struct SpatialLayer { unsigned short width; unsigned short height; + float maxFramerate; // fps. unsigned char numberOfTemporalLayers; unsigned int maxBitrate; // kilobits/sec. unsigned int targetBitrate; // kilobits/sec. diff --git a/media/engine/vp8_encoder_simulcast_proxy_unittest.cc b/media/engine/vp8_encoder_simulcast_proxy_unittest.cc index dfa35cbaf6..47e57aa85e 100644 --- a/media/engine/vp8_encoder_simulcast_proxy_unittest.cc +++ b/media/engine/vp8_encoder_simulcast_proxy_unittest.cc @@ -63,12 +63,30 @@ TEST(VP8EncoderSimulcastProxy, ChoosesCorrectImplementation) { "SimulcastEncoderAdapter (Fake, Fake, Fake)"; VideoCodec codec_settings; webrtc::test::CodecSettings(kVideoCodecVP8, &codec_settings); - codec_settings.simulcastStream[0] = { - test::kTestWidth, test::kTestHeight, 2, 2000, 1000, 1000, 56}; - codec_settings.simulcastStream[1] = { - test::kTestWidth, test::kTestHeight, 2, 3000, 1000, 1000, 56}; - codec_settings.simulcastStream[2] = { - test::kTestWidth, test::kTestHeight, 2, 5000, 1000, 1000, 56}; + codec_settings.simulcastStream[0] = {test::kTestWidth, + test::kTestHeight, + test::kTestFrameRate, + 2, + 2000, + 1000, + 1000, + 56}; + codec_settings.simulcastStream[1] = {test::kTestWidth, + test::kTestHeight, + test::kTestFrameRate, + 2, + 3000, + 1000, + 1000, + 56}; + codec_settings.simulcastStream[2] = {test::kTestWidth, + test::kTestHeight, + test::kTestFrameRate, + 2, + 5000, + 1000, + 1000, + 56}; codec_settings.numberOfSimulcastStreams = 3; NiceMock* mock_encoder = new NiceMock(); diff --git a/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc index a5c76ad5eb..2958a9af50 100644 --- a/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc +++ b/modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc @@ -70,11 +70,11 @@ void ConfigureSimulcast(VideoCodec* codec_settings) { void ConfigureSvc(VideoCodec* codec_settings) { RTC_CHECK_EQ(kVideoCodecVP9, codec_settings->codecType); - const std::vector layers = - GetSvcConfig(codec_settings->width, codec_settings->height, - codec_settings->VP9()->numberOfSpatialLayers, - codec_settings->VP9()->numberOfTemporalLayers, - /* is_screen_sharing = */ false); + const std::vector layers = GetSvcConfig( + codec_settings->width, codec_settings->height, kMaxFramerateFps, + codec_settings->VP9()->numberOfSpatialLayers, + codec_settings->VP9()->numberOfTemporalLayers, + /* is_screen_sharing = */ false); ASSERT_EQ(codec_settings->VP9()->numberOfSpatialLayers, layers.size()) << "GetSvcConfig returned fewer spatial layers than configured."; diff --git a/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc b/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc index be92b342fc..aaa10dd753 100644 --- a/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc +++ b/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc @@ -31,6 +31,7 @@ constexpr size_t kMaxPayloadSize = 1440; constexpr int kDefaultMinPixelsPerFrame = 320 * 180; constexpr int kWidth = 172; constexpr int kHeight = 144; +constexpr float kFramerateFps = 30; } // namespace class TestVp8Impl : public VideoCodecUnitTest { @@ -182,57 +183,57 @@ TEST_F(TestVp8Impl, DecodedQpEqualsEncodedQp) { TEST_F(TestVp8Impl, ChecksSimulcastSettings) { codec_settings_.numberOfSimulcastStreams = 2; // Reslutions are not scaled by 2, temporal layers do not match. - codec_settings_.simulcastStream[0] = {kWidth, kHeight, 2, 4000, - 3000, 2000, 80}; - codec_settings_.simulcastStream[1] = {kWidth, kHeight, 3, 4000, - 3000, 2000, 80}; + codec_settings_.simulcastStream[0] = {kWidth, kHeight, kFramerateFps, 2, + 4000, 3000, 2000, 80}; + codec_settings_.simulcastStream[1] = {kWidth, kHeight, 30, 3, + 4000, 3000, 2000, 80}; EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED, encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize)); codec_settings_.numberOfSimulcastStreams = 3; // Reslutions are not scaled by 2. - codec_settings_.simulcastStream[0] = {kWidth / 2, kHeight / 2, 1, 4000, - 3000, 2000, 80}; - codec_settings_.simulcastStream[1] = {kWidth / 2, kHeight / 2, 1, 4000, - 3000, 2000, 80}; - codec_settings_.simulcastStream[2] = {kWidth, kHeight, 1, 4000, - 3000, 2000, 80}; + codec_settings_.simulcastStream[0] = { + kWidth / 2, kHeight / 2, kFramerateFps, 1, 4000, 3000, 2000, 80}; + codec_settings_.simulcastStream[1] = { + kWidth / 2, kHeight / 2, kFramerateFps, 1, 4000, 3000, 2000, 80}; + codec_settings_.simulcastStream[2] = {kWidth, kHeight, 30, 1, + 4000, 3000, 2000, 80}; EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED, encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize)); // Reslutions are not scaled by 2. - codec_settings_.simulcastStream[0] = {kWidth, kHeight, 1, 4000, - 3000, 2000, 80}; - codec_settings_.simulcastStream[1] = {kWidth, kHeight, 1, 4000, - 3000, 2000, 80}; - codec_settings_.simulcastStream[2] = {kWidth, kHeight, 1, 4000, - 3000, 2000, 80}; + codec_settings_.simulcastStream[0] = {kWidth, kHeight, kFramerateFps, 1, + 4000, 3000, 2000, 80}; + codec_settings_.simulcastStream[1] = {kWidth, kHeight, kFramerateFps, 1, + 4000, 3000, 2000, 80}; + codec_settings_.simulcastStream[2] = {kWidth, kHeight, kFramerateFps, 1, + 4000, 3000, 2000, 80}; EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED, encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize)); // Temporal layers do not match. - codec_settings_.simulcastStream[0] = {kWidth / 4, kHeight / 4, 1, 4000, - 3000, 2000, 80}; - codec_settings_.simulcastStream[1] = {kWidth / 2, kHeight / 2, 2, 4000, - 3000, 2000, 80}; - codec_settings_.simulcastStream[2] = {kWidth, kHeight, 3, 4000, - 3000, 2000, 80}; + codec_settings_.simulcastStream[0] = { + kWidth / 4, kHeight / 4, kFramerateFps, 1, 4000, 3000, 2000, 80}; + codec_settings_.simulcastStream[1] = { + kWidth / 2, kHeight / 2, kFramerateFps, 2, 4000, 3000, 2000, 80}; + codec_settings_.simulcastStream[2] = {kWidth, kHeight, kFramerateFps, 3, + 4000, 3000, 2000, 80}; EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED, encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize)); // Resolutions do not match codec config. codec_settings_.simulcastStream[0] = { - kWidth / 4 + 1, kHeight / 4 + 1, 1, 4000, 3000, 2000, 80}; + kWidth / 4 + 1, kHeight / 4 + 1, kFramerateFps, 1, 4000, 3000, 2000, 80}; codec_settings_.simulcastStream[1] = { - kWidth / 2 + 2, kHeight / 2 + 2, 1, 4000, 3000, 2000, 80}; - codec_settings_.simulcastStream[2] = {kWidth + 4, kHeight + 4, 1, 4000, - 3000, 2000, 80}; + kWidth / 2 + 2, kHeight / 2 + 2, kFramerateFps, 1, 4000, 3000, 2000, 80}; + codec_settings_.simulcastStream[2] = { + kWidth + 4, kHeight + 4, kFramerateFps, 1, 4000, 3000, 2000, 80}; EXPECT_EQ(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED, encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize)); // Everything fine: scaling by 2, top resolution matches video, temporal // settings are the same for all layers. - codec_settings_.simulcastStream[0] = {kWidth / 4, kHeight / 4, 1, 4000, - 3000, 2000, 80}; - codec_settings_.simulcastStream[1] = {kWidth / 2, kHeight / 2, 1, 4000, - 3000, 2000, 80}; - codec_settings_.simulcastStream[2] = {kWidth, kHeight, 1, 4000, - 3000, 2000, 80}; + codec_settings_.simulcastStream[0] = { + kWidth / 4, kHeight / 4, kFramerateFps, 1, 4000, 3000, 2000, 80}; + codec_settings_.simulcastStream[1] = { + kWidth / 2, kHeight / 2, kFramerateFps, 1, 4000, 3000, 2000, 80}; + codec_settings_.simulcastStream[2] = {kWidth, kHeight, kFramerateFps, 1, + 4000, 3000, 2000, 80}; EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize)); } diff --git a/modules/video_coding/codecs/vp9/svc_config.cc b/modules/video_coding/codecs/vp9/svc_config.cc index 22902fecea..6807698246 100644 --- a/modules/video_coding/codecs/vp9/svc_config.cc +++ b/modules/video_coding/codecs/vp9/svc_config.cc @@ -23,11 +23,13 @@ namespace { const size_t kMinVp9SvcBitrateKbps = 30; const size_t kMaxNumLayersForScreenSharing = 2; +const float kMaxScreenSharingLayerFramerateFps[] = {5.0, 5.0}; const size_t kMaxScreenSharingLayerBitrateKbps[] = {200, 500}; } // namespace std::vector ConfigureSvcScreenSharing(size_t input_width, size_t input_height, + float max_framerate_fps, size_t num_spatial_layers) { num_spatial_layers = std::min(num_spatial_layers, kMaxNumLayersForScreenSharing); @@ -37,6 +39,8 @@ std::vector ConfigureSvcScreenSharing(size_t input_width, SpatialLayer spatial_layer = {0}; spatial_layer.width = input_width; spatial_layer.height = input_height; + spatial_layer.maxFramerate = + std::min(kMaxScreenSharingLayerFramerateFps[sl_idx], max_framerate_fps); spatial_layer.numberOfTemporalLayers = 1; spatial_layer.minBitrate = static_cast(kMinVp9SvcBitrateKbps); spatial_layer.maxBitrate = @@ -50,6 +54,7 @@ std::vector ConfigureSvcScreenSharing(size_t input_width, std::vector ConfigureSvcNormalVideo(size_t input_width, size_t input_height, + float max_framerate_fps, size_t num_spatial_layers, size_t num_temporal_layers) { std::vector spatial_layers; @@ -69,6 +74,7 @@ std::vector ConfigureSvcNormalVideo(size_t input_width, SpatialLayer spatial_layer = {0}; spatial_layer.width = input_width >> (num_spatial_layers - sl_idx - 1); spatial_layer.height = input_height >> (num_spatial_layers - sl_idx - 1); + spatial_layer.maxFramerate = max_framerate_fps; spatial_layer.numberOfTemporalLayers = num_temporal_layers; // minBitrate and maxBitrate formulas were derived from @@ -102,6 +108,7 @@ std::vector ConfigureSvcNormalVideo(size_t input_width, std::vector GetSvcConfig(size_t input_width, size_t input_height, + float max_framerate_fps, size_t num_spatial_layers, size_t num_temporal_layers, bool is_screen_sharing) { @@ -112,9 +119,9 @@ std::vector GetSvcConfig(size_t input_width, if (is_screen_sharing) { return ConfigureSvcScreenSharing(input_width, input_height, - num_spatial_layers); + max_framerate_fps, num_spatial_layers); } else { - return ConfigureSvcNormalVideo(input_width, input_height, + return ConfigureSvcNormalVideo(input_width, input_height, max_framerate_fps, num_spatial_layers, num_temporal_layers); } } diff --git a/modules/video_coding/codecs/vp9/svc_config.h b/modules/video_coding/codecs/vp9/svc_config.h index c8561a4c9d..f671cede98 100644 --- a/modules/video_coding/codecs/vp9/svc_config.h +++ b/modules/video_coding/codecs/vp9/svc_config.h @@ -18,6 +18,7 @@ namespace webrtc { std::vector GetSvcConfig(size_t input_width, size_t input_height, + float max_framerate_fps, size_t num_spatial_layers, size_t num_temporal_layers, bool is_screen_sharing); diff --git a/modules/video_coding/codecs/vp9/svc_config_unittest.cc b/modules/video_coding/codecs/vp9/svc_config_unittest.cc index ebefddab13..257c5df2f0 100644 --- a/modules/video_coding/codecs/vp9/svc_config_unittest.cc +++ b/modules/video_coding/codecs/vp9/svc_config_unittest.cc @@ -23,7 +23,7 @@ TEST(SvcConfig, NumSpatialLayers) { std::vector spatial_layers = GetSvcConfig(kMinVp9SpatialLayerWidth << (num_spatial_layers - 1), - kMinVp9SpatialLayerHeight << (num_spatial_layers - 1), + kMinVp9SpatialLayerHeight << (num_spatial_layers - 1), 30, max_num_spatial_layers, 1, false); EXPECT_EQ(spatial_layers.size(), num_spatial_layers); @@ -33,7 +33,7 @@ TEST(SvcConfig, BitrateThresholds) { const size_t num_spatial_layers = 3; std::vector spatial_layers = GetSvcConfig(kMinVp9SpatialLayerWidth << (num_spatial_layers - 1), - kMinVp9SpatialLayerHeight << (num_spatial_layers - 1), + kMinVp9SpatialLayerHeight << (num_spatial_layers - 1), 30, num_spatial_layers, 1, false); EXPECT_EQ(spatial_layers.size(), num_spatial_layers); @@ -47,13 +47,14 @@ TEST(SvcConfig, BitrateThresholds) { TEST(SvcConfig, ScreenSharing) { std::vector spatial_layers = - GetSvcConfig(1920, 1080, 3, 3, true); + GetSvcConfig(1920, 1080, 30, 3, 3, true); EXPECT_EQ(spatial_layers.size(), 2UL); for (const SpatialLayer& layer : spatial_layers) { EXPECT_EQ(layer.width, 1920); EXPECT_EQ(layer.height, 1080); + EXPECT_EQ(layer.maxFramerate, 5); EXPECT_EQ(layer.numberOfTemporalLayers, 1); EXPECT_LE(layer.minBitrate, layer.maxBitrate); EXPECT_LE(layer.minBitrate, layer.targetBitrate); diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc index 1b6aa10f4c..048bf7d694 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc @@ -29,7 +29,7 @@ static VideoCodec Configure(size_t width, : VideoCodecMode::kRealtimeVideo; std::vector spatial_layers = - GetSvcConfig(width, height, num_spatial_layers, num_temporal_layers, + GetSvcConfig(width, height, 30, num_spatial_layers, num_temporal_layers, is_screen_sharing); RTC_CHECK_LE(spatial_layers.size(), kMaxSpatialLayers); diff --git a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc index a6774a412d..5b5db88b6e 100644 --- a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc +++ b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc @@ -82,9 +82,9 @@ class TestVp9Impl : public VideoCodecUnitTest { codec_settings_.VP9()->numberOfTemporalLayers = 1; codec_settings_.VP9()->frameDroppingOn = false; - std::vector layers = - GetSvcConfig(codec_settings_.width, codec_settings_.height, - num_spatial_layers, 1, false); + std::vector layers = GetSvcConfig( + codec_settings_.width, codec_settings_.height, + codec_settings_.maxFramerate, num_spatial_layers, 1, false); for (size_t i = 0; i < layers.size(); ++i) { codec_settings_.spatialLayers[i] = layers[i]; } diff --git a/modules/video_coding/video_codec_initializer.cc b/modules/video_coding/video_codec_initializer.cc index fe59afc184..e7125ce278 100644 --- a/modules/video_coding/video_codec_initializer.cc +++ b/modules/video_coding/video_codec_initializer.cc @@ -126,6 +126,7 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec( sim_stream->width = static_cast(streams[i].width); sim_stream->height = static_cast(streams[i].height); + sim_stream->maxFramerate = streams[i].max_framerate; sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000; sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000; sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000; @@ -198,11 +199,11 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec( // Layering is set explicitly. spatial_layers = config.spatial_layers; } else { - spatial_layers = - GetSvcConfig(video_codec.width, video_codec.height, - video_codec.VP9()->numberOfSpatialLayers, - video_codec.VP9()->numberOfTemporalLayers, - video_codec.mode == VideoCodecMode::kScreensharing); + spatial_layers = GetSvcConfig( + video_codec.width, video_codec.height, video_codec.maxFramerate, + video_codec.VP9()->numberOfSpatialLayers, + video_codec.VP9()->numberOfTemporalLayers, + video_codec.mode == VideoCodecMode::kScreensharing); const bool no_spatial_layering = (spatial_layers.size() == 1); if (no_spatial_layering) {