Add support for reducing number of spatial layers via scalability mode.
Bug: webrtc:13960 Change-Id: Icf31d2e327e363dac24245cb5c9fc14cbaa9b3b4 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/275942 Reviewed-by: Sergey Silkin <ssilkin@webrtc.org> Commit-Queue: Åsa Persson <asapersson@webrtc.org> Cr-Commit-Position: refs/heads/main@{#38454}
This commit is contained in:
Åsa Persson
committed by
WebRTC LUCI CQ
WebRTC LUCI CQ
parent
1c8103d4db
commit
b8a4daa31c
@ -172,6 +172,21 @@ std::vector<SpatialLayer> GetVp9SvcConfig(VideoCodec& codec) {
|
||||
absl::optional<ScalabilityMode> scalability_mode = codec.GetScalabilityMode();
|
||||
RTC_DCHECK(scalability_mode.has_value());
|
||||
|
||||
// Limit number of spatial layers for given resolution.
|
||||
int limited_num_spatial_layers =
|
||||
GetLimitedNumSpatialLayers(codec.width, codec.height);
|
||||
if (limited_num_spatial_layers <
|
||||
ScalabilityModeToNumSpatialLayers(*scalability_mode)) {
|
||||
ScalabilityMode limited_scalability_mode =
|
||||
LimitNumSpatialLayers(*scalability_mode, limited_num_spatial_layers);
|
||||
RTC_LOG(LS_WARNING)
|
||||
<< "Reducing number of spatial layers due to low input resolution: "
|
||||
<< ScalabilityModeToString(*scalability_mode) << " to "
|
||||
<< ScalabilityModeToString(limited_scalability_mode);
|
||||
scalability_mode = limited_scalability_mode;
|
||||
codec.SetScalabilityMode(limited_scalability_mode);
|
||||
}
|
||||
|
||||
absl::optional<ScalableVideoController::StreamLayersConfig> info =
|
||||
ScalabilityStructureConfig(*scalability_mode);
|
||||
if (!info.has_value()) {
|
||||
@ -180,16 +195,6 @@ std::vector<SpatialLayer> GetVp9SvcConfig(VideoCodec& codec) {
|
||||
return {};
|
||||
}
|
||||
|
||||
if (static_cast<int>(GetLimitedNumSpatialLayers(codec.width, codec.height)) <
|
||||
info->num_spatial_layers) {
|
||||
// Layers will be reduced, do not use scalability mode for now.
|
||||
// TODO(bugs.webrtc.org/11607): Use a lower scalability mode once all lower
|
||||
// modes are supported.
|
||||
codec.UnsetScalabilityMode();
|
||||
codec.VP9()->interLayerPred =
|
||||
ScalabilityModeToInterLayerPredMode(*scalability_mode);
|
||||
}
|
||||
|
||||
// TODO(bugs.webrtc.org/11607): Add support for screensharing.
|
||||
std::vector<SpatialLayer> spatial_layers =
|
||||
GetSvcConfig(codec.width, codec.height, codec.maxFramerate,
|
||||
|
||||
@ -72,15 +72,14 @@ TEST(SvcConfig, NumSpatialLayersLimitedWithScalabilityMode) {
|
||||
codec.height = 270;
|
||||
codec.SetScalabilityMode(ScalabilityMode::kL3T3_KEY);
|
||||
|
||||
// Scalability mode reset, configuration should be in accordance to L2T3_KEY.
|
||||
// Scalability mode updated.
|
||||
std::vector<SpatialLayer> spatial_layers = GetVp9SvcConfig(codec);
|
||||
EXPECT_THAT(spatial_layers, ElementsAre(Field(&SpatialLayer::height, 135),
|
||||
Field(&SpatialLayer::height, 270)));
|
||||
EXPECT_THAT(spatial_layers,
|
||||
ElementsAre(Field(&SpatialLayer::numberOfTemporalLayers, 3),
|
||||
Field(&SpatialLayer::numberOfTemporalLayers, 3)));
|
||||
EXPECT_EQ(codec.VP9()->interLayerPred, InterLayerPredMode::kOnKeyPic);
|
||||
EXPECT_EQ(codec.GetScalabilityMode(), absl::nullopt);
|
||||
EXPECT_EQ(codec.GetScalabilityMode(), ScalabilityMode::kL2T3_KEY);
|
||||
}
|
||||
|
||||
TEST(SvcConfig, NumSpatialLayersLimitedWithScalabilityModePortrait) {
|
||||
@ -90,15 +89,14 @@ TEST(SvcConfig, NumSpatialLayersLimitedWithScalabilityModePortrait) {
|
||||
codec.height = 480;
|
||||
codec.SetScalabilityMode(ScalabilityMode::kL3T1);
|
||||
|
||||
// Scalability mode reset, configuration should be in accordance to L2T1.
|
||||
// Scalability mode updated.
|
||||
std::vector<SpatialLayer> spatial_layers = GetVp9SvcConfig(codec);
|
||||
EXPECT_THAT(spatial_layers, ElementsAre(Field(&SpatialLayer::width, 135),
|
||||
Field(&SpatialLayer::width, 270)));
|
||||
EXPECT_THAT(spatial_layers,
|
||||
ElementsAre(Field(&SpatialLayer::numberOfTemporalLayers, 1),
|
||||
Field(&SpatialLayer::numberOfTemporalLayers, 1)));
|
||||
EXPECT_EQ(codec.VP9()->interLayerPred, InterLayerPredMode::kOn);
|
||||
EXPECT_EQ(codec.GetScalabilityMode(), absl::nullopt);
|
||||
EXPECT_EQ(codec.GetScalabilityMode(), ScalabilityMode::kL2T1);
|
||||
}
|
||||
|
||||
TEST(SvcConfig, NumSpatialLayersWithScalabilityModeResolutionRatio1_5) {
|
||||
@ -122,15 +120,14 @@ TEST(SvcConfig, NumSpatialLayersLimitedWithScalabilityModeResolutionRatio1_5) {
|
||||
codec.codecType = kVideoCodecVP9;
|
||||
codec.width = 320;
|
||||
codec.height = 180;
|
||||
codec.SetScalabilityMode(ScalabilityMode::kL2T1h); // 1.5:1
|
||||
codec.SetScalabilityMode(ScalabilityMode::kL3T1h); // 1.5:1
|
||||
|
||||
// Scalability mode reset, configuration should be in accordance to L1T1.
|
||||
// Scalability mode updated.
|
||||
std::vector<SpatialLayer> spatial_layers = GetVp9SvcConfig(codec);
|
||||
EXPECT_THAT(spatial_layers, ElementsAre(Field(&SpatialLayer::width, 320)));
|
||||
EXPECT_THAT(spatial_layers,
|
||||
ElementsAre(Field(&SpatialLayer::numberOfTemporalLayers, 1)));
|
||||
EXPECT_EQ(codec.VP9()->interLayerPred, InterLayerPredMode::kOn);
|
||||
EXPECT_EQ(codec.GetScalabilityMode(), absl::nullopt);
|
||||
EXPECT_EQ(codec.GetScalabilityMode(), ScalabilityMode::kL1T1);
|
||||
}
|
||||
|
||||
TEST(SvcConfig, AlwaysSendsAtLeastOneLayer) {
|
||||
|
||||
@ -292,4 +292,99 @@ absl::optional<ScalabilityModeResolutionRatio> ScalabilityModeToResolutionRatio(
|
||||
RTC_CHECK_NOTREACHED();
|
||||
}
|
||||
|
||||
ScalabilityMode LimitNumSpatialLayers(ScalabilityMode scalability_mode,
|
||||
int max_spatial_layers) {
|
||||
int num_spatial_layers = ScalabilityModeToNumSpatialLayers(scalability_mode);
|
||||
if (max_spatial_layers >= num_spatial_layers) {
|
||||
return scalability_mode;
|
||||
}
|
||||
|
||||
switch (scalability_mode) {
|
||||
case ScalabilityMode::kL1T1:
|
||||
return ScalabilityMode::kL1T1;
|
||||
case ScalabilityMode::kL1T2:
|
||||
return ScalabilityMode::kL1T2;
|
||||
case ScalabilityMode::kL1T3:
|
||||
return ScalabilityMode::kL1T3;
|
||||
case ScalabilityMode::kL2T1:
|
||||
return ScalabilityMode::kL1T1;
|
||||
case ScalabilityMode::kL2T1h:
|
||||
return ScalabilityMode::kL1T1;
|
||||
case ScalabilityMode::kL2T1_KEY:
|
||||
return ScalabilityMode::kL1T1;
|
||||
case ScalabilityMode::kL2T2:
|
||||
return ScalabilityMode::kL1T2;
|
||||
case ScalabilityMode::kL2T2h:
|
||||
return ScalabilityMode::kL1T2;
|
||||
case ScalabilityMode::kL2T2_KEY:
|
||||
return ScalabilityMode::kL1T2;
|
||||
case ScalabilityMode::kL2T2_KEY_SHIFT:
|
||||
return ScalabilityMode::kL1T2;
|
||||
case ScalabilityMode::kL2T3:
|
||||
return ScalabilityMode::kL1T3;
|
||||
case ScalabilityMode::kL2T3h:
|
||||
return ScalabilityMode::kL1T3;
|
||||
case ScalabilityMode::kL2T3_KEY:
|
||||
return ScalabilityMode::kL1T3;
|
||||
case ScalabilityMode::kL3T1:
|
||||
return max_spatial_layers == 2 ? ScalabilityMode::kL2T1
|
||||
: ScalabilityMode::kL1T1;
|
||||
case ScalabilityMode::kL3T1h:
|
||||
return max_spatial_layers == 2 ? ScalabilityMode::kL2T1h
|
||||
: ScalabilityMode::kL1T1;
|
||||
case ScalabilityMode::kL3T1_KEY:
|
||||
return max_spatial_layers == 2 ? ScalabilityMode::kL2T1_KEY
|
||||
: ScalabilityMode::kL1T1;
|
||||
case ScalabilityMode::kL3T2:
|
||||
return max_spatial_layers == 2 ? ScalabilityMode::kL2T2
|
||||
: ScalabilityMode::kL1T2;
|
||||
case ScalabilityMode::kL3T2h:
|
||||
return max_spatial_layers == 2 ? ScalabilityMode::kL2T2h
|
||||
: ScalabilityMode::kL1T2;
|
||||
case ScalabilityMode::kL3T2_KEY:
|
||||
return max_spatial_layers == 2 ? ScalabilityMode::kL2T2_KEY
|
||||
: ScalabilityMode::kL1T2;
|
||||
case ScalabilityMode::kL3T3:
|
||||
return max_spatial_layers == 2 ? ScalabilityMode::kL2T3
|
||||
: ScalabilityMode::kL1T3;
|
||||
case ScalabilityMode::kL3T3h:
|
||||
return max_spatial_layers == 2 ? ScalabilityMode::kL2T3h
|
||||
: ScalabilityMode::kL1T3;
|
||||
case ScalabilityMode::kL3T3_KEY:
|
||||
return max_spatial_layers == 2 ? ScalabilityMode::kL2T3_KEY
|
||||
: ScalabilityMode::kL1T3;
|
||||
case ScalabilityMode::kS2T1:
|
||||
return ScalabilityMode::kL1T1;
|
||||
case ScalabilityMode::kS2T1h:
|
||||
return ScalabilityMode::kL1T1;
|
||||
case ScalabilityMode::kS2T2:
|
||||
return ScalabilityMode::kL1T2;
|
||||
case ScalabilityMode::kS2T2h:
|
||||
return ScalabilityMode::kL1T2;
|
||||
case ScalabilityMode::kS2T3:
|
||||
return ScalabilityMode::kL1T3;
|
||||
case ScalabilityMode::kS2T3h:
|
||||
return ScalabilityMode::kL1T3;
|
||||
case ScalabilityMode::kS3T1:
|
||||
return max_spatial_layers == 2 ? ScalabilityMode::kS2T1
|
||||
: ScalabilityMode::kL1T1;
|
||||
case ScalabilityMode::kS3T1h:
|
||||
return max_spatial_layers == 2 ? ScalabilityMode::kS2T1h
|
||||
: ScalabilityMode::kL1T1;
|
||||
case ScalabilityMode::kS3T2:
|
||||
return max_spatial_layers == 2 ? ScalabilityMode::kS2T2
|
||||
: ScalabilityMode::kL1T2;
|
||||
case ScalabilityMode::kS3T2h:
|
||||
return max_spatial_layers == 2 ? ScalabilityMode::kS2T2h
|
||||
: ScalabilityMode::kL1T2;
|
||||
case ScalabilityMode::kS3T3:
|
||||
return max_spatial_layers == 2 ? ScalabilityMode::kS2T3
|
||||
: ScalabilityMode::kL1T3;
|
||||
case ScalabilityMode::kS3T3h:
|
||||
return max_spatial_layers == 2 ? ScalabilityMode::kS2T3h
|
||||
: ScalabilityMode::kL1T3;
|
||||
}
|
||||
RTC_CHECK_NOTREACHED();
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
@ -36,6 +36,9 @@ int ScalabilityModeToNumTemporalLayers(ScalabilityMode scalability_mode);
|
||||
absl::optional<ScalabilityModeResolutionRatio> ScalabilityModeToResolutionRatio(
|
||||
ScalabilityMode scalability_mode);
|
||||
|
||||
ScalabilityMode LimitNumSpatialLayers(ScalabilityMode scalability_mode,
|
||||
int max_spatial_layers);
|
||||
|
||||
} // namespace webrtc
|
||||
|
||||
#endif // MODULES_VIDEO_CODING_SVC_SCALABILITY_MODE_UTIL_H_
|
||||
|
||||
@ -10,6 +10,10 @@
|
||||
|
||||
#include "modules/video_coding/svc/scalability_mode_util.h"
|
||||
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/video_codecs/scalability_mode.h"
|
||||
@ -30,7 +34,7 @@ TEST(ScalabilityModeUtil, RejectsUnknownString) {
|
||||
|
||||
// Check roundtrip conversion of all enum values.
|
||||
TEST(ScalabilityModeUtil, ConvertsAllToAndFromString) {
|
||||
const ScalabilityMode kLastEnum = ScalabilityMode::kS3T3;
|
||||
const ScalabilityMode kLastEnum = ScalabilityMode::kS3T3h;
|
||||
for (int numerical_enum = 0; numerical_enum <= static_cast<int>(kLastEnum);
|
||||
numerical_enum++) {
|
||||
ScalabilityMode scalability_mode =
|
||||
@ -43,5 +47,70 @@ TEST(ScalabilityModeUtil, ConvertsAllToAndFromString) {
|
||||
}
|
||||
}
|
||||
|
||||
struct TestParams {
|
||||
std::string scalability_mode;
|
||||
std::vector<std::tuple<std::vector<int>, std::string>>
|
||||
limited_scalability_mode;
|
||||
};
|
||||
|
||||
class NumSpatialLayersTest : public ::testing::TestWithParam<TestParams> {};
|
||||
|
||||
INSTANTIATE_TEST_SUITE_P(
|
||||
MaxLayers,
|
||||
NumSpatialLayersTest,
|
||||
::testing::ValuesIn<TestParams>(
|
||||
{{"L1T1", {{{0, 1}, "L1T1"}, {{2}, "L1T1"}, {{3}, "L1T1"}}},
|
||||
{"L1T2", {{{0, 1}, "L1T2"}, {{2}, "L1T2"}, {{3}, "L1T2"}}},
|
||||
{"L1T3", {{{0, 1}, "L1T3"}, {{2}, "L1T3"}, {{3}, "L1T3"}}},
|
||||
{"L2T1", {{{0, 1}, "L1T1"}, {{2}, "L2T1"}, {{3}, "L2T1"}}},
|
||||
{"L2T1h", {{{0, 1}, "L1T1"}, {{2}, "L2T1h"}, {{3}, "L2T1h"}}},
|
||||
{"L2T1_KEY", {{{0, 1}, "L1T1"}, {{2}, "L2T1_KEY"}, {{3}, "L2T1_KEY"}}},
|
||||
{"L2T2", {{{0, 1}, "L1T2"}, {{2}, "L2T2"}, {{3}, "L2T2"}}},
|
||||
{"L2T2h", {{{0, 1}, "L1T2"}, {{2}, "L2T2h"}, {{3}, "L2T2h"}}},
|
||||
{"L2T2_KEY", {{{0, 1}, "L1T2"}, {{2}, "L2T2_KEY"}, {{3}, "L2T2_KEY"}}},
|
||||
{"L2T2_KEY_SHIFT",
|
||||
{{{0, 1}, "L1T2"}, {{2}, "L2T2_KEY_SHIFT"}, {{3}, "L2T2_KEY_SHIFT"}}},
|
||||
{"L2T3", {{{0, 1}, "L1T3"}, {{2}, "L2T3"}, {{3}, "L2T3"}}},
|
||||
{"L2T3h", {{{0, 1}, "L1T3"}, {{2}, "L2T3h"}, {{3}, "L2T3h"}}},
|
||||
{"L2T3_KEY", {{{0, 1}, "L1T3"}, {{2}, "L2T3_KEY"}, {{3}, "L2T3_KEY"}}},
|
||||
{"L3T1", {{{0, 1}, "L1T1"}, {{2}, "L2T1"}, {{3}, "L3T1"}}},
|
||||
{"L3T1h", {{{0, 1}, "L1T1"}, {{2}, "L2T1h"}, {{3}, "L3T1h"}}},
|
||||
{"L3T1_KEY", {{{0, 1}, "L1T1"}, {{2}, "L2T1_KEY"}, {{3}, "L3T1_KEY"}}},
|
||||
{"L3T2", {{{0, 1}, "L1T2"}, {{2}, "L2T2"}, {{3}, "L3T2"}}},
|
||||
{"L3T2h", {{{0, 1}, "L1T2"}, {{2}, "L2T2h"}, {{3}, "L3T2h"}}},
|
||||
{"L3T2_KEY", {{{0, 1}, "L1T2"}, {{2}, "L2T2_KEY"}, {{3}, "L3T2_KEY"}}},
|
||||
{"L3T3", {{{0, 1}, "L1T3"}, {{2}, "L2T3"}, {{3}, "L3T3"}}},
|
||||
{"L3T3h", {{{0, 1}, "L1T3"}, {{2}, "L2T3h"}, {{3}, "L3T3h"}}},
|
||||
{"L3T3_KEY", {{{0, 1}, "L1T3"}, {{2}, "L2T3_KEY"}, {{3}, "L3T3_KEY"}}},
|
||||
{"S2T1", {{{0, 1}, "L1T1"}, {{2}, "S2T1"}, {{3}, "S2T1"}}},
|
||||
{"S2T1h", {{{0, 1}, "L1T1"}, {{2}, "S2T1h"}, {{3}, "S2T1h"}}},
|
||||
{"S2T2", {{{0, 1}, "L1T2"}, {{2}, "S2T2"}, {{3}, "S2T2"}}},
|
||||
{"S2T2h", {{{0, 1}, "L1T2"}, {{2}, "S2T2h"}, {{3}, "S2T2h"}}},
|
||||
{"S2T3", {{{0, 1}, "L1T3"}, {{2}, "S2T3"}, {{3}, "S2T3"}}},
|
||||
{"S2T3h", {{{0, 1}, "L1T3"}, {{2}, "S2T3h"}, {{3}, "S2T3h"}}},
|
||||
{"S3T1", {{{0, 1}, "L1T1"}, {{2}, "S2T1"}, {{3}, "S3T1"}}},
|
||||
{"S3T1h", {{{0, 1}, "L1T1"}, {{2}, "S2T1h"}, {{3}, "S3T1h"}}},
|
||||
{"S3T2", {{{0, 1}, "L1T2"}, {{2}, "S2T2"}, {{3}, "S3T2"}}},
|
||||
{"S3T2h", {{{0, 1}, "L1T2"}, {{2}, "S2T2h"}, {{3}, "S3T2h"}}},
|
||||
{"S3T3", {{{0, 1}, "L1T3"}, {{2}, "S2T3"}, {{3}, "S3T3"}}},
|
||||
{"S3T3h", {{{0, 1}, "L1T3"}, {{2}, "S2T3h"}, {{3}, "S3T3h"}}}}),
|
||||
[](const ::testing::TestParamInfo<TestParams>& info) {
|
||||
return info.param.scalability_mode;
|
||||
});
|
||||
|
||||
TEST_P(NumSpatialLayersTest, LimitsSpatialLayers) {
|
||||
const ScalabilityMode mode =
|
||||
*ScalabilityModeFromString(GetParam().scalability_mode);
|
||||
for (const auto& param : GetParam().limited_scalability_mode) {
|
||||
const std::vector<int> max_num_spatial_layers =
|
||||
std::get<std::vector<int>>(param);
|
||||
const ScalabilityMode expected_mode =
|
||||
*ScalabilityModeFromString(std::get<std::string>(param));
|
||||
for (const auto& max_layers : max_num_spatial_layers) {
|
||||
EXPECT_EQ(expected_mode, LimitNumSpatialLayers(mode, max_layers));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace webrtc
|
||||
|
||||
Reference in New Issue
Block a user