Add support for reducing number of spatial layers via scalability mode.

Bug: webrtc:13960
Change-Id: Icf31d2e327e363dac24245cb5c9fc14cbaa9b3b4
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/275942
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Commit-Queue: Åsa Persson <asapersson@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38454}

modules/video_coding/codecs/vp9/svc_config.cc

@@ -172,6 +172,21 @@ std::vector<SpatialLayer> GetVp9SvcConfig(VideoCodec& codec) {
   absl::optional<ScalabilityMode> scalability_mode = codec.GetScalabilityMode();
   RTC_DCHECK(scalability_mode.has_value());
 
+  // Limit number of spatial layers for given resolution.
+  int limited_num_spatial_layers =
+      GetLimitedNumSpatialLayers(codec.width, codec.height);
+  if (limited_num_spatial_layers <
+      ScalabilityModeToNumSpatialLayers(*scalability_mode)) {
+    ScalabilityMode limited_scalability_mode =
+        LimitNumSpatialLayers(*scalability_mode, limited_num_spatial_layers);
+    RTC_LOG(LS_WARNING)
+        << "Reducing number of spatial layers due to low input resolution: "
+        << ScalabilityModeToString(*scalability_mode) << " to "
+        << ScalabilityModeToString(limited_scalability_mode);
+    scalability_mode = limited_scalability_mode;
+    codec.SetScalabilityMode(limited_scalability_mode);
+  }
+
   absl::optional<ScalableVideoController::StreamLayersConfig> info =
       ScalabilityStructureConfig(*scalability_mode);
   if (!info.has_value()) {
@@ -180,16 +195,6 @@ std::vector<SpatialLayer> GetVp9SvcConfig(VideoCodec& codec) {
     return {};
   }
 
-  if (static_cast<int>(GetLimitedNumSpatialLayers(codec.width, codec.height)) <
-      info->num_spatial_layers) {
-    // Layers will be reduced, do not use scalability mode for now.
-    // TODO(bugs.webrtc.org/11607): Use a lower scalability mode once all lower
-    // modes are supported.
-    codec.UnsetScalabilityMode();
-    codec.VP9()->interLayerPred =
-        ScalabilityModeToInterLayerPredMode(*scalability_mode);
-  }
-
   // TODO(bugs.webrtc.org/11607): Add support for screensharing.
   std::vector<SpatialLayer> spatial_layers =
       GetSvcConfig(codec.width, codec.height, codec.maxFramerate,

modules/video_coding/codecs/vp9/svc_config_unittest.cc

@@ -72,15 +72,14 @@ TEST(SvcConfig, NumSpatialLayersLimitedWithScalabilityMode) {
   codec.height = 270;
   codec.SetScalabilityMode(ScalabilityMode::kL3T3_KEY);
 
-  // Scalability mode reset, configuration should be in accordance to L2T3_KEY.
+  // Scalability mode updated.
   std::vector<SpatialLayer> spatial_layers = GetVp9SvcConfig(codec);
   EXPECT_THAT(spatial_layers, ElementsAre(Field(&SpatialLayer::height, 135),
                                           Field(&SpatialLayer::height, 270)));
   EXPECT_THAT(spatial_layers,
               ElementsAre(Field(&SpatialLayer::numberOfTemporalLayers, 3),
                           Field(&SpatialLayer::numberOfTemporalLayers, 3)));
-  EXPECT_EQ(codec.VP9()->interLayerPred, InterLayerPredMode::kOnKeyPic);
+  EXPECT_EQ(codec.GetScalabilityMode(), ScalabilityMode::kL2T3_KEY);
-  EXPECT_EQ(codec.GetScalabilityMode(), absl::nullopt);
 }
 
 TEST(SvcConfig, NumSpatialLayersLimitedWithScalabilityModePortrait) {
@@ -90,15 +89,14 @@ TEST(SvcConfig, NumSpatialLayersLimitedWithScalabilityModePortrait) {
   codec.height = 480;
   codec.SetScalabilityMode(ScalabilityMode::kL3T1);
 
-  // Scalability mode reset, configuration should be in accordance to L2T1.
+  // Scalability mode updated.
   std::vector<SpatialLayer> spatial_layers = GetVp9SvcConfig(codec);
   EXPECT_THAT(spatial_layers, ElementsAre(Field(&SpatialLayer::width, 135),
                                           Field(&SpatialLayer::width, 270)));
   EXPECT_THAT(spatial_layers,
               ElementsAre(Field(&SpatialLayer::numberOfTemporalLayers, 1),
                           Field(&SpatialLayer::numberOfTemporalLayers, 1)));
-  EXPECT_EQ(codec.VP9()->interLayerPred, InterLayerPredMode::kOn);
+  EXPECT_EQ(codec.GetScalabilityMode(), ScalabilityMode::kL2T1);
-  EXPECT_EQ(codec.GetScalabilityMode(), absl::nullopt);
 }
 
 TEST(SvcConfig, NumSpatialLayersWithScalabilityModeResolutionRatio1_5) {
@@ -122,15 +120,14 @@ TEST(SvcConfig, NumSpatialLayersLimitedWithScalabilityModeResolutionRatio1_5) {
   codec.codecType = kVideoCodecVP9;
   codec.width = 320;
   codec.height = 180;
-  codec.SetScalabilityMode(ScalabilityMode::kL2T1h);  // 1.5:1
+  codec.SetScalabilityMode(ScalabilityMode::kL3T1h);  // 1.5:1
 
-  // Scalability mode reset, configuration should be in accordance to L1T1.
+  // Scalability mode updated.
   std::vector<SpatialLayer> spatial_layers = GetVp9SvcConfig(codec);
   EXPECT_THAT(spatial_layers, ElementsAre(Field(&SpatialLayer::width, 320)));
   EXPECT_THAT(spatial_layers,
               ElementsAre(Field(&SpatialLayer::numberOfTemporalLayers, 1)));
-  EXPECT_EQ(codec.VP9()->interLayerPred, InterLayerPredMode::kOn);
+  EXPECT_EQ(codec.GetScalabilityMode(), ScalabilityMode::kL1T1);
-  EXPECT_EQ(codec.GetScalabilityMode(), absl::nullopt);
 }
 
 TEST(SvcConfig, AlwaysSendsAtLeastOneLayer) {

modules/video_coding/svc/scalability_mode_util.cc

@@ -292,4 +292,99 @@ absl::optional<ScalabilityModeResolutionRatio> ScalabilityModeToResolutionRatio(
   RTC_CHECK_NOTREACHED();
 }
 
+ScalabilityMode LimitNumSpatialLayers(ScalabilityMode scalability_mode,
+                                      int max_spatial_layers) {
+  int num_spatial_layers = ScalabilityModeToNumSpatialLayers(scalability_mode);
+  if (max_spatial_layers >= num_spatial_layers) {
+    return scalability_mode;
+  }
+
+  switch (scalability_mode) {
+    case ScalabilityMode::kL1T1:
+      return ScalabilityMode::kL1T1;
+    case ScalabilityMode::kL1T2:
+      return ScalabilityMode::kL1T2;
+    case ScalabilityMode::kL1T3:
+      return ScalabilityMode::kL1T3;
+    case ScalabilityMode::kL2T1:
+      return ScalabilityMode::kL1T1;
+    case ScalabilityMode::kL2T1h:
+      return ScalabilityMode::kL1T1;
+    case ScalabilityMode::kL2T1_KEY:
+      return ScalabilityMode::kL1T1;
+    case ScalabilityMode::kL2T2:
+      return ScalabilityMode::kL1T2;
+    case ScalabilityMode::kL2T2h:
+      return ScalabilityMode::kL1T2;
+    case ScalabilityMode::kL2T2_KEY:
+      return ScalabilityMode::kL1T2;
+    case ScalabilityMode::kL2T2_KEY_SHIFT:
+      return ScalabilityMode::kL1T2;
+    case ScalabilityMode::kL2T3:
+      return ScalabilityMode::kL1T3;
+    case ScalabilityMode::kL2T3h:
+      return ScalabilityMode::kL1T3;
+    case ScalabilityMode::kL2T3_KEY:
+      return ScalabilityMode::kL1T3;
+    case ScalabilityMode::kL3T1:
+      return max_spatial_layers == 2 ? ScalabilityMode::kL2T1
+                                     : ScalabilityMode::kL1T1;
+    case ScalabilityMode::kL3T1h:
+      return max_spatial_layers == 2 ? ScalabilityMode::kL2T1h
+                                     : ScalabilityMode::kL1T1;
+    case ScalabilityMode::kL3T1_KEY:
+      return max_spatial_layers == 2 ? ScalabilityMode::kL2T1_KEY
+                                     : ScalabilityMode::kL1T1;
+    case ScalabilityMode::kL3T2:
+      return max_spatial_layers == 2 ? ScalabilityMode::kL2T2
+                                     : ScalabilityMode::kL1T2;
+    case ScalabilityMode::kL3T2h:
+      return max_spatial_layers == 2 ? ScalabilityMode::kL2T2h
+                                     : ScalabilityMode::kL1T2;
+    case ScalabilityMode::kL3T2_KEY:
+      return max_spatial_layers == 2 ? ScalabilityMode::kL2T2_KEY
+                                     : ScalabilityMode::kL1T2;
+    case ScalabilityMode::kL3T3:
+      return max_spatial_layers == 2 ? ScalabilityMode::kL2T3
+                                     : ScalabilityMode::kL1T3;
+    case ScalabilityMode::kL3T3h:
+      return max_spatial_layers == 2 ? ScalabilityMode::kL2T3h
+                                     : ScalabilityMode::kL1T3;
+    case ScalabilityMode::kL3T3_KEY:
+      return max_spatial_layers == 2 ? ScalabilityMode::kL2T3_KEY
+                                     : ScalabilityMode::kL1T3;
+    case ScalabilityMode::kS2T1:
+      return ScalabilityMode::kL1T1;
+    case ScalabilityMode::kS2T1h:
+      return ScalabilityMode::kL1T1;
+    case ScalabilityMode::kS2T2:
+      return ScalabilityMode::kL1T2;
+    case ScalabilityMode::kS2T2h:
+      return ScalabilityMode::kL1T2;
+    case ScalabilityMode::kS2T3:
+      return ScalabilityMode::kL1T3;
+    case ScalabilityMode::kS2T3h:
+      return ScalabilityMode::kL1T3;
+    case ScalabilityMode::kS3T1:
+      return max_spatial_layers == 2 ? ScalabilityMode::kS2T1
+                                     : ScalabilityMode::kL1T1;
+    case ScalabilityMode::kS3T1h:
+      return max_spatial_layers == 2 ? ScalabilityMode::kS2T1h
+                                     : ScalabilityMode::kL1T1;
+    case ScalabilityMode::kS3T2:
+      return max_spatial_layers == 2 ? ScalabilityMode::kS2T2
+                                     : ScalabilityMode::kL1T2;
+    case ScalabilityMode::kS3T2h:
+      return max_spatial_layers == 2 ? ScalabilityMode::kS2T2h
+                                     : ScalabilityMode::kL1T2;
+    case ScalabilityMode::kS3T3:
+      return max_spatial_layers == 2 ? ScalabilityMode::kS2T3
+                                     : ScalabilityMode::kL1T3;
+    case ScalabilityMode::kS3T3h:
+      return max_spatial_layers == 2 ? ScalabilityMode::kS2T3h
+                                     : ScalabilityMode::kL1T3;
+  }
+  RTC_CHECK_NOTREACHED();
+}
+
 }  // namespace webrtc

modules/video_coding/svc/scalability_mode_util.h

@@ -36,6 +36,9 @@ int ScalabilityModeToNumTemporalLayers(ScalabilityMode scalability_mode);
 absl::optional<ScalabilityModeResolutionRatio> ScalabilityModeToResolutionRatio(
     ScalabilityMode scalability_mode);
 
+ScalabilityMode LimitNumSpatialLayers(ScalabilityMode scalability_mode,
+                                      int max_spatial_layers);
+
 }  // namespace webrtc
 
 #endif  // MODULES_VIDEO_CODING_SVC_SCALABILITY_MODE_UTIL_H_

modules/video_coding/svc/scalability_mode_util_unittest.cc

@@ -10,6 +10,10 @@
 
 #include "modules/video_coding/svc/scalability_mode_util.h"
 
+#include <string>
+#include <tuple>
+#include <vector>
+
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 #include "api/video_codecs/scalability_mode.h"
@@ -30,7 +34,7 @@ TEST(ScalabilityModeUtil, RejectsUnknownString) {
 
 // Check roundtrip conversion of all enum values.
 TEST(ScalabilityModeUtil, ConvertsAllToAndFromString) {
-  const ScalabilityMode kLastEnum = ScalabilityMode::kS3T3;
+  const ScalabilityMode kLastEnum = ScalabilityMode::kS3T3h;
   for (int numerical_enum = 0; numerical_enum <= static_cast<int>(kLastEnum);
        numerical_enum++) {
     ScalabilityMode scalability_mode =
@@ -43,5 +47,70 @@ TEST(ScalabilityModeUtil, ConvertsAllToAndFromString) {
   }
 }
 
+struct TestParams {
+  std::string scalability_mode;
+  std::vector<std::tuple<std::vector<int>, std::string>>
+      limited_scalability_mode;
+};
+
+class NumSpatialLayersTest : public ::testing::TestWithParam<TestParams> {};
+
+INSTANTIATE_TEST_SUITE_P(
+    MaxLayers,
+    NumSpatialLayersTest,
+    ::testing::ValuesIn<TestParams>(
+        {{"L1T1", {{{0, 1}, "L1T1"}, {{2}, "L1T1"}, {{3}, "L1T1"}}},
+         {"L1T2", {{{0, 1}, "L1T2"}, {{2}, "L1T2"}, {{3}, "L1T2"}}},
+         {"L1T3", {{{0, 1}, "L1T3"}, {{2}, "L1T3"}, {{3}, "L1T3"}}},
+         {"L2T1", {{{0, 1}, "L1T1"}, {{2}, "L2T1"}, {{3}, "L2T1"}}},
+         {"L2T1h", {{{0, 1}, "L1T1"}, {{2}, "L2T1h"}, {{3}, "L2T1h"}}},
+         {"L2T1_KEY", {{{0, 1}, "L1T1"}, {{2}, "L2T1_KEY"}, {{3}, "L2T1_KEY"}}},
+         {"L2T2", {{{0, 1}, "L1T2"}, {{2}, "L2T2"}, {{3}, "L2T2"}}},
+         {"L2T2h", {{{0, 1}, "L1T2"}, {{2}, "L2T2h"}, {{3}, "L2T2h"}}},
+         {"L2T2_KEY", {{{0, 1}, "L1T2"}, {{2}, "L2T2_KEY"}, {{3}, "L2T2_KEY"}}},
+         {"L2T2_KEY_SHIFT",
+          {{{0, 1}, "L1T2"}, {{2}, "L2T2_KEY_SHIFT"}, {{3}, "L2T2_KEY_SHIFT"}}},
+         {"L2T3", {{{0, 1}, "L1T3"}, {{2}, "L2T3"}, {{3}, "L2T3"}}},
+         {"L2T3h", {{{0, 1}, "L1T3"}, {{2}, "L2T3h"}, {{3}, "L2T3h"}}},
+         {"L2T3_KEY", {{{0, 1}, "L1T3"}, {{2}, "L2T3_KEY"}, {{3}, "L2T3_KEY"}}},
+         {"L3T1", {{{0, 1}, "L1T1"}, {{2}, "L2T1"}, {{3}, "L3T1"}}},
+         {"L3T1h", {{{0, 1}, "L1T1"}, {{2}, "L2T1h"}, {{3}, "L3T1h"}}},
+         {"L3T1_KEY", {{{0, 1}, "L1T1"}, {{2}, "L2T1_KEY"}, {{3}, "L3T1_KEY"}}},
+         {"L3T2", {{{0, 1}, "L1T2"}, {{2}, "L2T2"}, {{3}, "L3T2"}}},
+         {"L3T2h", {{{0, 1}, "L1T2"}, {{2}, "L2T2h"}, {{3}, "L3T2h"}}},
+         {"L3T2_KEY", {{{0, 1}, "L1T2"}, {{2}, "L2T2_KEY"}, {{3}, "L3T2_KEY"}}},
+         {"L3T3", {{{0, 1}, "L1T3"}, {{2}, "L2T3"}, {{3}, "L3T3"}}},
+         {"L3T3h", {{{0, 1}, "L1T3"}, {{2}, "L2T3h"}, {{3}, "L3T3h"}}},
+         {"L3T3_KEY", {{{0, 1}, "L1T3"}, {{2}, "L2T3_KEY"}, {{3}, "L3T3_KEY"}}},
+         {"S2T1", {{{0, 1}, "L1T1"}, {{2}, "S2T1"}, {{3}, "S2T1"}}},
+         {"S2T1h", {{{0, 1}, "L1T1"}, {{2}, "S2T1h"}, {{3}, "S2T1h"}}},
+         {"S2T2", {{{0, 1}, "L1T2"}, {{2}, "S2T2"}, {{3}, "S2T2"}}},
+         {"S2T2h", {{{0, 1}, "L1T2"}, {{2}, "S2T2h"}, {{3}, "S2T2h"}}},
+         {"S2T3", {{{0, 1}, "L1T3"}, {{2}, "S2T3"}, {{3}, "S2T3"}}},
+         {"S2T3h", {{{0, 1}, "L1T3"}, {{2}, "S2T3h"}, {{3}, "S2T3h"}}},
+         {"S3T1", {{{0, 1}, "L1T1"}, {{2}, "S2T1"}, {{3}, "S3T1"}}},
+         {"S3T1h", {{{0, 1}, "L1T1"}, {{2}, "S2T1h"}, {{3}, "S3T1h"}}},
+         {"S3T2", {{{0, 1}, "L1T2"}, {{2}, "S2T2"}, {{3}, "S3T2"}}},
+         {"S3T2h", {{{0, 1}, "L1T2"}, {{2}, "S2T2h"}, {{3}, "S3T2h"}}},
+         {"S3T3", {{{0, 1}, "L1T3"}, {{2}, "S2T3"}, {{3}, "S3T3"}}},
+         {"S3T3h", {{{0, 1}, "L1T3"}, {{2}, "S2T3h"}, {{3}, "S3T3h"}}}}),
+    [](const ::testing::TestParamInfo<TestParams>& info) {
+      return info.param.scalability_mode;
+    });
+
+TEST_P(NumSpatialLayersTest, LimitsSpatialLayers) {
+  const ScalabilityMode mode =
+      *ScalabilityModeFromString(GetParam().scalability_mode);
+  for (const auto& param : GetParam().limited_scalability_mode) {
+    const std::vector<int> max_num_spatial_layers =
+        std::get<std::vector<int>>(param);
+    const ScalabilityMode expected_mode =
+        *ScalabilityModeFromString(std::get<std::string>(param));
+    for (const auto& max_layers : max_num_spatial_layers) {
+      EXPECT_EQ(expected_mode, LimitNumSpatialLayers(mode, max_layers));
+    }
+  }
+}
+
 }  // namespace
 }  // namespace webrtc