In av1 add structure with custom spatial scalability factor

Bug: webrtc:11042
Change-Id: I93f125fba1fa21d060de47c96435798525f6b374
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/178566
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Commit-Queue: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31617}

modules/video_coding/codecs/av1/BUILD.gn

@@ -62,6 +62,8 @@ rtc_source_set("scalability_structures") {
     "scalability_structure_l2t1.h",
     "scalability_structure_l2t1_key.cc",
     "scalability_structure_l2t1_key.h",
+    "scalability_structure_l2t1h.cc",
+    "scalability_structure_l2t1h.h",
     "scalability_structure_l2t2.cc",
     "scalability_structure_l2t2.h",
     "scalability_structure_l2t2_key.cc",

modules/video_coding/codecs/av1/libaom_av1_encoder.cc

@@ -322,11 +322,9 @@ bool LibaomAv1Encoder::SetSvcParams(
         1 << (svc_config.num_temporal_layers - tid - 1);
   }
 
-  // TODO(danilchap): Add support for custom resolution factor.
   for (int sid = 0; sid < svc_config.num_spatial_layers; ++sid) {
-    svc_params.scaling_factor_num[sid] = 1;
+    svc_params.scaling_factor_num[sid] = svc_config.scaling_factor_num[sid];
-    svc_params.scaling_factor_den[sid] =
+    svc_params.scaling_factor_den[sid] = svc_config.scaling_factor_den[sid];
-        1 << (svc_config.num_spatial_layers - sid - 1);
   }
 
   return true;
@@ -517,6 +515,18 @@ int32_t LibaomAv1Encoder::Encode(
       if (is_keyframe && codec_specific_info.generic_frame_info) {
         codec_specific_info.template_structure =
             svc_controller_->DependencyStructure();
+        auto& resolutions = codec_specific_info.template_structure->resolutions;
+        if (SvcEnabled()) {
+          resolutions.resize(svc_params_->number_spatial_layers);
+          for (int sid = 0; sid < svc_params_->number_spatial_layers; ++sid) {
+            int n = svc_params_->scaling_factor_num[sid];
+            int d = svc_params_->scaling_factor_den[sid];
+            resolutions[sid] =
+                RenderResolution(cfg_.g_w * n / d, cfg_.g_h * n / d);
+          }
+        } else {
+          resolutions = {RenderResolution(cfg_.g_w, cfg_.g_h)};
+        }
       }
       encoded_image_callback_->OnEncodedImage(encoded_image,
                                               &codec_specific_info, nullptr);

modules/video_coding/codecs/av1/libaom_av1_unittest.cc

@@ -28,6 +28,7 @@
 #include "modules/video_coding/codecs/av1/scalability_structure_l1t3.h"
 #include "modules/video_coding/codecs/av1/scalability_structure_l2t1.h"
 #include "modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t1h.h"
 #include "modules/video_coding/codecs/av1/scalability_structure_l2t2.h"
 #include "modules/video_coding/codecs/av1/scalability_structure_l2t2_key.h"
 #include "modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.h"
@@ -317,6 +318,11 @@ INSTANTIATE_TEST_SUITE_P(
                         /*configured_bitrates=*/
                         {{{0, 0}, DataRate::KilobitsPerSec(30)},
                          {{1, 0}, DataRate::KilobitsPerSec(70)}}},
+           SvcTestParam{std::make_unique<ScalabilityStructureL2T1h>,
+                        /*num_frames_to_generate=*/3,
+                        /*configured_bitrates=*/
+                        {{{0, 0}, DataRate::KilobitsPerSec(30)},
+                         {{1, 0}, DataRate::KilobitsPerSec(70)}}},
            SvcTestParam{std::make_unique<ScalabilityStructureL2T1Key>,
                         /*num_frames_to_generate=*/3},
            SvcTestParam{std::make_unique<ScalabilityStructureL3T1>,

modules/video_coding/codecs/av1/scalability_structure_l2t1.cc

@@ -40,6 +40,8 @@ ScalabilityStructureL2T1::StreamConfig() const {
   StreamLayersConfig result;
   result.num_spatial_layers = 2;
   result.num_temporal_layers = 1;
+  result.scaling_factor_num[0] = 1;
+  result.scaling_factor_den[0] = 2;
   return result;
 }
 

modules/video_coding/codecs/av1/scalability_structure_l2t1_key.cc

@@ -38,6 +38,8 @@ ScalabilityStructureL2T1Key::StreamConfig() const {
   StreamLayersConfig result;
   result.num_spatial_layers = 2;
   result.num_temporal_layers = 1;
+  result.scaling_factor_num[0] = 1;
+  result.scaling_factor_den[0] = 2;
   return result;
 }
 

modules/video_coding/codecs/av1/scalability_structure_l2t1h.cc

@@ -0,0 +1,35 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t1h.h"
+
+#include <utility>
+#include <vector>
+
+#include "absl/base/macros.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+
+ScalabilityStructureL2T1h::~ScalabilityStructureL2T1h() = default;
+
+ScalableVideoController::StreamLayersConfig
+ScalabilityStructureL2T1h::StreamConfig() const {
+  StreamLayersConfig result;
+  result.num_spatial_layers = 2;
+  result.num_temporal_layers = 1;
+  // 1.5:1 scaling, see https://w3c.github.io/webrtc-svc/#scalabilitymodes*
+  result.scaling_factor_num[0] = 2;
+  result.scaling_factor_den[0] = 3;
+  return result;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l2t1h.h

@@ -0,0 +1,27 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1H_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1H_H_
+
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t1.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+class ScalabilityStructureL2T1h : public ScalabilityStructureL2T1 {
+ public:
+  ~ScalabilityStructureL2T1h() override;
+
+  StreamLayersConfig StreamConfig() const override;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1H_H_

modules/video_coding/codecs/av1/scalability_structure_l2t2.cc

@@ -44,6 +44,8 @@ ScalabilityStructureL2T2::StreamConfig() const {
   StreamLayersConfig result;
   result.num_spatial_layers = 2;
   result.num_temporal_layers = 2;
+  result.scaling_factor_num[0] = 1;
+  result.scaling_factor_den[0] = 2;
   return result;
 }
 

modules/video_coding/codecs/av1/scalability_structure_l2t2_key.cc

@@ -43,6 +43,8 @@ ScalabilityStructureL2T2Key::StreamConfig() const {
   StreamLayersConfig result;
   result.num_spatial_layers = 2;
   result.num_temporal_layers = 2;
+  result.scaling_factor_num[0] = 1;
+  result.scaling_factor_den[0] = 2;
   return result;
 }
 

modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.cc

@@ -42,6 +42,8 @@ ScalabilityStructureL2T2KeyShift::StreamConfig() const {
   StreamLayersConfig result;
   result.num_spatial_layers = 2;
   result.num_temporal_layers = 2;
+  result.scaling_factor_num[0] = 1;
+  result.scaling_factor_den[0] = 2;
   return result;
 }
 

modules/video_coding/codecs/av1/scalability_structure_l3t1.cc

@@ -41,6 +41,10 @@ ScalabilityStructureL3T1::StreamConfig() const {
   StreamLayersConfig result;
   result.num_spatial_layers = 3;
   result.num_temporal_layers = 1;
+  result.scaling_factor_num[0] = 1;
+  result.scaling_factor_den[0] = 4;
+  result.scaling_factor_num[1] = 1;
+  result.scaling_factor_den[1] = 2;
   return result;
 }
 

modules/video_coding/codecs/av1/scalability_structure_l3t3.cc

@@ -85,6 +85,10 @@ ScalabilityStructureL3T3::StreamConfig() const {
   StreamLayersConfig result;
   result.num_spatial_layers = 3;
   result.num_temporal_layers = 3;
+  result.scaling_factor_num[0] = 1;
+  result.scaling_factor_den[0] = 4;
+  result.scaling_factor_num[1] = 1;
+  result.scaling_factor_den[1] = 2;
   return result;
 }
 

modules/video_coding/codecs/av1/scalability_structure_s2t1.cc

@@ -37,6 +37,8 @@ ScalabilityStructureS2T1::StreamConfig() const {
   StreamLayersConfig result;
   result.num_spatial_layers = 2;
   result.num_temporal_layers = 1;
+  result.scaling_factor_num[0] = 1;
+  result.scaling_factor_den[0] = 2;
   return result;
 }
 

modules/video_coding/codecs/av1/scalable_video_controller.h

@@ -28,6 +28,10 @@ class ScalableVideoController {
   struct StreamLayersConfig {
     int num_spatial_layers = 1;
     int num_temporal_layers = 1;
+    // Spatial layers scaling. Frames with spatial_id = i expected to be encoded
+    // with original_resolution * scaling_factor_num[i] / scaling_factor_den[i].
+    int scaling_factor_num[DependencyDescriptor::kMaxSpatialIds] = {1, 1, 1, 1};
+    int scaling_factor_den[DependencyDescriptor::kMaxSpatialIds] = {1, 1, 1, 1};
   };
   class LayerFrameConfig {
    public: