Add av1 test with spatial scalability.

Bug: webrtc:11404
Change-Id: I6faa72a86d6f48b21b1e1cd6c2a1d748e168d018
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/176366
Commit-Queue: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31410}

modules/video_coding/codecs/av1/BUILD.gn

@@ -55,6 +55,12 @@ rtc_source_set("scalability_structures") {
   sources = [
     "scalability_structure_l1t2.cc",
     "scalability_structure_l1t2.h",
+    "scalability_structure_l2t1.cc",
+    "scalability_structure_l2t1.h",
+    "scalability_structure_l2t1_key.cc",
+    "scalability_structure_l2t1_key.h",
+    "scalability_structure_s2t1.cc",
+    "scalability_structure_s2t1.h",
   ]
   deps = [
     ":scalable_video_controller",

modules/video_coding/codecs/av1/libaom_av1_unittest.cc

@@ -22,6 +22,9 @@
 #include "modules/video_coding/codecs/av1/libaom_av1_decoder.h"
 #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
 #include "modules/video_coding/codecs/av1/scalability_structure_l1t2.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t1.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_s2t1.h"
 #include "modules/video_coding/codecs/av1/scalable_video_controller.h"
 #include "modules/video_coding/codecs/av1/scalable_video_controller_no_layering.h"
 #include "modules/video_coding/include/video_codec_interface.h"
@@ -275,7 +278,13 @@ INSTANTIATE_TEST_SUITE_P(
     Values(SvcTestParam{std::make_unique<ScalableVideoControllerNoLayering>,
                         /*num_frames_to_generate=*/4},
            SvcTestParam{std::make_unique<ScalabilityStructureL1T2>,
-                        /*num_frames_to_generate=*/4}));
+                        /*num_frames_to_generate=*/4},
+           SvcTestParam{std::make_unique<ScalabilityStructureL2T1>,
+                        /*num_frames_to_generate=*/3},
+           SvcTestParam{std::make_unique<ScalabilityStructureL2T1Key>,
+                        /*num_frames_to_generate=*/3},
+           SvcTestParam{std::make_unique<ScalabilityStructureS2T1>,
+                        /*num_frames_to_generate=*/3}));
 
 }  // namespace
 }  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l2t1.cc

@@ -0,0 +1,122 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t1.h"
+
+#include <utility>
+#include <vector>
+
+#include "absl/base/macros.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+namespace {
+
+constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent;
+constexpr auto kSwitch = DecodeTargetIndication::kSwitch;
+constexpr auto kRequired = DecodeTargetIndication::kRequired;
+
+constexpr DecodeTargetIndication kDtis[4][2] = {
+    {kSwitch, kSwitch},        // Key, S0
+    {kNotPresent, kSwitch},    // Key, S1
+    {kSwitch, kRequired},      // Delta, S0
+    {kNotPresent, kRequired},  // Delta, S1
+};
+
+}  // namespace
+
+ScalabilityStructureL2T1::~ScalabilityStructureL2T1() = default;
+
+ScalableVideoController::StreamLayersConfig
+ScalabilityStructureL2T1::StreamConfig() const {
+  StreamLayersConfig result;
+  result.num_spatial_layers = 2;
+  result.num_temporal_layers = 1;
+  return result;
+}
+
+FrameDependencyStructure ScalabilityStructureL2T1::DependencyStructure() const {
+  using Builder = GenericFrameInfo::Builder;
+  FrameDependencyStructure structure;
+  structure.num_decode_targets = 2;
+  structure.num_chains = 2;
+  structure.decode_target_protected_by_chain = {0, 1};
+  structure.templates = {
+      Builder().S(0).Dtis("SR").Fdiffs({2}).ChainDiffs({2, 1}).Build(),
+      Builder().S(0).Dtis("SS").ChainDiffs({0, 0}).Build(),
+      Builder().S(1).Dtis("-R").Fdiffs({1, 2}).ChainDiffs({1, 2}).Build(),
+      Builder().S(1).Dtis("-S").Fdiffs({1}).ChainDiffs({1, 1}).Build(),
+  };
+  return structure;
+}
+
+ScalableVideoController::LayerFrameConfig
+ScalabilityStructureL2T1::KeyFrameConfig() const {
+  LayerFrameConfig result;
+  result.id = 0;
+  result.spatial_id = 0;
+  result.is_keyframe = true;
+  result.buffers = {{/*id=*/0, /*references=*/false, /*updates=*/true}};
+  return result;
+}
+
+std::vector<ScalableVideoController::LayerFrameConfig>
+ScalabilityStructureL2T1::NextFrameConfig(bool restart) {
+  std::vector<LayerFrameConfig> result(2);
+
+  // Buffer0 keeps latest S0 frame, Buffer1 keeps latest S1 frame.
+  if (restart || keyframe_) {
+    result[0] = KeyFrameConfig();
+
+    result[1].id = 1;
+    result[1].spatial_id = 1;
+    result[1].is_keyframe = false;
+    result[1].buffers = {{/*id=*/0, /*references=*/true, /*updates=*/false},
+                         {/*id=*/1, /*references=*/false, /*updates=*/true}};
+
+    keyframe_ = false;
+  } else {
+    result[0].id = 2;
+    result[0].spatial_id = 0;
+    result[0].is_keyframe = false;
+    result[0].buffers = {{/*id=*/0, /*references=*/true, /*updates=*/true}};
+
+    result[1].id = 3;
+    result[1].spatial_id = 1;
+    result[1].is_keyframe = false;
+    result[1].buffers = {{/*id=*/0, /*references=*/true, /*updates=*/false},
+                         {/*id=*/1, /*references=*/true, /*updates=*/true}};
+  }
+  return result;
+}
+
+absl::optional<GenericFrameInfo> ScalabilityStructureL2T1::OnEncodeDone(
+    LayerFrameConfig config) {
+  absl::optional<GenericFrameInfo> frame_info;
+  if (config.is_keyframe) {
+    config = KeyFrameConfig();
+  }
+
+  if (config.id < 0 || config.id >= int{ABSL_ARRAYSIZE(kDtis)}) {
+    RTC_LOG(LS_ERROR) << "Unexpected config id " << config.id;
+    return frame_info;
+  }
+  frame_info.emplace();
+  frame_info->spatial_id = config.spatial_id;
+  frame_info->temporal_id = config.temporal_id;
+  frame_info->encoder_buffers = std::move(config.buffers);
+  frame_info->decode_target_indications.assign(std::begin(kDtis[config.id]),
+                                               std::end(kDtis[config.id]));
+  frame_info->part_of_chain = {config.spatial_id == 0, true};
+  return frame_info;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l2t1.h

@@ -0,0 +1,43 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_H_
+
+#include <vector>
+
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+// S1  0--0--0-
+//     |  |  | ...
+// S0  0--0--0-
+class ScalabilityStructureL2T1 : public ScalableVideoController {
+ public:
+  ~ScalabilityStructureL2T1() override;
+
+  StreamLayersConfig StreamConfig() const override;
+  FrameDependencyStructure DependencyStructure() const override;
+
+  std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override;
+  absl::optional<GenericFrameInfo> OnEncodeDone(
+      LayerFrameConfig config) override;
+
+ private:
+  LayerFrameConfig KeyFrameConfig() const;
+
+  bool keyframe_ = true;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_H_

modules/video_coding/codecs/av1/scalability_structure_l2t1_key.cc

@@ -0,0 +1,126 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h"
+
+#include <utility>
+#include <vector>
+
+#include "absl/base/macros.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+namespace {
+
+constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent;
+constexpr auto kSwitch = DecodeTargetIndication::kSwitch;
+
+constexpr DecodeTargetIndication kDtis[3][2] = {
+    {kSwitch, kSwitch},      // Key, S0
+    {kSwitch, kNotPresent},  // Delta, S0
+    {kNotPresent, kSwitch},  // Key and Delta, S1
+};
+
+}  // namespace
+
+ScalabilityStructureL2T1Key::~ScalabilityStructureL2T1Key() = default;
+
+ScalableVideoController::StreamLayersConfig
+ScalabilityStructureL2T1Key::StreamConfig() const {
+  StreamLayersConfig result;
+  result.num_spatial_layers = 2;
+  result.num_temporal_layers = 1;
+  return result;
+}
+
+FrameDependencyStructure ScalabilityStructureL2T1Key::DependencyStructure()
+    const {
+  using Builder = GenericFrameInfo::Builder;
+  FrameDependencyStructure structure;
+  structure.num_decode_targets = 2;
+  structure.num_chains = 2;
+  structure.decode_target_protected_by_chain = {0, 1};
+  structure.templates = {
+      Builder().S(0).Dtis("S-").Fdiffs({2}).ChainDiffs({2, 1}).Build(),
+      Builder().S(0).Dtis("SS").ChainDiffs({0, 0}).Build(),
+      Builder().S(1).Dtis("-S").Fdiffs({2}).ChainDiffs({1, 2}).Build(),
+      Builder().S(1).Dtis("-S").Fdiffs({1}).ChainDiffs({1, 1}).Build(),
+  };
+  return structure;
+}
+
+ScalableVideoController::LayerFrameConfig
+ScalabilityStructureL2T1Key::KeyFrameConfig() const {
+  LayerFrameConfig result;
+  result.id = 0;
+  result.spatial_id = 0;
+  result.is_keyframe = true;
+  result.buffers = {{/*id=*/0, /*references=*/false, /*updates=*/true}};
+  return result;
+}
+
+std::vector<ScalableVideoController::LayerFrameConfig>
+ScalabilityStructureL2T1Key::NextFrameConfig(bool restart) {
+  std::vector<LayerFrameConfig> result(2);
+
+  // Buffer0 keeps latest S0T0 frame, Buffer1 keeps latest S1T0 frame.
+  if (restart || keyframe_) {
+    result[0] = KeyFrameConfig();
+
+    result[1].id = 2;
+    result[1].spatial_id = 1;
+    result[1].is_keyframe = false;
+    result[1].buffers = {{/*id=*/0, /*references=*/true, /*updates=*/false},
+                         {/*id=*/1, /*references=*/false, /*updates=*/true}};
+
+    keyframe_ = false;
+  } else {
+    result[0].id = 1;
+    result[0].spatial_id = 0;
+    result[0].is_keyframe = false;
+    result[0].buffers = {{/*id=*/0, /*references=*/true, /*updates=*/true}};
+
+    result[1].id = 2;
+    result[1].spatial_id = 1;
+    result[1].is_keyframe = false;
+    result[1].buffers = {{/*id=*/0, /*references=*/false, /*updates=*/false},
+                         {/*id=*/1, /*references=*/true, /*updates=*/true}};
+  }
+  return result;
+}
+
+absl::optional<GenericFrameInfo> ScalabilityStructureL2T1Key::OnEncodeDone(
+    LayerFrameConfig config) {
+  absl::optional<GenericFrameInfo> frame_info;
+  if (config.is_keyframe) {
+    config = KeyFrameConfig();
+  }
+
+  if (config.id < 0 || config.id >= int{ABSL_ARRAYSIZE(kDtis)}) {
+    RTC_LOG(LS_ERROR) << "Unexpected config id " << config.id;
+    return frame_info;
+  }
+  frame_info.emplace();
+  frame_info->spatial_id = config.spatial_id;
+  frame_info->temporal_id = config.temporal_id;
+  frame_info->encoder_buffers = std::move(config.buffers);
+  frame_info->decode_target_indications.assign(std::begin(kDtis[config.id]),
+                                               std::end(kDtis[config.id]));
+  if (config.is_keyframe) {
+    frame_info->part_of_chain = {true, true};
+  } else {
+    frame_info->part_of_chain = {config.spatial_id == 0,
+                                 config.spatial_id == 1};
+  }
+  return frame_info;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h

@@ -0,0 +1,43 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_KEY_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_KEY_H_
+
+#include <vector>
+
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+// S1  0--0--0-
+//     |       ...
+// S0  0--0--0-
+class ScalabilityStructureL2T1Key : public ScalableVideoController {
+ public:
+  ~ScalabilityStructureL2T1Key() override;
+
+  StreamLayersConfig StreamConfig() const override;
+  FrameDependencyStructure DependencyStructure() const override;
+
+  std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override;
+  absl::optional<GenericFrameInfo> OnEncodeDone(
+      LayerFrameConfig config) override;
+
+ private:
+  LayerFrameConfig KeyFrameConfig() const;
+
+  bool keyframe_ = true;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_KEY_H_

modules/video_coding/codecs/av1/scalability_structure_s2t1.cc

@@ -0,0 +1,99 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/scalability_structure_s2t1.h"
+
+#include <utility>
+#include <vector>
+
+#include "absl/base/macros.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+namespace {
+
+constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent;
+constexpr auto kSwitch = DecodeTargetIndication::kSwitch;
+
+constexpr DecodeTargetIndication kDtis[2][2] = {
+    {kSwitch, kNotPresent},  // S0
+    {kNotPresent, kSwitch},  // S1
+};
+
+}  // namespace
+
+ScalabilityStructureS2T1::~ScalabilityStructureS2T1() = default;
+
+ScalableVideoController::StreamLayersConfig
+ScalabilityStructureS2T1::StreamConfig() const {
+  StreamLayersConfig result;
+  result.num_spatial_layers = 2;
+  result.num_temporal_layers = 1;
+  return result;
+}
+
+FrameDependencyStructure ScalabilityStructureS2T1::DependencyStructure() const {
+  using Builder = GenericFrameInfo::Builder;
+  FrameDependencyStructure structure;
+  structure.num_decode_targets = 2;
+  structure.num_chains = 2;
+  structure.decode_target_protected_by_chain = {0, 1};
+  structure.templates = {
+      Builder().S(0).Dtis("S-").Fdiffs({2}).ChainDiffs({2, 1}).Build(),
+      Builder().S(0).Dtis("S-").ChainDiffs({0, 0}).Build(),
+      Builder().S(1).Dtis("-S").Fdiffs({2}).ChainDiffs({1, 2}).Build(),
+      Builder().S(1).Dtis("-S").ChainDiffs({1, 0}).Build(),
+  };
+  return structure;
+}
+
+std::vector<ScalableVideoController::LayerFrameConfig>
+ScalabilityStructureS2T1::NextFrameConfig(bool restart) {
+  if (restart) {
+    keyframe_ = true;
+  }
+  std::vector<LayerFrameConfig> result(2);
+
+  // Buffer0 keeps latest S0T0 frame, Buffer1 keeps latest S1T0 frame.
+  result[0].spatial_id = 0;
+  result[0].is_keyframe = keyframe_;
+  result[0].buffers = {{/*id=*/0, /*references=*/!keyframe_, /*updates=*/true}};
+
+  result[1].spatial_id = 1;
+  result[1].is_keyframe = keyframe_;
+  result[1].buffers = {{/*id=*/1, /*references=*/!keyframe_, /*updates=*/true}};
+
+  keyframe_ = false;
+  return result;
+}
+
+absl::optional<GenericFrameInfo> ScalabilityStructureS2T1::OnEncodeDone(
+    LayerFrameConfig config) {
+  absl::optional<GenericFrameInfo> frame_info;
+  if (config.id != 0) {
+    RTC_LOG(LS_ERROR) << "Unexpected config id " << config.id;
+  }
+  if (config.spatial_id < 0 ||
+      config.spatial_id >= int{ABSL_ARRAYSIZE(kDtis)}) {
+    RTC_LOG(LS_ERROR) << "Unexpected spatial id " << config.spatial_id;
+    return frame_info;
+  }
+  frame_info.emplace();
+  frame_info->spatial_id = config.spatial_id;
+  frame_info->temporal_id = config.temporal_id;
+  frame_info->encoder_buffers = std::move(config.buffers);
+  frame_info->decode_target_indications.assign(
+      std::begin(kDtis[config.spatial_id]), std::end(kDtis[config.spatial_id]));
+  frame_info->part_of_chain = {config.spatial_id == 0, config.spatial_id == 1};
+  return frame_info;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_s2t1.h

@@ -0,0 +1,41 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_S2T1_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_S2T1_H_
+
+#include <vector>
+
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+// S1  0--0--0-
+//             ...
+// S0  0--0--0-
+class ScalabilityStructureS2T1 : public ScalableVideoController {
+ public:
+  ~ScalabilityStructureS2T1() override;
+
+  StreamLayersConfig StreamConfig() const override;
+  FrameDependencyStructure DependencyStructure() const override;
+
+  std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override;
+  absl::optional<GenericFrameInfo> OnEncodeDone(
+      LayerFrameConfig config) override;
+
+ private:
+  bool keyframe_ = true;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_S2T1_H_