Merge remote tracking branch 'upstream-master'

Bug: 153469641
Test: run cuttlefish locally
Change-Id: Ida3bfe62ef5c6549278f4c155a1f690b008e9b9d

modules/video_coding/codecs/av1/BUILD.gn

@@ -13,10 +13,8 @@ rtc_library("libaom_av1_decoder") {
   visibility = [ "*" ]
   poisonous = [ "software_video_codecs" ]
   public = [ "libaom_av1_decoder.h" ]
-  deps = [
-    "../../../../api/video_codecs:video_codecs_api",
-    "//third_party/abseil-cpp/absl/base:core_headers",
-  ]
+  deps = [ "../../../../api/video_codecs:video_codecs_api" ]
+  absl_deps = [ "//third_party/abseil-cpp/absl/base:core_headers" ]
 
   if (enable_libaom) {
     sources = [ "libaom_av1_decoder.cc" ]
@@ -27,22 +25,86 @@ rtc_library("libaom_av1_decoder") {
       "../../../../api/video:video_frame_i420",
       "../../../../common_video",
       "../../../../rtc_base:logging",
-      "//third_party/abseil-cpp/absl/types:optional",
       "//third_party/libaom",
       "//third_party/libyuv",
     ]
+    absl_deps += [ "//third_party/abseil-cpp/absl/types:optional" ]
   } else {
     sources = [ "libaom_av1_decoder_absent.cc" ]
   }
 }
 
+rtc_source_set("scalable_video_controller") {
+  sources = [
+    "scalable_video_controller.h",
+    "scalable_video_controller_no_layering.cc",
+    "scalable_video_controller_no_layering.h",
+  ]
+  deps = [
+    "../../../../api/transport/rtp:dependency_descriptor",
+    "../../../../api/video:video_bitrate_allocation",
+    "../../../../common_video/generic_frame_descriptor",
+    "../../../../rtc_base:checks",
+  ]
+  absl_deps = [
+    "//third_party/abseil-cpp/absl/container:inlined_vector",
+    "//third_party/abseil-cpp/absl/types:optional",
+  ]
+}
+
+rtc_source_set("scalability_structures") {
+  sources = [
+    "create_scalability_structure.cc",
+    "create_scalability_structure.h",
+    "scalability_structure_l1t2.cc",
+    "scalability_structure_l1t2.h",
+    "scalability_structure_l1t3.cc",
+    "scalability_structure_l1t3.h",
+    "scalability_structure_l2t1.cc",
+    "scalability_structure_l2t1.h",
+    "scalability_structure_l2t1_key.cc",
+    "scalability_structure_l2t1_key.h",
+    "scalability_structure_l2t1h.cc",
+    "scalability_structure_l2t1h.h",
+    "scalability_structure_l2t2.cc",
+    "scalability_structure_l2t2.h",
+    "scalability_structure_l2t2_key.cc",
+    "scalability_structure_l2t2_key.h",
+    "scalability_structure_l2t2_key_shift.cc",
+    "scalability_structure_l2t2_key_shift.h",
+    "scalability_structure_l3t1.cc",
+    "scalability_structure_l3t1.h",
+    "scalability_structure_l3t3.cc",
+    "scalability_structure_l3t3.h",
+    "scalability_structure_s2t1.cc",
+    "scalability_structure_s2t1.h",
+  ]
+  deps = [
+    ":scalable_video_controller",
+    "../../../../api/transport/rtp:dependency_descriptor",
+    "../../../../common_video/generic_frame_descriptor",
+    "../../../../rtc_base:checks",
+    "../../../../rtc_base:logging",
+  ]
+  absl_deps = [
+    "//third_party/abseil-cpp/absl/base:core_headers",
+    "//third_party/abseil-cpp/absl/strings",
+    "//third_party/abseil-cpp/absl/types:optional",
+  ]
+}
+
 rtc_library("libaom_av1_encoder") {
   visibility = [ "*" ]
   poisonous = [ "software_video_codecs" ]
   public = [ "libaom_av1_encoder.h" ]
   deps = [
+    ":scalable_video_controller",
     "../../../../api/video_codecs:video_codecs_api",
+  ]
+  absl_deps = [
+    "//third_party/abseil-cpp/absl/algorithm:container",
     "//third_party/abseil-cpp/absl/base:core_headers",
+    "//third_party/abseil-cpp/absl/types:optional",
   ]
 
   if (enable_libaom) {
@@ -56,7 +118,6 @@ rtc_library("libaom_av1_encoder") {
       "../../../../common_video",
       "../../../../rtc_base:checks",
       "../../../../rtc_base:logging",
-      "//third_party/abseil-cpp/absl/algorithm:container",
       "//third_party/libaom",
     ]
   } else {
@@ -65,6 +126,21 @@ rtc_library("libaom_av1_encoder") {
 }
 
 if (rtc_include_tests) {
+  rtc_library("scalability_structure_tests") {
+    testonly = true
+    sources = [ "scalability_structure_unittest.cc" ]
+    deps = [
+      ":scalability_structures",
+      ":scalable_video_controller",
+      "../..:chain_diff_calculator",
+      "../..:frame_dependencies_calculator",
+      "../../../../api/transport/rtp:dependency_descriptor",
+      "../../../../api/video:video_frame_type",
+      "../../../../test:test_support",
+    ]
+    absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+  }
+
   rtc_library("video_coding_codecs_av1_tests") {
     testonly = true
 
@@ -76,13 +152,18 @@ if (rtc_include_tests) {
       deps = [
         ":libaom_av1_decoder",
         ":libaom_av1_encoder",
+        ":scalability_structures",
+        ":scalable_video_controller",
+        "../..:encoded_video_frame_producer",
         "../..:video_codec_interface",
-        "../../../../api:create_frame_generator",
-        "../../../../api:frame_generator_api",
+        "../../../../api:mock_video_encoder",
+        "../../../../api/units:data_size",
+        "../../../../api/units:time_delta",
+        "../../../../api/video:video_frame_i420",
         "../../../../api/video_codecs:video_codecs_api",
         "../../../../test:test_support",
-        "//third_party/abseil-cpp/absl/types:optional",
       ]
+      absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
     }
   }
 }

modules/video_coding/codecs/av1/create_scalability_structure.cc

@@ -0,0 +1,73 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/create_scalability_structure.h"
+
+#include <memory>
+
+#include "absl/strings/string_view.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_l1t2.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_l1t3.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t1.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t1h.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t2.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t2_key.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_l3t1.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_l3t3.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_s2t1.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller_no_layering.h"
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+namespace {
+
+struct NamedStructureFactory {
+  absl::string_view name;
+  // Use function pointer to make NamedStructureFactory trivally destructable.
+  std::unique_ptr<ScalableVideoController> (*factory)();
+};
+
+// Wrap std::make_unique function to have correct return type.
+template <typename T>
+std::unique_ptr<ScalableVideoController> Create() {
+  return std::make_unique<T>();
+}
+
+constexpr NamedStructureFactory kFactories[] = {
+    {"NONE", Create<ScalableVideoControllerNoLayering>},
+    {"L1T2", Create<ScalabilityStructureL1T2>},
+    {"L1T3", Create<ScalabilityStructureL1T3>},
+    {"L2T1", Create<ScalabilityStructureL2T1>},
+    {"L2T1h", Create<ScalabilityStructureL2T1h>},
+    {"L2T1_KEY", Create<ScalabilityStructureL2T1Key>},
+    {"L2T2", Create<ScalabilityStructureL2T2>},
+    {"L2T2_KEY", Create<ScalabilityStructureL2T2Key>},
+    {"L2T2_KEY_SHIFT", Create<ScalabilityStructureL2T2KeyShift>},
+    {"L3T1", Create<ScalabilityStructureL3T1>},
+    {"L3T3", Create<ScalabilityStructureL3T3>},
+    {"S2T1", Create<ScalabilityStructureS2T1>},
+};
+
+}  // namespace
+
+std::unique_ptr<ScalableVideoController> CreateScalabilityStructure(
+    absl::string_view name) {
+  RTC_DCHECK(!name.empty());
+  for (const auto& entry : kFactories) {
+    if (entry.name == name) {
+      return entry.factory();
+    }
+  }
+  return nullptr;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/create_scalability_structure.h

@@ -0,0 +1,29 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_CREATE_SCALABILITY_STRUCTURE_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_CREATE_SCALABILITY_STRUCTURE_H_
+
+#include <memory>
+#include <vector>
+
+#include "absl/strings/string_view.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+// Creates a structure by name according to
+// https://w3c.github.io/webrtc-svc/#scalabilitymodes*
+// Returns nullptr for unknown name.
+std::unique_ptr<ScalableVideoController> CreateScalabilityStructure(
+    absl::string_view name);
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_CREATE_SCALABILITY_STRUCTURE_H_

modules/video_coding/codecs/av1/libaom_av1_decoder.cc

@@ -53,6 +53,8 @@ class LibaomAv1Decoder final : public VideoDecoder {
 
   int32_t Release() override;
 
+  const char* ImplementationName() const override;
+
  private:
   aom_codec_ctx_t context_;
   bool inited_;
@@ -127,7 +129,7 @@ int32_t LibaomAv1Decoder::Decode(const EncodedImage& encoded_image,
 
     // Return decoded frame data.
     int qp;
-    ret = aom_codec_control_(&context_, AOMD_GET_LAST_QUANTIZER, &qp);
+    ret = aom_codec_control(&context_, AOMD_GET_LAST_QUANTIZER, &qp);
     if (ret != AOM_CODEC_OK) {
       RTC_LOG(LS_WARNING) << "LibaomAv1Decoder::Decode returned " << ret
                           << " on control AOME_GET_LAST_QUANTIZER.";
@@ -180,6 +182,10 @@ int32_t LibaomAv1Decoder::Release() {
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
+const char* LibaomAv1Decoder::ImplementationName() const {
+  return "libaom";
+}
+
 }  // namespace
 
 const bool kIsLibaomAv1DecoderSupported = true;

modules/video_coding/codecs/av1/libaom_av1_encoder.cc

@@ -13,15 +13,20 @@
 #include <stdint.h>
 
 #include <memory>
+#include <utility>
 #include <vector>
 
 #include "absl/algorithm/container.h"
+#include "absl/base/macros.h"
+#include "absl/types/optional.h"
 #include "api/scoped_refptr.h"
 #include "api/video/encoded_image.h"
 #include "api/video/i420_buffer.h"
 #include "api/video/video_frame.h"
 #include "api/video_codecs/video_codec.h"
 #include "api/video_codecs/video_encoder.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller_no_layering.h"
 #include "modules/video_coding/include/video_codec_interface.h"
 #include "modules/video_coding/include/video_error_codes.h"
 #include "rtc_base/checks.h"
@@ -34,9 +39,7 @@ namespace webrtc {
 namespace {
 
 // Encoder configuration parameters
-constexpr int kQpMax = 56;
 constexpr int kQpMin = 10;
-constexpr int kDefaultEncSpeed = 7;  // Use values 6, 7, or 8 for RTC.
 constexpr int kUsageProfile = 1;     // 0 = good quality; 1 = real-time.
 constexpr int kMinQindex = 58;       // Min qindex threshold for QP scaling.
 constexpr int kMaxQindex = 180;      // Max qindex threshold for QP scaling.
@@ -45,9 +48,23 @@ constexpr int kLagInFrames = 0;  // No look ahead.
 constexpr int kRtpTicksPerSecond = 90000;
 constexpr float kMinimumFrameRate = 1.0;
 
+// Only positive speeds, range for real-time coding currently is: 6 - 8.
+// Lower means slower/better quality, higher means fastest/lower quality.
+int GetCpuSpeed(int width, int height, int number_of_cores) {
+  // For smaller resolutions, use lower speed setting (get some coding gain at
+  // the cost of increased encoding complexity).
+  if (number_of_cores > 2 && width * height <= 320 * 180)
+    return 6;
+  else if (width * height >= 1280 * 720)
+    return 8;
+  else
+    return 7;
+}
+
 class LibaomAv1Encoder final : public VideoEncoder {
  public:
-  LibaomAv1Encoder();
+  explicit LibaomAv1Encoder(
+      std::unique_ptr<ScalableVideoController> svc_controller);
   ~LibaomAv1Encoder();
 
   int InitEncode(const VideoCodec* codec_settings,
@@ -66,8 +83,19 @@ class LibaomAv1Encoder final : public VideoEncoder {
   EncoderInfo GetEncoderInfo() const override;
 
  private:
+  bool SvcEnabled() const { return svc_params_.has_value(); }
+  // Fills svc_params_ memeber value. Returns false on error.
+  bool SetSvcParams(ScalableVideoController::StreamLayersConfig svc_config);
+  // Configures the encoder with layer for the next frame.
+  void SetSvcLayerId(
+      const ScalableVideoController::LayerFrameConfig& layer_frame);
+  // Configures the encoder which buffers next frame updates and can reference.
+  void SetSvcRefFrameConfig(
+      const ScalableVideoController::LayerFrameConfig& layer_frame);
+
+  const std::unique_ptr<ScalableVideoController> svc_controller_;
   bool inited_;
-  bool keyframe_required_;
+  absl::optional<aom_svc_params_t> svc_params_;
   VideoCodec encoder_settings_;
   aom_image_t* frame_for_encode_;
   aom_codec_ctx_t ctx_;
@@ -100,11 +128,14 @@ int32_t VerifyCodecSettings(const VideoCodec& codec_settings) {
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
-LibaomAv1Encoder::LibaomAv1Encoder()
-    : inited_(false),
-      keyframe_required_(true),
+LibaomAv1Encoder::LibaomAv1Encoder(
+    std::unique_ptr<ScalableVideoController> svc_controller)
+    : svc_controller_(std::move(svc_controller)),
+      inited_(false),
       frame_for_encode_(nullptr),
-      encoded_image_callback_(nullptr) {}
+      encoded_image_callback_(nullptr) {
+  RTC_DCHECK(svc_controller_);
+}
 
 LibaomAv1Encoder::~LibaomAv1Encoder() {
   Release();
@@ -134,6 +165,10 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings,
     return result;
   }
 
+  if (!SetSvcParams(svc_controller_->StreamConfig())) {
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
   // Initialize encoder configuration structure with default values
   aom_codec_err_t ret =
       aom_codec_enc_config_default(aom_codec_av1_cx(), &cfg_, 0);
@@ -153,9 +188,9 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings,
   cfg_.g_input_bit_depth = kBitDepth;
   cfg_.kf_mode = AOM_KF_DISABLED;
   cfg_.rc_min_quantizer = kQpMin;
-  cfg_.rc_max_quantizer = kQpMax;
+  cfg_.rc_max_quantizer = encoder_settings_.qpMax;
   cfg_.g_usage = kUsageProfile;
-
+  cfg_.g_error_resilient = 0;
   // Low-latency settings.
   cfg_.rc_end_usage = AOM_CBR;          // Constant Bit Rate (CBR) mode
   cfg_.g_pass = AOM_RC_ONE_PASS;        // One-pass rate control
@@ -180,7 +215,9 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings,
   inited_ = true;
 
   // Set control parameters
-  ret = aom_codec_control(&ctx_, AOME_SET_CPUUSED, kDefaultEncSpeed);
+  ret = aom_codec_control(
+      &ctx_, AOME_SET_CPUUSED,
+      GetCpuSpeed(cfg_.g_w, cfg_.g_h, settings.number_of_cores));
   if (ret != AOM_CODEC_OK) {
     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
                         << " on control AV1E_SET_CPUUSED.";
@@ -198,16 +235,147 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings,
                         << " on control AV1E_SET_DELTAQ_MODE.";
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
+  ret = aom_codec_control(&ctx_, AV1E_SET_ENABLE_ORDER_HINT, 0);
+  if (ret != AOM_CODEC_OK) {
+    RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
+                        << " on control AV1E_SET_ENABLE_ORDER_HINT.";
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
   ret = aom_codec_control(&ctx_, AV1E_SET_AQ_MODE, 3);
   if (ret != AOM_CODEC_OK) {
     RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
                         << " on control AV1E_SET_AQ_MODE.";
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
+  if (SvcEnabled()) {
+    ret = aom_codec_control(&ctx_, AV1E_SET_SVC_PARAMS, &*svc_params_);
+    if (ret != AOM_CODEC_OK) {
+      RTC_LOG(LS_WARNING) << "LibaomAV1Encoder::EncodeInit returned " << ret
+                          << " on control AV1E_SET_SVC_PARAMS.";
+      return false;
+    }
+  }
+
+  ret = aom_codec_control(&ctx_, AOME_SET_MAX_INTRA_BITRATE_PCT, 300);
+  if (ret != AOM_CODEC_OK) {
+    RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
+                        << " on control AV1E_SET_MAX_INTRA_BITRATE_PCT.";
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  ret = aom_codec_control(&ctx_, AV1E_SET_COEFF_COST_UPD_FREQ, 2);
+  if (ret != AOM_CODEC_OK) {
+    RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
+                        << " on control AV1E_SET_COEFF_COST_UPD_FREQ.";
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  ret = aom_codec_control(&ctx_, AV1E_SET_MODE_COST_UPD_FREQ, 2);
+  if (ret != AOM_CODEC_OK) {
+    RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
+                        << " on control AV1E_SET_MODE_COST_UPD_FREQ.";
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  ret = aom_codec_control(&ctx_, AV1E_SET_MV_COST_UPD_FREQ, 3);
+  if (ret != AOM_CODEC_OK) {
+    RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::EncodeInit returned " << ret
+                        << " on control AV1E_SET_MV_COST_UPD_FREQ.";
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
 
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
+bool LibaomAv1Encoder::SetSvcParams(
+    ScalableVideoController::StreamLayersConfig svc_config) {
+  bool svc_enabled =
+      svc_config.num_spatial_layers > 1 || svc_config.num_temporal_layers > 1;
+  if (!svc_enabled) {
+    svc_params_ = absl::nullopt;
+    return true;
+  }
+  if (svc_config.num_spatial_layers < 1 || svc_config.num_spatial_layers > 4) {
+    RTC_LOG(LS_WARNING) << "Av1 supports up to 4 spatial layers. "
+                        << svc_config.num_spatial_layers << " configured.";
+    return false;
+  }
+  if (svc_config.num_temporal_layers < 1 ||
+      svc_config.num_temporal_layers > 8) {
+    RTC_LOG(LS_WARNING) << "Av1 supports up to 8 temporal layers. "
+                        << svc_config.num_temporal_layers << " configured.";
+    return false;
+  }
+  aom_svc_params_t& svc_params = svc_params_.emplace();
+  svc_params.number_spatial_layers = svc_config.num_spatial_layers;
+  svc_params.number_temporal_layers = svc_config.num_temporal_layers;
+
+  int num_layers =
+      svc_config.num_spatial_layers * svc_config.num_temporal_layers;
+  for (int i = 0; i < num_layers; ++i) {
+    svc_params.min_quantizers[i] = kQpMin;
+    svc_params.max_quantizers[i] = encoder_settings_.qpMax;
+  }
+
+  // Assume each temporal layer doubles framerate.
+  for (int tid = 0; tid < svc_config.num_temporal_layers; ++tid) {
+    svc_params.framerate_factor[tid] =
+        1 << (svc_config.num_temporal_layers - tid - 1);
+  }
+
+  for (int sid = 0; sid < svc_config.num_spatial_layers; ++sid) {
+    svc_params.scaling_factor_num[sid] = svc_config.scaling_factor_num[sid];
+    svc_params.scaling_factor_den[sid] = svc_config.scaling_factor_den[sid];
+  }
+
+  return true;
+}
+
+void LibaomAv1Encoder::SetSvcLayerId(
+    const ScalableVideoController::LayerFrameConfig& layer_frame) {
+  aom_svc_layer_id_t layer_id = {};
+  layer_id.spatial_layer_id = layer_frame.SpatialId();
+  layer_id.temporal_layer_id = layer_frame.TemporalId();
+  aom_codec_err_t ret =
+      aom_codec_control(&ctx_, AV1E_SET_SVC_LAYER_ID, &layer_id);
+  if (ret != AOM_CODEC_OK) {
+    RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
+                        << " on control AV1E_SET_SVC_LAYER_ID.";
+  }
+}
+
+void LibaomAv1Encoder::SetSvcRefFrameConfig(
+    const ScalableVideoController::LayerFrameConfig& layer_frame) {
+  // Buffer name to use for each layer_frame.buffers position. In particular
+  // when there are 2 buffers are referenced, prefer name them last and golden,
+  // because av1 bitstream format has dedicated fields for these two names.
+  // See last_frame_idx and golden_frame_idx in the av1 spec
+  // https://aomediacodec.github.io/av1-spec/av1-spec.pdf
+  static constexpr int kPreferedSlotName[] = {0,  // Last
+                                              3,  // Golden
+                                              1, 2, 4, 5, 6};
+  static constexpr int kAv1NumBuffers = 8;
+
+  aom_svc_ref_frame_config_t ref_frame_config = {};
+  RTC_CHECK_LE(layer_frame.Buffers().size(), ABSL_ARRAYSIZE(kPreferedSlotName));
+  for (size_t i = 0; i < layer_frame.Buffers().size(); ++i) {
+    const CodecBufferUsage& buffer = layer_frame.Buffers()[i];
+    int slot_name = kPreferedSlotName[i];
+    RTC_CHECK_GE(buffer.id, 0);
+    RTC_CHECK_LT(buffer.id, kAv1NumBuffers);
+    ref_frame_config.ref_idx[slot_name] = buffer.id;
+    if (buffer.referenced) {
+      ref_frame_config.reference[slot_name] = 1;
+    }
+    if (buffer.updated) {
+      ref_frame_config.refresh[buffer.id] = 1;
+    }
+  }
+  aom_codec_err_t ret = aom_codec_control(&ctx_, AV1E_SET_SVC_REF_FRAME_CONFIG,
+                                          &ref_frame_config);
+  if (ret != AOM_CODEC_OK) {
+    RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
+                        << " on control AV1_SET_SVC_REF_FRAME_CONFIG.";
+  }
+}
+
 int32_t LibaomAv1Encoder::RegisterEncodeCompleteCallback(
     EncodedImageCallback* encoded_image_callback) {
   encoded_image_callback_ = encoded_image_callback;
@@ -235,10 +403,18 @@ int32_t LibaomAv1Encoder::Encode(
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
   }
 
-  keyframe_required_ =
+  bool keyframe_required =
       frame_types != nullptr &&
       absl::c_linear_search(*frame_types, VideoFrameType::kVideoFrameKey);
 
+  std::vector<ScalableVideoController::LayerFrameConfig> layer_frames =
+      svc_controller_->NextFrameConfig(keyframe_required);
+
+  if (layer_frames.empty()) {
+    RTC_LOG(LS_ERROR) << "SVCController returned no configuration for a frame.";
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
   // Convert input frame to I420, if needed.
   VideoFrame prepped_input_frame = frame;
   if (prepped_input_frame.video_frame_buffer()->type() !=
@@ -263,75 +439,96 @@ int32_t LibaomAv1Encoder::Encode(
 
   const uint32_t duration =
       kRtpTicksPerSecond / static_cast<float>(encoder_settings_.maxFramerate);
-  aom_enc_frame_flags_t flags = (keyframe_required_) ? AOM_EFLAG_FORCE_KF : 0;
 
-  // Encode a frame.
-  aom_codec_err_t ret = aom_codec_encode(&ctx_, frame_for_encode_,
-                                         frame.timestamp(), duration, flags);
-  if (ret != AOM_CODEC_OK) {
-    RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
-                        << " on aom_codec_encode.";
-    return WEBRTC_VIDEO_CODEC_ERROR;
-  }
+  for (ScalableVideoController::LayerFrameConfig& layer_frame : layer_frames) {
+    aom_enc_frame_flags_t flags =
+        layer_frame.IsKeyframe() ? AOM_EFLAG_FORCE_KF : 0;
 
-  // Get encoded image data.
-  EncodedImage encoded_image;
-  encoded_image._completeFrame = true;
-  aom_codec_iter_t iter = nullptr;
-  int data_pkt_count = 0;
-  while (const aom_codec_cx_pkt_t* pkt = aom_codec_get_cx_data(&ctx_, &iter)) {
-    if (pkt->kind == AOM_CODEC_CX_FRAME_PKT && pkt->data.frame.sz > 0) {
-      if (data_pkt_count > 0) {
-        RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encoder returned more than "
-                               "one data packet for an input video frame.";
-        Release();
-      }
-      // TODO(bugs.webrtc.org/11174): Remove this hack when
-      // webrtc_pc_e2e::SingleProcessEncodedImageDataInjector not used or fixed
-      // not to assume that encoded image transfered as is.
-      const uint8_t* data = static_cast<const uint8_t*>(pkt->data.frame.buf);
-      size_t size = pkt->data.frame.sz;
-      if (size > 2 && data[0] == 0b0'0010'010 && data[1] == 0) {
-        // Typically frame starts with a Temporal Delimter OBU of size 0 that is
-        // not need by any component in webrtc and discarded during rtp
-        // packetization. Before discarded it confuses test framework that
-        // assumes received encoded frame is exactly same as sent frame.
-        data += 2;
-        size -= 2;
-      }
-      encoded_image.SetEncodedData(EncodedImageBuffer::Create(data, size));
-
-      bool is_key_frame = ((pkt->data.frame.flags & AOM_EFLAG_FORCE_KF) != 0);
-      encoded_image._frameType = is_key_frame
-                                     ? VideoFrameType::kVideoFrameKey
-                                     : VideoFrameType::kVideoFrameDelta;
-      encoded_image.SetTimestamp(frame.timestamp());
-      encoded_image.capture_time_ms_ = frame.render_time_ms();
-      encoded_image.rotation_ = frame.rotation();
-      encoded_image.content_type_ = VideoContentType::UNSPECIFIED;
-      // If encoded image width/height info are added to aom_codec_cx_pkt_t,
-      // use those values in lieu of the values in frame.
-      encoded_image._encodedHeight = frame.height();
-      encoded_image._encodedWidth = frame.width();
-      encoded_image.timing_.flags = VideoSendTiming::kInvalid;
-      int qp = -1;
-      ret = aom_codec_control(&ctx_, AOME_GET_LAST_QUANTIZER, &qp);
-      if (ret != AOM_CODEC_OK) {
-        RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
-                            << " on control AOME_GET_LAST_QUANTIZER.";
-        return WEBRTC_VIDEO_CODEC_ERROR;
-      }
-      encoded_image.qp_ = qp;
-      encoded_image.SetColorSpace(frame.color_space());
-      ++data_pkt_count;
+    if (SvcEnabled()) {
+      SetSvcLayerId(layer_frame);
+      SetSvcRefFrameConfig(layer_frame);
     }
-  }
 
-  // Deliver encoded image data.
-  if (encoded_image.size() > 0) {
-    CodecSpecificInfo codec_specific_info;
-    encoded_image_callback_->OnEncodedImage(encoded_image, &codec_specific_info,
-                                            nullptr);
+    // Encode a frame.
+    aom_codec_err_t ret = aom_codec_encode(&ctx_, frame_for_encode_,
+                                           frame.timestamp(), duration, flags);
+    if (ret != AOM_CODEC_OK) {
+      RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
+                          << " on aom_codec_encode.";
+      return WEBRTC_VIDEO_CODEC_ERROR;
+    }
+
+    // Get encoded image data.
+    EncodedImage encoded_image;
+    encoded_image._completeFrame = true;
+    aom_codec_iter_t iter = nullptr;
+    int data_pkt_count = 0;
+    while (const aom_codec_cx_pkt_t* pkt =
+               aom_codec_get_cx_data(&ctx_, &iter)) {
+      if (pkt->kind == AOM_CODEC_CX_FRAME_PKT && pkt->data.frame.sz > 0) {
+        if (data_pkt_count > 0) {
+          RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encoder returned more than "
+                                 "one data packet for an input video frame.";
+          Release();
+        }
+        encoded_image.SetEncodedData(EncodedImageBuffer::Create(
+            /*data=*/static_cast<const uint8_t*>(pkt->data.frame.buf),
+            /*size=*/pkt->data.frame.sz));
+
+        if ((pkt->data.frame.flags & AOM_EFLAG_FORCE_KF) != 0) {
+          layer_frame.Keyframe();
+        }
+        encoded_image._frameType = layer_frame.IsKeyframe()
+                                       ? VideoFrameType::kVideoFrameKey
+                                       : VideoFrameType::kVideoFrameDelta;
+        encoded_image.SetTimestamp(frame.timestamp());
+        encoded_image.capture_time_ms_ = frame.render_time_ms();
+        encoded_image.rotation_ = frame.rotation();
+        encoded_image.content_type_ = VideoContentType::UNSPECIFIED;
+        // If encoded image width/height info are added to aom_codec_cx_pkt_t,
+        // use those values in lieu of the values in frame.
+        encoded_image._encodedHeight = frame.height();
+        encoded_image._encodedWidth = frame.width();
+        encoded_image.timing_.flags = VideoSendTiming::kInvalid;
+        int qp = -1;
+        ret = aom_codec_control(&ctx_, AOME_GET_LAST_QUANTIZER, &qp);
+        if (ret != AOM_CODEC_OK) {
+          RTC_LOG(LS_WARNING) << "LibaomAv1Encoder::Encode returned " << ret
+                              << " on control AOME_GET_LAST_QUANTIZER.";
+          return WEBRTC_VIDEO_CODEC_ERROR;
+        }
+        encoded_image.qp_ = qp;
+        encoded_image.SetColorSpace(frame.color_space());
+        ++data_pkt_count;
+      }
+    }
+
+    // Deliver encoded image data.
+    if (encoded_image.size() > 0) {
+      CodecSpecificInfo codec_specific_info;
+      codec_specific_info.codecType = kVideoCodecAV1;
+      bool is_keyframe = layer_frame.IsKeyframe();
+      codec_specific_info.generic_frame_info =
+          svc_controller_->OnEncodeDone(std::move(layer_frame));
+      if (is_keyframe && codec_specific_info.generic_frame_info) {
+        codec_specific_info.template_structure =
+            svc_controller_->DependencyStructure();
+        auto& resolutions = codec_specific_info.template_structure->resolutions;
+        if (SvcEnabled()) {
+          resolutions.resize(svc_params_->number_spatial_layers);
+          for (int sid = 0; sid < svc_params_->number_spatial_layers; ++sid) {
+            int n = svc_params_->scaling_factor_num[sid];
+            int d = svc_params_->scaling_factor_den[sid];
+            resolutions[sid] =
+                RenderResolution(cfg_.g_w * n / d, cfg_.g_h * n / d);
+          }
+        } else {
+          resolutions = {RenderResolution(cfg_.g_w, cfg_.g_h)};
+        }
+      }
+      encoded_image_callback_->OnEncodedImage(encoded_image,
+                                              &codec_specific_info, nullptr);
+    }
   }
 
   return WEBRTC_VIDEO_CODEC_OK;
@@ -359,9 +556,28 @@ void LibaomAv1Encoder::SetRates(const RateControlParameters& parameters) {
     RTC_DCHECK_LE(rc_target_bitrate_kbps, encoder_settings_.maxBitrate);
   RTC_DCHECK_GE(rc_target_bitrate_kbps, encoder_settings_.minBitrate);
 
+  svc_controller_->OnRatesUpdated(parameters.bitrate);
   // Set target bit rate.
   cfg_.rc_target_bitrate = rc_target_bitrate_kbps;
 
+  if (SvcEnabled()) {
+    for (int sid = 0; sid < svc_params_->number_spatial_layers; ++sid) {
+      // libaom bitrate for spatial id S and temporal id T means bitrate
+      // of frames with spatial_id=S and temporal_id<=T
+      // while `parameters.bitrate` provdies bitrate of frames with
+      // spatial_id=S and temporal_id=T
+      int accumulated_bitrate_bps = 0;
+      for (int tid = 0; tid < svc_params_->number_temporal_layers; ++tid) {
+        int layer_index = sid * svc_params_->number_temporal_layers + tid;
+        accumulated_bitrate_bps += parameters.bitrate.GetBitrate(sid, tid);
+        // `svc_params.layer_target_bitrate` expects bitrate in kbps.
+        svc_params_->layer_target_bitrate[layer_index] =
+            accumulated_bitrate_bps / 1000;
+      }
+    }
+    aom_codec_control(&ctx_, AV1E_SET_SVC_PARAMS, &*svc_params_);
+  }
+
   // Set frame rate to closest integer value.
   encoder_settings_.maxFramerate =
       static_cast<uint32_t>(parameters.framerate_fps + 0.5);
@@ -389,7 +605,13 @@ VideoEncoder::EncoderInfo LibaomAv1Encoder::GetEncoderInfo() const {
 const bool kIsLibaomAv1EncoderSupported = true;
 
 std::unique_ptr<VideoEncoder> CreateLibaomAv1Encoder() {
-  return std::make_unique<LibaomAv1Encoder>();
+  return std::make_unique<LibaomAv1Encoder>(
+      std::make_unique<ScalableVideoControllerNoLayering>());
+}
+
+std::unique_ptr<VideoEncoder> CreateLibaomAv1Encoder(
+    std::unique_ptr<ScalableVideoController> svc_controller) {
+  return std::make_unique<LibaomAv1Encoder>(std::move(svc_controller));
 }
 
 }  // namespace webrtc

modules/video_coding/codecs/av1/libaom_av1_encoder.h

@@ -14,12 +14,15 @@
 
 #include "absl/base/attributes.h"
 #include "api/video_codecs/video_encoder.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
 
 namespace webrtc {
 
 ABSL_CONST_INIT extern const bool kIsLibaomAv1EncoderSupported;
 
 std::unique_ptr<VideoEncoder> CreateLibaomAv1Encoder();
+std::unique_ptr<VideoEncoder> CreateLibaomAv1Encoder(
+    std::unique_ptr<ScalableVideoController> controller);
 
 }  // namespace webrtc
 

modules/video_coding/codecs/av1/libaom_av1_encoder_unittest.cc

@@ -11,15 +11,38 @@
 #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
 
 #include <memory>
+#include <vector>
 
+#include "absl/types/optional.h"
 #include "api/video_codecs/video_codec.h"
 #include "api/video_codecs/video_encoder.h"
+#include "modules/video_coding/codecs/av1/scalability_structure_l1t2.h"
+#include "modules/video_coding/codecs/test/encoded_video_frame_producer.h"
 #include "modules/video_coding/include/video_error_codes.h"
+#include "test/gmock.h"
 #include "test/gtest.h"
 
 namespace webrtc {
 namespace {
 
+using ::testing::SizeIs;
+
+VideoCodec DefaultCodecSettings() {
+  VideoCodec codec_settings;
+  codec_settings.width = 320;
+  codec_settings.height = 180;
+  codec_settings.maxFramerate = 30;
+  codec_settings.maxBitrate = 1000;
+  codec_settings.qpMax = 63;
+  return codec_settings;
+}
+
+VideoEncoder::Settings DefaultEncoderSettings() {
+  return VideoEncoder::Settings(
+      VideoEncoder::Capabilities(/*loss_notification=*/false),
+      /*number_of_cores=*/1, /*max_payload_size=*/1200);
+}
+
 TEST(LibaomAv1EncoderTest, CanCreate) {
   std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder();
   EXPECT_TRUE(encoder);
@@ -28,17 +51,37 @@ TEST(LibaomAv1EncoderTest, CanCreate) {
 TEST(LibaomAv1EncoderTest, InitAndRelease) {
   std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder();
   ASSERT_TRUE(encoder);
-  VideoCodec codec_settings;
-  codec_settings.width = 1280;
-  codec_settings.height = 720;
-  codec_settings.maxFramerate = 30;
-  VideoEncoder::Capabilities capabilities(/*loss_notification=*/false);
-  VideoEncoder::Settings encoder_settings(capabilities, /*number_of_cores=*/1,
-                                          /*max_payload_size=*/1200);
-  EXPECT_EQ(encoder->InitEncode(&codec_settings, encoder_settings),
+  VideoCodec codec_settings = DefaultCodecSettings();
+  EXPECT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()),
             WEBRTC_VIDEO_CODEC_OK);
   EXPECT_EQ(encoder->Release(), WEBRTC_VIDEO_CODEC_OK);
 }
 
+TEST(LibaomAv1EncoderTest, NoBitrateOnTopLayerRefecltedInActiveDecodeTargets) {
+  // Configure encoder with 2 temporal layers.
+  std::unique_ptr<VideoEncoder> encoder =
+      CreateLibaomAv1Encoder(std::make_unique<ScalabilityStructureL1T2>());
+  VideoCodec codec_settings = DefaultCodecSettings();
+  ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()),
+            WEBRTC_VIDEO_CODEC_OK);
+
+  VideoEncoder::RateControlParameters rate_parameters;
+  rate_parameters.framerate_fps = 30;
+  rate_parameters.bitrate.SetBitrate(0, /*temporal_index=*/0, 300'000);
+  rate_parameters.bitrate.SetBitrate(0, /*temporal_index=*/1, 0);
+  encoder->SetRates(rate_parameters);
+
+  std::vector<EncodedVideoFrameProducer::EncodedFrame> encoded_frames =
+      EncodedVideoFrameProducer(*encoder).SetNumInputFrames(1).Encode();
+  ASSERT_THAT(encoded_frames, SizeIs(1));
+  ASSERT_NE(encoded_frames[0].codec_specific_info.generic_frame_info,
+            absl::nullopt);
+  // Assuming L1T2 structure uses 1st decode target for T0 and 2nd decode target
+  // for T0+T1 frames, expect only 1st decode target is active.
+  EXPECT_EQ(encoded_frames[0]
+                .codec_specific_info.generic_frame_info->active_decode_targets,
+            0b01);
+}
+
 }  // namespace
 }  // namespace webrtc

modules/video_coding/codecs/av1/libaom_av1_unittest.cc

@@ -11,16 +11,23 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include <map>
 #include <memory>
+#include <ostream>
+#include <tuple>
 #include <vector>
 
 #include "absl/types/optional.h"
-#include "api/test/create_frame_generator.h"
-#include "api/test/frame_generator_interface.h"
+#include "api/units/data_size.h"
+#include "api/units/time_delta.h"
 #include "api/video_codecs/video_codec.h"
 #include "api/video_codecs/video_encoder.h"
+#include "modules/video_coding/codecs/av1/create_scalability_structure.h"
 #include "modules/video_coding/codecs/av1/libaom_av1_decoder.h"
 #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller_no_layering.h"
+#include "modules/video_coding/codecs/test/encoded_video_frame_producer.h"
 #include "modules/video_coding/include/video_codec_interface.h"
 #include "modules/video_coding/include/video_error_codes.h"
 #include "test/gmock.h"
@@ -29,79 +36,44 @@
 namespace webrtc {
 namespace {
 
+using ::testing::ContainerEq;
+using ::testing::Each;
 using ::testing::ElementsAreArray;
+using ::testing::Ge;
 using ::testing::IsEmpty;
 using ::testing::Not;
 using ::testing::NotNull;
+using ::testing::Pointwise;
+using ::testing::SizeIs;
+using ::testing::Truly;
+using ::testing::Values;
 
 // Use small resolution for this test to make it faster.
 constexpr int kWidth = 320;
 constexpr int kHeight = 180;
 constexpr int kFramerate = 30;
-constexpr int kRtpTicksPerSecond = 90000;
 
-class TestAv1Encoder {
- public:
-  struct Encoded {
-    EncodedImage encoded_image;
-    CodecSpecificInfo codec_specific_info;
-  };
-
-  TestAv1Encoder() : encoder_(CreateLibaomAv1Encoder()) {
-    RTC_CHECK(encoder_);
-    VideoCodec codec_settings;
-    codec_settings.width = kWidth;
-    codec_settings.height = kHeight;
-    codec_settings.maxFramerate = kFramerate;
-    VideoEncoder::Settings encoder_settings(
-        VideoEncoder::Capabilities(/*loss_notification=*/false),
-        /*number_of_cores=*/1, /*max_payload_size=*/1200);
-    EXPECT_EQ(encoder_->InitEncode(&codec_settings, encoder_settings),
-              WEBRTC_VIDEO_CODEC_OK);
-    EXPECT_EQ(encoder_->RegisterEncodeCompleteCallback(&callback_),
-              WEBRTC_VIDEO_CODEC_OK);
-  }
-  // This class requires pointer stability and thus not copyable nor movable.
-  TestAv1Encoder(const TestAv1Encoder&) = delete;
-  TestAv1Encoder& operator=(const TestAv1Encoder&) = delete;
-
-  void EncodeAndAppend(const VideoFrame& frame, std::vector<Encoded>* encoded) {
-    callback_.SetEncodeStorage(encoded);
-    std::vector<VideoFrameType> frame_types = {
-        VideoFrameType::kVideoFrameDelta};
-    EXPECT_EQ(encoder_->Encode(frame, &frame_types), WEBRTC_VIDEO_CODEC_OK);
-    // Prefer to crash checking nullptr rather than writing to random memory.
-    callback_.SetEncodeStorage(nullptr);
-  }
-
- private:
-  class EncoderCallback : public EncodedImageCallback {
-   public:
-    void SetEncodeStorage(std::vector<Encoded>* storage) { storage_ = storage; }
-
-   private:
-    Result OnEncodedImage(
-        const EncodedImage& encoded_image,
-        const CodecSpecificInfo* codec_specific_info,
-        const RTPFragmentationHeader* /*fragmentation*/) override {
-      RTC_CHECK(storage_);
-      storage_->push_back({encoded_image, *codec_specific_info});
-      return Result(Result::Error::OK);
-    }
-
-    std::vector<Encoded>* storage_ = nullptr;
-  };
-
-  EncoderCallback callback_;
-  std::unique_ptr<VideoEncoder> encoder_;
-};
+VideoCodec DefaultCodecSettings() {
+  VideoCodec codec_settings;
+  codec_settings.width = kWidth;
+  codec_settings.height = kHeight;
+  codec_settings.maxFramerate = kFramerate;
+  codec_settings.maxBitrate = 1000;
+  codec_settings.qpMax = 63;
+  return codec_settings;
+}
+VideoEncoder::Settings DefaultEncoderSettings() {
+  return VideoEncoder::Settings(
+      VideoEncoder::Capabilities(/*loss_notification=*/false),
+      /*number_of_cores=*/1, /*max_payload_size=*/1200);
+}
 
 class TestAv1Decoder {
  public:
-  TestAv1Decoder() {
-    decoder_ = CreateLibaomAv1Decoder();
+  explicit TestAv1Decoder(int decoder_id)
+      : decoder_id_(decoder_id), decoder_(CreateLibaomAv1Decoder()) {
     if (decoder_ == nullptr) {
-      ADD_FAILURE() << "Failed to create a decoder";
+      ADD_FAILURE() << "Failed to create a decoder#" << decoder_id_;
       return;
     }
     EXPECT_EQ(decoder_->InitDecode(/*codec_settings=*/nullptr,
@@ -116,20 +88,17 @@ class TestAv1Decoder {
 
   void Decode(int64_t frame_id, const EncodedImage& image) {
     ASSERT_THAT(decoder_, NotNull());
-    requested_ids_.push_back(frame_id);
     int32_t error = decoder_->Decode(image, /*missing_frames=*/false,
                                      /*render_time_ms=*/image.capture_time_ms_);
     if (error != WEBRTC_VIDEO_CODEC_OK) {
       ADD_FAILURE() << "Failed to decode frame id " << frame_id
-                    << " with error code " << error;
+                    << " with error code " << error << " by decoder#"
+                    << decoder_id_;
       return;
     }
     decoded_ids_.push_back(frame_id);
   }
 
-  const std::vector<int64_t>& requested_frame_ids() const {
-    return requested_ids_;
-  }
   const std::vector<int64_t>& decoded_frame_ids() const { return decoded_ids_; }
   size_t num_output_frames() const { return callback_.num_called(); }
 
@@ -156,51 +125,208 @@ class TestAv1Decoder {
     int num_called_ = 0;
   };
 
-  std::vector<int64_t> requested_ids_;
+  const int decoder_id_;
   std::vector<int64_t> decoded_ids_;
   DecoderCallback callback_;
-  std::unique_ptr<VideoDecoder> decoder_;
+  const std::unique_ptr<VideoDecoder> decoder_;
 };
 
-std::vector<VideoFrame> GenerateFrames(size_t num_frames) {
-  std::vector<VideoFrame> frames;
-  frames.reserve(num_frames);
-
-  auto input_frame_generator = test::CreateSquareFrameGenerator(
-      kWidth, kHeight, test::FrameGeneratorInterface::OutputType::kI420,
-      absl::nullopt);
-  uint32_t timestamp = 1000;
-  for (size_t i = 0; i < num_frames; ++i) {
-    frames.push_back(
-        VideoFrame::Builder()
-            .set_video_frame_buffer(input_frame_generator->NextFrame().buffer)
-            .set_timestamp_rtp(timestamp += kRtpTicksPerSecond / kFramerate)
-            .build());
-  }
-  return frames;
-}
-
 TEST(LibaomAv1Test, EncodeDecode) {
-  TestAv1Decoder decoder;
-  TestAv1Encoder encoder;
+  TestAv1Decoder decoder(0);
+  std::unique_ptr<VideoEncoder> encoder = CreateLibaomAv1Encoder();
+  VideoCodec codec_settings = DefaultCodecSettings();
+  ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()),
+            WEBRTC_VIDEO_CODEC_OK);
 
-  std::vector<TestAv1Encoder::Encoded> encoded_frames;
-  for (const VideoFrame& frame : GenerateFrames(/*num_frames=*/4)) {
-    encoder.EncodeAndAppend(frame, &encoded_frames);
-  }
-  for (size_t frame_idx = 0; frame_idx < encoded_frames.size(); ++frame_idx) {
-    decoder.Decode(static_cast<int64_t>(frame_idx),
-                   encoded_frames[frame_idx].encoded_image);
+  std::vector<EncodedVideoFrameProducer::EncodedFrame> encoded_frames =
+      EncodedVideoFrameProducer(*encoder).SetNumInputFrames(4).Encode();
+  for (size_t frame_id = 0; frame_id < encoded_frames.size(); ++frame_id) {
+    decoder.Decode(static_cast<int64_t>(frame_id),
+                   encoded_frames[frame_id].encoded_image);
   }
 
   // Check encoder produced some frames for decoder to decode.
   ASSERT_THAT(encoded_frames, Not(IsEmpty()));
   // Check decoder found all of them valid.
-  EXPECT_THAT(decoder.decoded_frame_ids(),
-              ElementsAreArray(decoder.requested_frame_ids()));
+  EXPECT_THAT(decoder.decoded_frame_ids(), SizeIs(encoded_frames.size()));
   // Check each of them produced an output frame.
   EXPECT_EQ(decoder.num_output_frames(), decoder.decoded_frame_ids().size());
 }
 
+struct LayerId {
+  friend bool operator==(const LayerId& lhs, const LayerId& rhs) {
+    return std::tie(lhs.spatial_id, lhs.temporal_id) ==
+           std::tie(rhs.spatial_id, rhs.temporal_id);
+  }
+  friend bool operator<(const LayerId& lhs, const LayerId& rhs) {
+    return std::tie(lhs.spatial_id, lhs.temporal_id) <
+           std::tie(rhs.spatial_id, rhs.temporal_id);
+  }
+  friend std::ostream& operator<<(std::ostream& s, const LayerId& layer) {
+    return s << "S" << layer.spatial_id << "T" << layer.temporal_id;
+  }
+
+  int spatial_id = 0;
+  int temporal_id = 0;
+};
+
+struct SvcTestParam {
+  std::string name;
+  int num_frames_to_generate;
+  std::map<LayerId, DataRate> configured_bitrates;
+};
+
+class LibaomAv1SvcTest : public ::testing::TestWithParam<SvcTestParam> {};
+
+TEST_P(LibaomAv1SvcTest, EncodeAndDecodeAllDecodeTargets) {
+  std::unique_ptr<ScalableVideoController> svc_controller =
+      CreateScalabilityStructure(GetParam().name);
+  size_t num_decode_targets =
+      svc_controller->DependencyStructure().num_decode_targets;
+
+  std::unique_ptr<VideoEncoder> encoder =
+      CreateLibaomAv1Encoder(std::move(svc_controller));
+  VideoCodec codec_settings = DefaultCodecSettings();
+  ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()),
+            WEBRTC_VIDEO_CODEC_OK);
+  std::vector<EncodedVideoFrameProducer::EncodedFrame> encoded_frames =
+      EncodedVideoFrameProducer(*encoder)
+          .SetNumInputFrames(GetParam().num_frames_to_generate)
+          .SetResolution({kWidth, kHeight})
+          .Encode();
+
+  ASSERT_THAT(
+      encoded_frames,
+      Each(Truly([&](const EncodedVideoFrameProducer::EncodedFrame& frame) {
+        return frame.codec_specific_info.generic_frame_info &&
+               frame.codec_specific_info.generic_frame_info
+                       ->decode_target_indications.size() == num_decode_targets;
+      })));
+
+  for (size_t dt = 0; dt < num_decode_targets; ++dt) {
+    TestAv1Decoder decoder(dt);
+    std::vector<int64_t> requested_ids;
+    for (int64_t frame_id = 0;
+         frame_id < static_cast<int64_t>(encoded_frames.size()); ++frame_id) {
+      const EncodedVideoFrameProducer::EncodedFrame& frame =
+          encoded_frames[frame_id];
+      if (frame.codec_specific_info.generic_frame_info
+              ->decode_target_indications[dt] !=
+          DecodeTargetIndication::kNotPresent) {
+        requested_ids.push_back(frame_id);
+        decoder.Decode(frame_id, frame.encoded_image);
+      }
+    }
+
+    ASSERT_THAT(requested_ids, SizeIs(Ge(2u)));
+    // Check decoder found all of them valid.
+    EXPECT_THAT(decoder.decoded_frame_ids(), ContainerEq(requested_ids))
+        << "Decoder#" << dt;
+    // Check each of them produced an output frame.
+    EXPECT_EQ(decoder.num_output_frames(), decoder.decoded_frame_ids().size())
+        << "Decoder#" << dt;
+  }
+}
+
+MATCHER(SameLayerIdAndBitrateIsNear, "") {
+  // First check if layer id is the same.
+  return std::get<0>(arg).first == std::get<1>(arg).first &&
+         // check measured bitrate is not much lower than requested.
+         std::get<0>(arg).second >= std::get<1>(arg).second * 0.8 &&
+         // check measured bitrate is not much larger than requested.
+         std::get<0>(arg).second <= std::get<1>(arg).second * 1.1;
+}
+
+TEST_P(LibaomAv1SvcTest, SetRatesMatchMeasuredBitrate) {
+  const SvcTestParam param = GetParam();
+  if (param.configured_bitrates.empty()) {
+    // Rates are not configured for this particular structure, skip the test.
+    return;
+  }
+  constexpr TimeDelta kDuration = TimeDelta::Seconds(5);
+
+  VideoBitrateAllocation allocation;
+  for (const auto& kv : param.configured_bitrates) {
+    allocation.SetBitrate(kv.first.spatial_id, kv.first.temporal_id,
+                          kv.second.bps());
+  }
+
+  std::unique_ptr<VideoEncoder> encoder =
+      CreateLibaomAv1Encoder(CreateScalabilityStructure(param.name));
+  ASSERT_TRUE(encoder);
+  VideoCodec codec_settings = DefaultCodecSettings();
+  codec_settings.maxBitrate = allocation.get_sum_kbps();
+  codec_settings.maxFramerate = 30;
+  ASSERT_EQ(encoder->InitEncode(&codec_settings, DefaultEncoderSettings()),
+            WEBRTC_VIDEO_CODEC_OK);
+
+  encoder->SetRates(VideoEncoder::RateControlParameters(
+      allocation, codec_settings.maxFramerate));
+
+  std::vector<EncodedVideoFrameProducer::EncodedFrame> encoded_frames =
+      EncodedVideoFrameProducer(*encoder)
+          .SetNumInputFrames(codec_settings.maxFramerate * kDuration.seconds())
+          .SetResolution({codec_settings.width, codec_settings.height})
+          .SetFramerateFps(codec_settings.maxFramerate)
+          .Encode();
+
+  // Calculate size of each layer.
+  std::map<LayerId, DataSize> layer_size;
+  for (const auto& frame : encoded_frames) {
+    ASSERT_TRUE(frame.codec_specific_info.generic_frame_info);
+    const auto& layer = *frame.codec_specific_info.generic_frame_info;
+    LayerId layer_id = {layer.spatial_id, layer.temporal_id};
+    // This is almost same as
+    // layer_size[layer_id] += DataSize::Bytes(frame.encoded_image.size());
+    // but avoids calling deleted default constructor for DataSize.
+    layer_size.emplace(layer_id, DataSize::Zero()).first->second +=
+        DataSize::Bytes(frame.encoded_image.size());
+  }
+  // Convert size of the layer into bitrate of that layer.
+  std::vector<std::pair<LayerId, DataRate>> measured_bitrates;
+  for (const auto& kv : layer_size) {
+    measured_bitrates.emplace_back(kv.first, kv.second / kDuration);
+  }
+  EXPECT_THAT(measured_bitrates, Pointwise(SameLayerIdAndBitrateIsNear(),
+                                           param.configured_bitrates));
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    Svc,
+    LibaomAv1SvcTest,
+    Values(SvcTestParam{"NONE", /*num_frames_to_generate=*/4},
+           SvcTestParam{"L1T2",
+                        /*num_frames_to_generate=*/4,
+                        /*configured_bitrates=*/
+                        {{{0, 0}, DataRate::KilobitsPerSec(60)},
+                         {{0, 1}, DataRate::KilobitsPerSec(40)}}},
+           SvcTestParam{"L1T3", /*num_frames_to_generate=*/8},
+           SvcTestParam{"L2T1",
+                        /*num_frames_to_generate=*/3,
+                        /*configured_bitrates=*/
+                        {{{0, 0}, DataRate::KilobitsPerSec(30)},
+                         {{1, 0}, DataRate::KilobitsPerSec(70)}}},
+           SvcTestParam{"L2T1h",
+                        /*num_frames_to_generate=*/3,
+                        /*configured_bitrates=*/
+                        {{{0, 0}, DataRate::KilobitsPerSec(30)},
+                         {{1, 0}, DataRate::KilobitsPerSec(70)}}},
+           SvcTestParam{"L2T1_KEY", /*num_frames_to_generate=*/3},
+           SvcTestParam{"L3T1", /*num_frames_to_generate=*/3},
+           SvcTestParam{"L3T3", /*num_frames_to_generate=*/8},
+           SvcTestParam{"S2T1", /*num_frames_to_generate=*/3},
+           SvcTestParam{"L2T2", /*num_frames_to_generate=*/4},
+           SvcTestParam{"L2T2_KEY", /*num_frames_to_generate=*/4},
+           SvcTestParam{"L2T2_KEY_SHIFT",
+                        /*num_frames_to_generate=*/4,
+                        /*configured_bitrates=*/
+                        {{{0, 0}, DataRate::KilobitsPerSec(70)},
+                         {{0, 1}, DataRate::KilobitsPerSec(30)},
+                         {{1, 0}, DataRate::KilobitsPerSec(110)},
+                         {{1, 1}, DataRate::KilobitsPerSec(80)}}}),
+    [](const testing::TestParamInfo<SvcTestParam>& info) {
+      return info.param.name;
+    });
+
 }  // namespace
 }  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l1t2.cc

@@ -0,0 +1,123 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/scalability_structure_l1t2.h"
+
+#include <utility>
+#include <vector>
+
+#include "absl/base/macros.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+namespace {
+
+constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent;
+constexpr auto kDiscardable = DecodeTargetIndication::kDiscardable;
+constexpr auto kSwitch = DecodeTargetIndication::kSwitch;
+
+constexpr DecodeTargetIndication kDtis[3][2] = {
+    {kSwitch, kSwitch},           // KeyFrame
+    {kNotPresent, kDiscardable},  // DeltaFrame T1
+    {kSwitch, kSwitch},           // DeltaFrame T0
+};
+
+}  // namespace
+
+ScalabilityStructureL1T2::~ScalabilityStructureL1T2() = default;
+
+ScalableVideoController::StreamLayersConfig
+ScalabilityStructureL1T2::StreamConfig() const {
+  StreamLayersConfig result;
+  result.num_spatial_layers = 1;
+  result.num_temporal_layers = 2;
+  return result;
+}
+
+FrameDependencyStructure ScalabilityStructureL1T2::DependencyStructure() const {
+  FrameDependencyStructure structure;
+  structure.num_decode_targets = 2;
+  structure.num_chains = 1;
+  structure.decode_target_protected_by_chain = {0, 0};
+  structure.templates.resize(3);
+  structure.templates[0].T(0).Dtis("SS").ChainDiffs({0});
+  structure.templates[1].T(0).Dtis("SS").ChainDiffs({2}).FrameDiffs({2});
+  structure.templates[2].T(1).Dtis("-D").ChainDiffs({1}).FrameDiffs({1});
+  return structure;
+}
+
+std::vector<ScalableVideoController::LayerFrameConfig>
+ScalabilityStructureL1T2::NextFrameConfig(bool restart) {
+  if (!active_decode_targets_[0]) {
+    RTC_LOG(LS_WARNING) << "No bitrate allocated for temporal layer 0, yet "
+                           "frame is requested. No frame will be encoded.";
+    return {};
+  }
+  if (restart) {
+    next_pattern_ = kKeyFrame;
+  } else if (!active_decode_targets_[1]) {
+    next_pattern_ = kDeltaFrameT0;
+  }
+  std::vector<LayerFrameConfig> result(1);
+
+  switch (next_pattern_) {
+    case kKeyFrame:
+      result[0].Id(0).T(0).Keyframe().Update(0);
+      next_pattern_ = kDeltaFrameT1;
+      break;
+    case kDeltaFrameT1:
+      result[0].Id(1).T(1).Reference(0);
+      next_pattern_ = kDeltaFrameT0;
+      break;
+    case kDeltaFrameT0:
+      result[0].Id(2).T(0).ReferenceAndUpdate(0);
+      next_pattern_ = kDeltaFrameT1;
+      break;
+  }
+  return result;
+}
+
+absl::optional<GenericFrameInfo> ScalabilityStructureL1T2::OnEncodeDone(
+    LayerFrameConfig config) {
+  // Encoder may have generated a keyframe even when not asked for it. Treat
+  // such frame same as requested keyframe, in particular restart the sequence.
+  if (config.IsKeyframe()) {
+    config = NextFrameConfig(/*restart=*/true).front();
+  }
+
+  absl::optional<GenericFrameInfo> frame_info;
+  if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) {
+    RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id();
+    return frame_info;
+  }
+  frame_info.emplace();
+  frame_info->temporal_id = config.TemporalId();
+  frame_info->encoder_buffers = config.Buffers();
+  frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]),
+                                               std::end(kDtis[config.Id()]));
+  frame_info->part_of_chain = {config.TemporalId() == 0};
+  frame_info->active_decode_targets = active_decode_targets_;
+  return frame_info;
+}
+
+void ScalabilityStructureL1T2::OnRatesUpdated(
+    const VideoBitrateAllocation& bitrates) {
+  if (bitrates.GetBitrate(0, 0) == 0) {
+    // It is unclear what frame can be produced when base layer is disabled,
+    // so mark all decode targets as inactive to produce no frames.
+    active_decode_targets_.reset();
+    return;
+  }
+  active_decode_targets_.set(0, true);
+  active_decode_targets_.set(1, bitrates.GetBitrate(0, 1) > 0);
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l1t2.h

@@ -0,0 +1,48 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T2_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T2_H_
+
+#include <bitset>
+#include <vector>
+
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+class ScalabilityStructureL1T2 : public ScalableVideoController {
+ public:
+  ~ScalabilityStructureL1T2() override;
+
+  StreamLayersConfig StreamConfig() const override;
+  FrameDependencyStructure DependencyStructure() const override;
+
+  std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override;
+  absl::optional<GenericFrameInfo> OnEncodeDone(
+      LayerFrameConfig config) override;
+
+  void OnRatesUpdated(const VideoBitrateAllocation& bitrates) override;
+
+ private:
+  enum FramePattern {
+    kKeyFrame,
+    kDeltaFrameT1,
+    kDeltaFrameT0,
+  };
+
+  FramePattern next_pattern_ = kKeyFrame;
+  std::bitset<32> active_decode_targets_ = 0b11;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T2_H_

modules/video_coding/codecs/av1/scalability_structure_l1t3.cc

@@ -0,0 +1,109 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/scalability_structure_l1t3.h"
+
+#include <utility>
+#include <vector>
+
+#include "absl/base/macros.h"
+#include "absl/types/optional.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+namespace {
+
+constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent;
+constexpr auto kDiscardable = DecodeTargetIndication::kDiscardable;
+constexpr auto kSwitch = DecodeTargetIndication::kSwitch;
+
+constexpr DecodeTargetIndication kDtis[3][3] = {
+    {kSwitch, kSwitch, kSwitch},               // T0
+    {kNotPresent, kDiscardable, kSwitch},      // T1
+    {kNotPresent, kNotPresent, kDiscardable},  // T2
+};
+
+}  // namespace
+
+ScalabilityStructureL1T3::~ScalabilityStructureL1T3() = default;
+
+ScalableVideoController::StreamLayersConfig
+ScalabilityStructureL1T3::StreamConfig() const {
+  StreamLayersConfig result;
+  result.num_spatial_layers = 1;
+  result.num_temporal_layers = 3;
+  return result;
+}
+
+FrameDependencyStructure ScalabilityStructureL1T3::DependencyStructure() const {
+  FrameDependencyStructure structure;
+  structure.num_decode_targets = 3;
+  structure.num_chains = 1;
+  structure.decode_target_protected_by_chain = {0, 0, 0};
+  structure.templates.resize(5);
+  structure.templates[0].T(0).Dtis("SSS").ChainDiffs({0});
+  structure.templates[1].T(0).Dtis("SSS").ChainDiffs({4}).FrameDiffs({4});
+  structure.templates[2].T(1).Dtis("-DS").ChainDiffs({2}).FrameDiffs({2});
+  structure.templates[3].T(2).Dtis("--D").ChainDiffs({1}).FrameDiffs({1});
+  structure.templates[4].T(2).Dtis("--D").ChainDiffs({3}).FrameDiffs({1});
+  return structure;
+}
+
+std::vector<ScalableVideoController::LayerFrameConfig>
+ScalabilityStructureL1T3::NextFrameConfig(bool restart) {
+  if (restart) {
+    next_pattern_ = kKeyFrame;
+  }
+  std::vector<LayerFrameConfig> config(1);
+
+  switch (next_pattern_) {
+    case kKeyFrame:
+      config[0].T(0).Keyframe().Update(0);
+      next_pattern_ = kDeltaFrameT2A;
+      break;
+    case kDeltaFrameT2A:
+      config[0].T(2).Reference(0);
+      next_pattern_ = kDeltaFrameT1;
+      break;
+    case kDeltaFrameT1:
+      config[0].T(1).Reference(0).Update(1);
+      next_pattern_ = kDeltaFrameT2B;
+      break;
+    case kDeltaFrameT2B:
+      config[0].T(2).Reference(1);
+      next_pattern_ = kDeltaFrameT0;
+      break;
+    case kDeltaFrameT0:
+      config[0].T(0).ReferenceAndUpdate(0);
+      next_pattern_ = kDeltaFrameT2A;
+      break;
+  }
+  return config;
+}
+
+absl::optional<GenericFrameInfo> ScalabilityStructureL1T3::OnEncodeDone(
+    LayerFrameConfig config) {
+  absl::optional<GenericFrameInfo> frame_info;
+  if (config.TemporalId() < 0 ||
+      config.TemporalId() >= int{ABSL_ARRAYSIZE(kDtis)}) {
+    RTC_LOG(LS_ERROR) << "Unexpected temporal id " << config.TemporalId();
+    return frame_info;
+  }
+  frame_info.emplace();
+  frame_info->temporal_id = config.TemporalId();
+  frame_info->encoder_buffers = config.Buffers();
+  frame_info->decode_target_indications.assign(
+      std::begin(kDtis[config.TemporalId()]),
+      std::end(kDtis[config.TemporalId()]));
+  frame_info->part_of_chain = {config.TemporalId() == 0};
+  return frame_info;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l1t3.h

@@ -0,0 +1,53 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T3_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T3_H_
+
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+// T2       0   0   0   0
+//          |  /    |  /
+// T1       / 0     / 0  ...
+//         |_/     |_/
+// T0     0-------0------
+// Time-> 0 1 2 3 4 5 6 7
+class ScalabilityStructureL1T3 : public ScalableVideoController {
+ public:
+  ~ScalabilityStructureL1T3() override;
+
+  StreamLayersConfig StreamConfig() const override;
+  FrameDependencyStructure DependencyStructure() const override;
+
+  std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override;
+  absl::optional<GenericFrameInfo> OnEncodeDone(
+      LayerFrameConfig config) override;
+
+ private:
+  enum FramePattern {
+    kKeyFrame,
+    kDeltaFrameT2A,
+    kDeltaFrameT1,
+    kDeltaFrameT2B,
+    kDeltaFrameT0,
+  };
+
+  FramePattern next_pattern_ = kKeyFrame;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L1T3_H_

modules/video_coding/codecs/av1/scalability_structure_l2t1.cc

@@ -0,0 +1,102 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t1.h"
+
+#include <utility>
+#include <vector>
+
+#include "absl/base/macros.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+namespace {
+
+constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent;
+constexpr auto kSwitch = DecodeTargetIndication::kSwitch;
+constexpr auto kRequired = DecodeTargetIndication::kRequired;
+
+constexpr DecodeTargetIndication kDtis[4][2] = {
+    {kSwitch, kSwitch},        // Key, S0
+    {kNotPresent, kSwitch},    // Key, S1
+    {kSwitch, kRequired},      // Delta, S0
+    {kNotPresent, kRequired},  // Delta, S1
+};
+
+}  // namespace
+
+ScalabilityStructureL2T1::~ScalabilityStructureL2T1() = default;
+
+ScalableVideoController::StreamLayersConfig
+ScalabilityStructureL2T1::StreamConfig() const {
+  StreamLayersConfig result;
+  result.num_spatial_layers = 2;
+  result.num_temporal_layers = 1;
+  result.scaling_factor_num[0] = 1;
+  result.scaling_factor_den[0] = 2;
+  return result;
+}
+
+FrameDependencyStructure ScalabilityStructureL2T1::DependencyStructure() const {
+  FrameDependencyStructure structure;
+  structure.num_decode_targets = 2;
+  structure.num_chains = 2;
+  structure.decode_target_protected_by_chain = {0, 1};
+  structure.templates.resize(4);
+  structure.templates[0].S(0).Dtis("SR").ChainDiffs({2, 1}).FrameDiffs({2});
+  structure.templates[1].S(0).Dtis("SS").ChainDiffs({0, 0});
+  structure.templates[2].S(1).Dtis("-R").ChainDiffs({1, 1}).FrameDiffs({2, 1});
+  structure.templates[3].S(1).Dtis("-S").ChainDiffs({1, 1}).FrameDiffs({1});
+  return structure;
+}
+
+ScalableVideoController::LayerFrameConfig
+ScalabilityStructureL2T1::KeyFrameConfig() const {
+  return LayerFrameConfig().Id(0).S(0).Keyframe().Update(0);
+}
+
+std::vector<ScalableVideoController::LayerFrameConfig>
+ScalabilityStructureL2T1::NextFrameConfig(bool restart) {
+  std::vector<LayerFrameConfig> result(2);
+  // Buffer0 keeps latest S0 frame, Buffer1 keeps latest S1 frame.
+  if (restart || keyframe_) {
+    result[0] = KeyFrameConfig();
+    result[1].Id(1).S(1).Reference(0).Update(1);
+    keyframe_ = false;
+  } else {
+    result[0].Id(2).S(0).ReferenceAndUpdate(0);
+    result[1].Id(3).S(1).Reference(0).ReferenceAndUpdate(1);
+  }
+  return result;
+}
+
+absl::optional<GenericFrameInfo> ScalabilityStructureL2T1::OnEncodeDone(
+    LayerFrameConfig config) {
+  absl::optional<GenericFrameInfo> frame_info;
+  if (config.IsKeyframe()) {
+    config = KeyFrameConfig();
+  }
+
+  if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) {
+    RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id();
+    return frame_info;
+  }
+  frame_info.emplace();
+  frame_info->spatial_id = config.SpatialId();
+  frame_info->temporal_id = config.TemporalId();
+  frame_info->encoder_buffers = std::move(config.Buffers());
+  frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]),
+                                               std::end(kDtis[config.Id()]));
+  frame_info->part_of_chain = {config.SpatialId() == 0, true};
+  return frame_info;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l2t1.h

@@ -0,0 +1,43 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_H_
+
+#include <vector>
+
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+// S1  0--0--0-
+//     |  |  | ...
+// S0  0--0--0-
+class ScalabilityStructureL2T1 : public ScalableVideoController {
+ public:
+  ~ScalabilityStructureL2T1() override;
+
+  StreamLayersConfig StreamConfig() const override;
+  FrameDependencyStructure DependencyStructure() const override;
+
+  std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override;
+  absl::optional<GenericFrameInfo> OnEncodeDone(
+      LayerFrameConfig config) override;
+
+ private:
+  LayerFrameConfig KeyFrameConfig() const;
+
+  bool keyframe_ = true;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_H_

modules/video_coding/codecs/av1/scalability_structure_l2t1_key.cc

@@ -0,0 +1,107 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h"
+
+#include <utility>
+#include <vector>
+
+#include "absl/base/macros.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+namespace {
+
+constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent;
+constexpr auto kSwitch = DecodeTargetIndication::kSwitch;
+
+constexpr DecodeTargetIndication kDtis[3][2] = {
+    {kSwitch, kSwitch},      // Key, S0
+    {kSwitch, kNotPresent},  // Delta, S0
+    {kNotPresent, kSwitch},  // Key and Delta, S1
+};
+
+}  // namespace
+
+ScalabilityStructureL2T1Key::~ScalabilityStructureL2T1Key() = default;
+
+ScalableVideoController::StreamLayersConfig
+ScalabilityStructureL2T1Key::StreamConfig() const {
+  StreamLayersConfig result;
+  result.num_spatial_layers = 2;
+  result.num_temporal_layers = 1;
+  result.scaling_factor_num[0] = 1;
+  result.scaling_factor_den[0] = 2;
+  return result;
+}
+
+FrameDependencyStructure ScalabilityStructureL2T1Key::DependencyStructure()
+    const {
+  FrameDependencyStructure structure;
+  structure.num_decode_targets = 2;
+  structure.num_chains = 2;
+  structure.decode_target_protected_by_chain = {0, 1};
+  structure.templates.resize(4);
+  structure.templates[0].S(0).Dtis("S-").ChainDiffs({2, 1}).FrameDiffs({2});
+  structure.templates[1].S(0).Dtis("SS").ChainDiffs({0, 0});
+  structure.templates[2].S(1).Dtis("-S").ChainDiffs({1, 2}).FrameDiffs({2});
+  structure.templates[3].S(1).Dtis("-S").ChainDiffs({1, 1}).FrameDiffs({1});
+  return structure;
+}
+
+ScalableVideoController::LayerFrameConfig
+ScalabilityStructureL2T1Key::KeyFrameConfig() const {
+  return LayerFrameConfig().Id(0).S(0).Keyframe().Update(0);
+}
+
+std::vector<ScalableVideoController::LayerFrameConfig>
+ScalabilityStructureL2T1Key::NextFrameConfig(bool restart) {
+  std::vector<LayerFrameConfig> result(2);
+
+  // Buffer0 keeps latest S0T0 frame, Buffer1 keeps latest S1T0 frame.
+  if (restart || keyframe_) {
+    result[0] = KeyFrameConfig();
+    result[1].Id(2).S(1).Reference(0).Update(1);
+    keyframe_ = false;
+  } else {
+    result[0].Id(1).S(0).ReferenceAndUpdate(0);
+    result[1].Id(2).S(1).ReferenceAndUpdate(1);
+  }
+  return result;
+}
+
+absl::optional<GenericFrameInfo> ScalabilityStructureL2T1Key::OnEncodeDone(
+    LayerFrameConfig config) {
+  absl::optional<GenericFrameInfo> frame_info;
+  if (config.IsKeyframe()) {
+    config = KeyFrameConfig();
+  }
+
+  if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) {
+    RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id();
+    return frame_info;
+  }
+  frame_info.emplace();
+  frame_info->spatial_id = config.SpatialId();
+  frame_info->temporal_id = config.TemporalId();
+  frame_info->encoder_buffers = std::move(config.Buffers());
+  frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]),
+                                               std::end(kDtis[config.Id()]));
+  if (config.IsKeyframe()) {
+    frame_info->part_of_chain = {true, true};
+  } else {
+    frame_info->part_of_chain = {config.SpatialId() == 0,
+                                 config.SpatialId() == 1};
+  }
+  return frame_info;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l2t1_key.h

@@ -0,0 +1,43 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_KEY_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_KEY_H_
+
+#include <vector>
+
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+// S1  0--0--0-
+//     |       ...
+// S0  0--0--0-
+class ScalabilityStructureL2T1Key : public ScalableVideoController {
+ public:
+  ~ScalabilityStructureL2T1Key() override;
+
+  StreamLayersConfig StreamConfig() const override;
+  FrameDependencyStructure DependencyStructure() const override;
+
+  std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override;
+  absl::optional<GenericFrameInfo> OnEncodeDone(
+      LayerFrameConfig config) override;
+
+ private:
+  LayerFrameConfig KeyFrameConfig() const;
+
+  bool keyframe_ = true;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1_KEY_H_

modules/video_coding/codecs/av1/scalability_structure_l2t1h.cc

@@ -0,0 +1,35 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t1h.h"
+
+#include <utility>
+#include <vector>
+
+#include "absl/base/macros.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+
+ScalabilityStructureL2T1h::~ScalabilityStructureL2T1h() = default;
+
+ScalableVideoController::StreamLayersConfig
+ScalabilityStructureL2T1h::StreamConfig() const {
+  StreamLayersConfig result;
+  result.num_spatial_layers = 2;
+  result.num_temporal_layers = 1;
+  // 1.5:1 scaling, see https://w3c.github.io/webrtc-svc/#scalabilitymodes*
+  result.scaling_factor_num[0] = 2;
+  result.scaling_factor_den[0] = 3;
+  return result;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l2t1h.h

@@ -0,0 +1,27 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1H_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1H_H_
+
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t1.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+class ScalabilityStructureL2T1h : public ScalabilityStructureL2T1 {
+ public:
+  ~ScalabilityStructureL2T1h() override;
+
+  StreamLayersConfig StreamConfig() const override;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T1H_H_

modules/video_coding/codecs/av1/scalability_structure_l2t2.cc

@@ -0,0 +1,128 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t2.h"
+
+#include <utility>
+#include <vector>
+
+#include "absl/base/macros.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+namespace {
+
+constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent;
+constexpr auto kDiscardable = DecodeTargetIndication::kDiscardable;
+constexpr auto kSwitch = DecodeTargetIndication::kSwitch;
+constexpr auto kRequired = DecodeTargetIndication::kRequired;
+
+// decode targets: S0T0, S0T1, S1T0, S1T1
+constexpr DecodeTargetIndication kDtis[6][4] = {
+    {kSwitch, kSwitch, kSwitch, kSwitch},                   // kKey, S0
+    {kNotPresent, kNotPresent, kSwitch, kSwitch},           // kKey, S1
+    {kNotPresent, kDiscardable, kNotPresent, kRequired},    // kDeltaT1, S0
+    {kNotPresent, kNotPresent, kNotPresent, kDiscardable},  // kDeltaT1, S1
+    {kSwitch, kSwitch, kRequired, kRequired},               // kDeltaT0, S0
+    {kNotPresent, kNotPresent, kSwitch, kSwitch},           // kDeltaT0, S1
+};
+
+}  // namespace
+
+ScalabilityStructureL2T2::~ScalabilityStructureL2T2() = default;
+
+ScalableVideoController::StreamLayersConfig
+ScalabilityStructureL2T2::StreamConfig() const {
+  StreamLayersConfig result;
+  result.num_spatial_layers = 2;
+  result.num_temporal_layers = 2;
+  result.scaling_factor_num[0] = 1;
+  result.scaling_factor_den[0] = 2;
+  return result;
+}
+
+FrameDependencyStructure ScalabilityStructureL2T2::DependencyStructure() const {
+  FrameDependencyStructure structure;
+  structure.num_decode_targets = 4;
+  structure.num_chains = 2;
+  structure.decode_target_protected_by_chain = {0, 0, 1, 1};
+  structure.templates.resize(6);
+  auto& templates = structure.templates;
+  templates[0].S(0).T(0).Dtis("SSSS").ChainDiffs({0, 0});
+  templates[1].S(0).T(0).Dtis("SSRR").ChainDiffs({4, 3}).FrameDiffs({4});
+  templates[2].S(0).T(1).Dtis("-D-R").ChainDiffs({2, 1}).FrameDiffs({2});
+  templates[3].S(1).T(0).Dtis("--SS").ChainDiffs({1, 1}).FrameDiffs({1});
+  templates[4].S(1).T(0).Dtis("--SS").ChainDiffs({1, 1}).FrameDiffs({4, 1});
+  templates[5].S(1).T(1).Dtis("---D").ChainDiffs({3, 2}).FrameDiffs({2, 1});
+  return structure;
+}
+
+ScalableVideoController::LayerFrameConfig
+ScalabilityStructureL2T2::KeyFrameConfig() const {
+  return LayerFrameConfig().Id(0).Keyframe().S(0).T(0).Update(0);
+}
+
+std::vector<ScalableVideoController::LayerFrameConfig>
+ScalabilityStructureL2T2::NextFrameConfig(bool restart) {
+  if (restart) {
+    next_pattern_ = kKey;
+  }
+  std::vector<LayerFrameConfig> result(2);
+
+  // Buffer0 keeps latest S0T0 frame,
+  // Buffer1 keeps latest S1T0 frame.
+  // Buffer2 keeps latest S0T1 frame.
+  switch (next_pattern_) {
+    case kKey:
+      result[0] = KeyFrameConfig();
+      result[1].Id(1).S(1).T(0).Reference(0).Update(1);
+      next_pattern_ = kDeltaT1;
+      break;
+    case kDeltaT1:
+      result[0].Id(2).S(0).T(1).Reference(0).Update(2);
+      result[1].Id(3).S(1).T(1).Reference(2).Reference(1);
+      next_pattern_ = kDeltaT0;
+      break;
+    case kDeltaT0:
+      result[0].Id(4).S(0).T(0).ReferenceAndUpdate(0);
+      result[1].Id(5).S(1).T(0).Reference(0).ReferenceAndUpdate(1);
+      next_pattern_ = kDeltaT1;
+      break;
+  }
+  return result;
+}
+
+absl::optional<GenericFrameInfo> ScalabilityStructureL2T2::OnEncodeDone(
+    LayerFrameConfig config) {
+  if (config.IsKeyframe()) {
+    config = KeyFrameConfig();
+  }
+
+  absl::optional<GenericFrameInfo> frame_info;
+  if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) {
+    RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id();
+    return frame_info;
+  }
+  frame_info.emplace();
+  frame_info->spatial_id = config.SpatialId();
+  frame_info->temporal_id = config.TemporalId();
+  frame_info->encoder_buffers = config.Buffers();
+  frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]),
+                                               std::end(kDtis[config.Id()]));
+  if (config.TemporalId() == 0) {
+    frame_info->part_of_chain = {config.SpatialId() == 0, true};
+  } else {
+    frame_info->part_of_chain = {false, false};
+  }
+  return frame_info;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l2t2.h

@@ -0,0 +1,53 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_H_
+
+#include <vector>
+
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+// S1T1     0   0
+//         /|  /|  /
+// S1T0   0-+-0-+-0
+//        | | | | | ...
+// S0T1   | 0 | 0 |
+//        |/  |/  |/
+// S0T0   0---0---0--
+// Time-> 0 1 2 3 4
+class ScalabilityStructureL2T2 : public ScalableVideoController {
+ public:
+  ~ScalabilityStructureL2T2() override;
+
+  StreamLayersConfig StreamConfig() const override;
+  FrameDependencyStructure DependencyStructure() const override;
+
+  std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override;
+  absl::optional<GenericFrameInfo> OnEncodeDone(
+      LayerFrameConfig config) override;
+
+ private:
+  enum FramePattern {
+    kKey,
+    kDeltaT1,
+    kDeltaT0,
+  };
+  LayerFrameConfig KeyFrameConfig() const;
+
+  FramePattern next_pattern_ = kKey;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_H_

modules/video_coding/codecs/av1/scalability_structure_l2t2_key.cc

@@ -0,0 +1,130 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t2_key.h"
+
+#include <utility>
+#include <vector>
+
+#include "absl/base/macros.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+namespace {
+
+constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent;
+constexpr auto kDiscardable = DecodeTargetIndication::kDiscardable;
+constexpr auto kSwitch = DecodeTargetIndication::kSwitch;
+
+// decode targets: S0T0, S0T1, S1T0, S1T1
+constexpr DecodeTargetIndication kDtis[6][4] = {
+    {kSwitch, kSwitch, kSwitch, kSwitch},                   //    kKey, S0
+    {kNotPresent, kNotPresent, kSwitch, kSwitch},           //    kKey, S1
+    {kNotPresent, kDiscardable, kNotPresent, kNotPresent},  //    kDeltaT1, S0
+    {kNotPresent, kNotPresent, kNotPresent, kDiscardable},  //    kDeltaT1, S1
+    {kSwitch, kSwitch, kNotPresent, kNotPresent},           //    kDeltaT0, S0
+    {kNotPresent, kNotPresent, kSwitch, kSwitch},           //    kDeltaT0, S1
+};
+
+}  // namespace
+
+ScalabilityStructureL2T2Key::~ScalabilityStructureL2T2Key() = default;
+
+ScalableVideoController::StreamLayersConfig
+ScalabilityStructureL2T2Key::StreamConfig() const {
+  StreamLayersConfig result;
+  result.num_spatial_layers = 2;
+  result.num_temporal_layers = 2;
+  result.scaling_factor_num[0] = 1;
+  result.scaling_factor_den[0] = 2;
+  return result;
+}
+
+FrameDependencyStructure ScalabilityStructureL2T2Key::DependencyStructure()
+    const {
+  FrameDependencyStructure structure;
+  structure.num_decode_targets = 4;
+  structure.num_chains = 2;
+  structure.decode_target_protected_by_chain = {0, 0, 1, 1};
+  structure.templates.resize(6);
+  auto& templates = structure.templates;
+  templates[0].S(0).T(0).Dtis("SSSS").ChainDiffs({0, 0});
+  templates[1].S(0).T(0).Dtis("SS--").ChainDiffs({4, 3}).FrameDiffs({4});
+  templates[2].S(0).T(1).Dtis("-D--").ChainDiffs({2, 1}).FrameDiffs({2});
+  templates[3].S(1).T(0).Dtis("--SS").ChainDiffs({1, 1}).FrameDiffs({1});
+  templates[4].S(1).T(0).Dtis("--SS").ChainDiffs({1, 4}).FrameDiffs({4});
+  templates[5].S(1).T(1).Dtis("---D").ChainDiffs({3, 2}).FrameDiffs({2});
+  return structure;
+}
+
+ScalableVideoController::LayerFrameConfig
+ScalabilityStructureL2T2Key::KeyFrameConfig() const {
+  return LayerFrameConfig().Id(0).Keyframe().S(0).T(0).Update(0);
+}
+
+std::vector<ScalableVideoController::LayerFrameConfig>
+ScalabilityStructureL2T2Key::NextFrameConfig(bool restart) {
+  if (restart) {
+    next_pattern_ = kKey;
+  }
+  std::vector<LayerFrameConfig> result(2);
+
+  // Buffer0 keeps latest S0T0 frame,
+  // Buffer1 keeps latest S1T0 frame.
+  switch (next_pattern_) {
+    case kKey:
+      result[0] = KeyFrameConfig();
+      result[1].Id(1).S(1).T(0).Reference(0).Update(1);
+      next_pattern_ = kDeltaT1;
+      break;
+    case kDeltaT1:
+      result[0].Id(2).S(0).T(1).Reference(0);
+      result[1].Id(3).S(1).T(1).Reference(1);
+      next_pattern_ = kDeltaT0;
+      break;
+    case kDeltaT0:
+      result[0].Id(4).S(0).T(0).ReferenceAndUpdate(0);
+      result[1].Id(5).S(1).T(0).ReferenceAndUpdate(1);
+      next_pattern_ = kDeltaT1;
+      break;
+  }
+  return result;
+}
+
+absl::optional<GenericFrameInfo> ScalabilityStructureL2T2Key::OnEncodeDone(
+    LayerFrameConfig config) {
+  if (config.IsKeyframe()) {
+    config = KeyFrameConfig();
+  }
+
+  absl::optional<GenericFrameInfo> frame_info;
+  if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) {
+    RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id();
+    return frame_info;
+  }
+  frame_info.emplace();
+  frame_info->spatial_id = config.SpatialId();
+  frame_info->temporal_id = config.TemporalId();
+  frame_info->encoder_buffers = config.Buffers();
+  frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]),
+                                               std::end(kDtis[config.Id()]));
+  if (config.IsKeyframe()) {
+    frame_info->part_of_chain = {true, true};
+  } else if (config.TemporalId() == 0) {
+    frame_info->part_of_chain = {config.SpatialId() == 0,
+                                 config.SpatialId() == 1};
+  } else {
+    frame_info->part_of_chain = {false, false};
+  }
+  return frame_info;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l2t2_key.h

@@ -0,0 +1,53 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_KEY_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_KEY_H_
+
+#include <vector>
+
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+// S1T1     0   0
+//         /   /   /
+// S1T0   0---0---0
+//        |         ...
+// S0T1   | 0   0
+//        |/   /   /
+// S0T0   0---0---0
+// Time-> 0 1 2 3 4
+class ScalabilityStructureL2T2Key : public ScalableVideoController {
+ public:
+  ~ScalabilityStructureL2T2Key() override;
+
+  StreamLayersConfig StreamConfig() const override;
+  FrameDependencyStructure DependencyStructure() const override;
+
+  std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override;
+  absl::optional<GenericFrameInfo> OnEncodeDone(
+      LayerFrameConfig config) override;
+
+ private:
+  enum FramePattern {
+    kKey,
+    kDeltaT1,
+    kDeltaT0,
+  };
+  LayerFrameConfig KeyFrameConfig() const;
+
+  FramePattern next_pattern_ = kKey;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_KEY_H_

modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.cc

@@ -0,0 +1,130 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.h"
+
+#include <utility>
+#include <vector>
+
+#include "absl/base/macros.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+namespace {
+
+constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent;
+constexpr auto kDiscardable = DecodeTargetIndication::kDiscardable;
+constexpr auto kSwitch = DecodeTargetIndication::kSwitch;
+
+constexpr DecodeTargetIndication kDtis[6][4] = {
+    {kSwitch, kSwitch, kSwitch, kSwitch},                   //    kKey, S0T0
+    {kNotPresent, kNotPresent, kSwitch, kSwitch},           //    kKey, S1T0
+    {kSwitch, kSwitch, kNotPresent, kNotPresent},           //    kDelta0, S0T0
+    {kNotPresent, kNotPresent, kNotPresent, kDiscardable},  //    kDelta0, S1T1
+    {kNotPresent, kDiscardable, kNotPresent, kNotPresent},  //    kDelta1, S0T1
+    {kNotPresent, kNotPresent, kSwitch, kSwitch},           //    kDelta1, S1T0
+};
+
+}  // namespace
+
+ScalabilityStructureL2T2KeyShift::~ScalabilityStructureL2T2KeyShift() = default;
+
+ScalableVideoController::StreamLayersConfig
+ScalabilityStructureL2T2KeyShift::StreamConfig() const {
+  StreamLayersConfig result;
+  result.num_spatial_layers = 2;
+  result.num_temporal_layers = 2;
+  result.scaling_factor_num[0] = 1;
+  result.scaling_factor_den[0] = 2;
+  return result;
+}
+
+FrameDependencyStructure ScalabilityStructureL2T2KeyShift::DependencyStructure()
+    const {
+  FrameDependencyStructure structure;
+  structure.num_decode_targets = 4;
+  structure.num_chains = 2;
+  structure.decode_target_protected_by_chain = {0, 0, 1, 1};
+  structure.templates.resize(7);
+  auto& templates = structure.templates;
+  templates[0].S(0).T(0).Dtis("SSSS").ChainDiffs({0, 0});
+  templates[1].S(0).T(0).Dtis("SS--").ChainDiffs({2, 1}).FrameDiffs({2});
+  templates[2].S(0).T(0).Dtis("SS--").ChainDiffs({4, 1}).FrameDiffs({4});
+  templates[3].S(0).T(1).Dtis("-D--").ChainDiffs({2, 3}).FrameDiffs({2});
+  templates[4].S(1).T(0).Dtis("--SS").ChainDiffs({1, 1}).FrameDiffs({1});
+  templates[5].S(1).T(0).Dtis("--SS").ChainDiffs({3, 4}).FrameDiffs({4});
+  templates[6].S(1).T(1).Dtis("---D").ChainDiffs({1, 2}).FrameDiffs({2});
+  return structure;
+}
+
+ScalableVideoController::LayerFrameConfig
+ScalabilityStructureL2T2KeyShift::KeyFrameConfig() const {
+  return LayerFrameConfig().Id(0).Keyframe().S(0).T(0).Update(0);
+}
+
+std::vector<ScalableVideoController::LayerFrameConfig>
+ScalabilityStructureL2T2KeyShift::NextFrameConfig(bool restart) {
+  if (restart) {
+    next_pattern_ = kKey;
+  }
+  std::vector<LayerFrameConfig> result(2);
+
+  // Buffer0 keeps latest S0T0 frame,
+  // Buffer1 keeps latest S1T0 frame.
+  switch (next_pattern_) {
+    case kKey:
+      result[0] = KeyFrameConfig();
+      result[1].Id(1).S(1).T(0).Reference(0).Update(1);
+      next_pattern_ = kDelta0;
+      break;
+    case kDelta0:
+      result[0].Id(2).S(0).T(0).ReferenceAndUpdate(0);
+      result[1].Id(3).S(1).T(1).Reference(1);
+      next_pattern_ = kDelta1;
+      break;
+    case kDelta1:
+      result[0].Id(4).S(0).T(1).Reference(0);
+      result[1].Id(5).S(1).T(0).ReferenceAndUpdate(1);
+      next_pattern_ = kDelta0;
+      break;
+  }
+  return result;
+}
+
+absl::optional<GenericFrameInfo> ScalabilityStructureL2T2KeyShift::OnEncodeDone(
+    LayerFrameConfig config) {
+  if (config.IsKeyframe()) {
+    config = KeyFrameConfig();
+  }
+
+  absl::optional<GenericFrameInfo> frame_info;
+  if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) {
+    RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id();
+    return frame_info;
+  }
+  frame_info.emplace();
+  frame_info->spatial_id = config.SpatialId();
+  frame_info->temporal_id = config.TemporalId();
+  frame_info->encoder_buffers = config.Buffers();
+  frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]),
+                                               std::end(kDtis[config.Id()]));
+  if (config.IsKeyframe()) {
+    frame_info->part_of_chain = {true, true};
+  } else if (config.TemporalId() == 0) {
+    frame_info->part_of_chain = {config.SpatialId() == 0,
+                                 config.SpatialId() == 1};
+  } else {
+    frame_info->part_of_chain = {false, false};
+  }
+  return frame_info;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l2t2_key_shift.h

@@ -0,0 +1,53 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_KEY_SHIFT_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_KEY_SHIFT_H_
+
+#include <vector>
+
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+// S1T1     0   0
+//         /   /   /
+// S1T0   0---0---0
+//        |        ...
+// S0T1   |   0   0
+//        |  /   /
+// S0T0   0-0---0--
+// Time-> 0 1 2 3 4
+class ScalabilityStructureL2T2KeyShift : public ScalableVideoController {
+ public:
+  ~ScalabilityStructureL2T2KeyShift() override;
+
+  StreamLayersConfig StreamConfig() const override;
+  FrameDependencyStructure DependencyStructure() const override;
+
+  std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override;
+  absl::optional<GenericFrameInfo> OnEncodeDone(
+      LayerFrameConfig config) override;
+
+ private:
+  enum FramePattern {
+    kKey,
+    kDelta0,
+    kDelta1,
+  };
+  LayerFrameConfig KeyFrameConfig() const;
+
+  FramePattern next_pattern_ = kKey;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L2T2_KEY_SHIFT_H_

modules/video_coding/codecs/av1/scalability_structure_l3t1.cc

@@ -0,0 +1,112 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/scalability_structure_l3t1.h"
+
+#include <utility>
+#include <vector>
+
+#include "absl/base/macros.h"
+#include "absl/types/optional.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+namespace {
+
+constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent;
+constexpr auto kSwitch = DecodeTargetIndication::kSwitch;
+constexpr auto kRequired = DecodeTargetIndication::kRequired;
+
+constexpr DecodeTargetIndication kDtis[5][3] = {
+    {kSwitch, kSwitch, kSwitch},          // Key, S0
+    {kNotPresent, kSwitch, kSwitch},      // Key, S1
+    {kNotPresent, kNotPresent, kSwitch},  // Key and Delta, S2
+    {kSwitch, kRequired, kRequired},      // Delta, S0
+    {kNotPresent, kSwitch, kRequired},    // Delta, S1
+};
+
+}  // namespace
+
+ScalabilityStructureL3T1::~ScalabilityStructureL3T1() = default;
+
+ScalableVideoController::StreamLayersConfig
+ScalabilityStructureL3T1::StreamConfig() const {
+  StreamLayersConfig result;
+  result.num_spatial_layers = 3;
+  result.num_temporal_layers = 1;
+  result.scaling_factor_num[0] = 1;
+  result.scaling_factor_den[0] = 4;
+  result.scaling_factor_num[1] = 1;
+  result.scaling_factor_den[1] = 2;
+  return result;
+}
+
+FrameDependencyStructure ScalabilityStructureL3T1::DependencyStructure() const {
+  FrameDependencyStructure structure;
+  structure.num_decode_targets = 3;
+  structure.num_chains = 3;
+  structure.decode_target_protected_by_chain = {0, 1, 2};
+  auto& templates = structure.templates;
+  templates.resize(6);
+  templates[0].S(0).Dtis("SRR").ChainDiffs({3, 2, 1}).FrameDiffs({3});
+  templates[1].S(0).Dtis("SSS").ChainDiffs({0, 0, 0});
+  templates[2].S(1).Dtis("-SR").ChainDiffs({1, 1, 1}).FrameDiffs({3, 1});
+  templates[3].S(1).Dtis("-SS").ChainDiffs({1, 1, 1}).FrameDiffs({1});
+  templates[4].S(2).Dtis("--S").ChainDiffs({2, 1, 1}).FrameDiffs({3, 1});
+  templates[5].S(2).Dtis("--S").ChainDiffs({2, 1, 1}).FrameDiffs({1});
+  return structure;
+}
+
+std::vector<ScalableVideoController::LayerFrameConfig>
+ScalabilityStructureL3T1::NextFrameConfig(bool restart) {
+  std::vector<LayerFrameConfig> config(3);
+
+  // Buffer i keeps latest frame for spatial layer i
+  if (restart || keyframe_) {
+    config[0].Id(0).S(0).Keyframe().Update(0);
+    config[1].Id(1).S(1).Update(1).Reference(0);
+    config[2].Id(2).S(2).Update(2).Reference(1);
+    keyframe_ = false;
+  } else {
+    config[0].Id(3).S(0).ReferenceAndUpdate(0);
+    config[1].Id(4).S(1).ReferenceAndUpdate(1).Reference(0);
+    config[2].Id(2).S(2).ReferenceAndUpdate(2).Reference(1);
+  }
+  return config;
+}
+
+absl::optional<GenericFrameInfo> ScalabilityStructureL3T1::OnEncodeDone(
+    LayerFrameConfig config) {
+  absl::optional<GenericFrameInfo> frame_info;
+  if (config.IsKeyframe() && config.Id() != 0) {
+    // Encoder generated a key frame without asking to.
+    if (config.SpatialId() > 0) {
+      RTC_LOG(LS_WARNING) << "Unexpected spatial id " << config.SpatialId()
+                          << " for key frame.";
+    }
+    config = LayerFrameConfig().Id(0).S(0).Keyframe().Update(0);
+  }
+
+  if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) {
+    RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id();
+    return frame_info;
+  }
+  frame_info.emplace();
+  frame_info->spatial_id = config.SpatialId();
+  frame_info->temporal_id = config.TemporalId();
+  frame_info->encoder_buffers = config.Buffers();
+  frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]),
+                                               std::end(kDtis[config.Id()]));
+  frame_info->part_of_chain = {config.SpatialId() == 0, config.SpatialId() <= 1,
+                               true};
+  return frame_info;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l3t1.h

@@ -0,0 +1,45 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T1_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T1_H_
+
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+// S2     0-0-0-
+//        | | |
+// S1     0-0-0-...
+//        | | |
+// S0     0-0-0-
+// Time-> 0 1 2
+class ScalabilityStructureL3T1 : public ScalableVideoController {
+ public:
+  ~ScalabilityStructureL3T1() override;
+
+  StreamLayersConfig StreamConfig() const override;
+  FrameDependencyStructure DependencyStructure() const override;
+
+  std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override;
+  absl::optional<GenericFrameInfo> OnEncodeDone(
+      LayerFrameConfig config) override;
+
+ private:
+  bool keyframe_ = true;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T1_H_

modules/video_coding/codecs/av1/scalability_structure_l3t3.cc

@@ -0,0 +1,224 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/scalability_structure_l3t3.h"
+
+#include <utility>
+#include <vector>
+
+#include "absl/base/macros.h"
+#include "absl/types/optional.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+namespace {
+
+constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent;
+constexpr auto kDiscardable = DecodeTargetIndication::kDiscardable;
+constexpr auto kSwitch = DecodeTargetIndication::kSwitch;
+constexpr auto kRequired = DecodeTargetIndication::kRequired;
+
+constexpr DecodeTargetIndication kDtis[12][9] = {
+    // Key, S0
+    {kSwitch, kSwitch, kSwitch,   // S0
+     kSwitch, kSwitch, kSwitch,   // S1
+     kSwitch, kSwitch, kSwitch},  // S2
+    // Key, S1
+    {kNotPresent, kNotPresent, kNotPresent,  // S0
+     kSwitch, kSwitch, kSwitch,              // S1
+     kSwitch, kSwitch, kSwitch},             // S2
+    // Key, S2
+    {kNotPresent, kNotPresent, kNotPresent,  // S0
+     kNotPresent, kNotPresent, kNotPresent,  // S1
+     kSwitch, kSwitch, kSwitch},             // S2
+    // Delta, S0T2
+    {kNotPresent, kNotPresent, kDiscardable,  // S0
+     kNotPresent, kNotPresent, kRequired,     // S1
+     kNotPresent, kNotPresent, kRequired},    // S2
+    // Delta, S1T2
+    {kNotPresent, kNotPresent, kNotPresent,   // S0
+     kNotPresent, kNotPresent, kDiscardable,  // S1
+     kNotPresent, kNotPresent, kRequired},    // S2
+    // Delta, S2T2
+    {kNotPresent, kNotPresent, kNotPresent,    // S0
+     kNotPresent, kNotPresent, kNotPresent,    // S1
+     kNotPresent, kNotPresent, kDiscardable},  // S2
+    // Delta, S0T1
+    {kNotPresent, kDiscardable, kSwitch,  // S0
+     kNotPresent, kRequired, kRequired,   // S1
+     kNotPresent, kRequired, kRequired},  // S2
+    // Delta, S1T1
+    {kNotPresent, kNotPresent, kNotPresent,  // S0
+     kNotPresent, kDiscardable, kSwitch,     // S1
+     kNotPresent, kRequired, kRequired},     // S2
+    // Delta, S2T1
+    {kNotPresent, kNotPresent, kNotPresent,  // S0
+     kNotPresent, kNotPresent, kNotPresent,  // S1
+     kNotPresent, kDiscardable, kSwitch},    // S2
+    // Delta, S0T0
+    {kSwitch, kSwitch, kSwitch,         // S0
+     kRequired, kRequired, kRequired,   // S1
+     kRequired, kRequired, kRequired},  // S2
+    // Delta, S1T0
+    {kNotPresent, kNotPresent, kNotPresent,  // S0
+     kSwitch, kSwitch, kSwitch,              // S1
+     kRequired, kRequired, kRequired},       // S2
+    // Delta, S2T0
+    {kNotPresent, kNotPresent, kNotPresent,  // S0
+     kNotPresent, kNotPresent, kNotPresent,  // S1
+     kSwitch, kSwitch, kSwitch},             // S2
+};
+
+}  // namespace
+
+ScalabilityStructureL3T3::~ScalabilityStructureL3T3() = default;
+
+ScalableVideoController::StreamLayersConfig
+ScalabilityStructureL3T3::StreamConfig() const {
+  StreamLayersConfig result;
+  result.num_spatial_layers = 3;
+  result.num_temporal_layers = 3;
+  result.scaling_factor_num[0] = 1;
+  result.scaling_factor_den[0] = 4;
+  result.scaling_factor_num[1] = 1;
+  result.scaling_factor_den[1] = 2;
+  return result;
+}
+
+FrameDependencyStructure ScalabilityStructureL3T3::DependencyStructure() const {
+  FrameDependencyStructure structure;
+  structure.num_decode_targets = 9;
+  structure.num_chains = 3;
+  structure.decode_target_protected_by_chain = {0, 0, 0, 1, 1, 1, 2, 2, 2};
+  auto& t = structure.templates;
+  t.resize(15);
+  // Templates are shown in the order frames following them appear in the
+  // stream, but in `structure.templates` array templates are sorted by
+  // (`spatial_id`, `temporal_id`) since that is a dependency descriptor
+  // requirement. Indexes are written in hex for nicer alignment.
+  t[0x1].S(0).T(0).Dtis("SSSSSSSSS").ChainDiffs({0, 0, 0});
+  t[0x6].S(1).T(0).Dtis("---SSSSSS").ChainDiffs({1, 1, 1}).FrameDiffs({1});
+  t[0xB].S(2).T(0).Dtis("------SSS").ChainDiffs({2, 1, 1}).FrameDiffs({1});
+  t[0x3].S(0).T(2).Dtis("--D--R--R").ChainDiffs({3, 2, 1}).FrameDiffs({3});
+  t[0x8].S(1).T(2).Dtis("-----D--R").ChainDiffs({4, 3, 2}).FrameDiffs({3, 1});
+  t[0xD].S(2).T(2).Dtis("--------D").ChainDiffs({5, 4, 3}).FrameDiffs({3, 1});
+  t[0x2].S(0).T(1).Dtis("-DS-RR-RR").ChainDiffs({6, 5, 4}).FrameDiffs({6});
+  t[0x7].S(1).T(1).Dtis("----DS-RR").ChainDiffs({7, 6, 5}).FrameDiffs({6, 1});
+  t[0xC].S(2).T(1).Dtis("-------DS").ChainDiffs({8, 7, 6}).FrameDiffs({6, 1});
+  t[0x4].S(0).T(2).Dtis("--D--R--R").ChainDiffs({9, 8, 7}).FrameDiffs({3});
+  t[0x9].S(1).T(2).Dtis("-----D--R").ChainDiffs({10, 9, 8}).FrameDiffs({3, 1});
+  t[0xE].S(2).T(2).Dtis("--------D").ChainDiffs({11, 10, 9}).FrameDiffs({3, 1});
+  t[0x0].S(0).T(0).Dtis("SSSRRRRRR").ChainDiffs({12, 11, 10}).FrameDiffs({12});
+  t[0x5].S(1).T(0).Dtis("---SSSRRR").ChainDiffs({1, 1, 1}).FrameDiffs({12, 1});
+  t[0xA].S(2).T(0).Dtis("------SSS").ChainDiffs({2, 1, 1}).FrameDiffs({12, 1});
+  return structure;
+}
+
+ScalableVideoController::LayerFrameConfig
+ScalabilityStructureL3T3::KeyFrameConfig() const {
+  return LayerFrameConfig().Id(0).S(0).T(0).Keyframe().Update(0);
+}
+
+std::vector<ScalableVideoController::LayerFrameConfig>
+ScalabilityStructureL3T3::NextFrameConfig(bool restart) {
+  if (restart) {
+    next_pattern_ = kKeyFrame;
+  }
+  std::vector<LayerFrameConfig> config(3);
+
+  // For this structure name each of 8 buffers after the layer of the frame that
+  // buffer keeps.
+  static constexpr int kS0T0 = 0;
+  static constexpr int kS1T0 = 1;
+  static constexpr int kS2T0 = 2;
+  static constexpr int kS0T1 = 3;
+  static constexpr int kS1T1 = 4;
+  static constexpr int kS2T1 = 5;
+  static constexpr int kS0T2 = 6;
+  static constexpr int kS1T2 = 7;
+  switch (next_pattern_) {
+    case kKeyFrame:
+      config[0].Id(0).S(0).T(0).Keyframe().Update(kS0T0);
+      config[1].Id(1).S(1).T(0).Update(kS1T0).Reference(kS0T0);
+      config[2].Id(2).S(2).T(0).Update(kS2T0).Reference(kS1T0);
+      next_pattern_ = kDeltaFrameT2A;
+      break;
+    case kDeltaFrameT2A:
+      config[0].Id(3).S(0).T(2).Reference(kS0T0).Update(kS0T2);
+      config[1].Id(4).S(1).T(2).Reference(kS1T0).Reference(kS0T2).Update(kS1T2);
+      config[2].Id(5).S(2).T(2).Reference(kS2T0).Reference(kS1T2);
+      next_pattern_ = kDeltaFrameT1;
+      break;
+    case kDeltaFrameT1:
+      config[0].Id(6).S(0).T(1).Reference(kS0T0).Update(kS0T1);
+      config[1].Id(7).S(1).T(1).Reference(kS1T0).Reference(kS0T1).Update(kS1T1);
+      config[2].Id(8).S(2).T(1).Reference(kS2T0).Reference(kS1T1).Update(kS2T1);
+      next_pattern_ = kDeltaFrameT2B;
+      break;
+    case kDeltaFrameT2B:
+      config[0].Id(3).S(0).T(2).Reference(kS0T1).Update(kS0T2);
+      config[1].Id(4).S(1).T(2).Reference(kS1T1).Reference(kS0T2).Update(kS1T2);
+      config[2].Id(5).S(2).T(2).Reference(kS2T1).Reference(kS1T2);
+      next_pattern_ = kDeltaFrameT0;
+      break;
+    case kDeltaFrameT0:
+      config[0].Id(9).S(0).T(0).ReferenceAndUpdate(kS0T0);
+      config[1].Id(10).S(1).T(0).ReferenceAndUpdate(kS1T0).Reference(kS0T0);
+      config[2].Id(11).S(2).T(0).ReferenceAndUpdate(kS2T0).Reference(kS1T0);
+      next_pattern_ = kDeltaFrameT2A;
+      break;
+  }
+  return config;
+}
+
+absl::optional<GenericFrameInfo> ScalabilityStructureL3T3::OnEncodeDone(
+    LayerFrameConfig config) {
+  if (config.IsKeyframe() && config.Id() != 0) {
+    // Encoder generated a key frame without asking to.
+    if (config.SpatialId() > 0) {
+      RTC_LOG(LS_WARNING) << "Unexpected spatial id " << config.SpatialId()
+                          << " for key frame.";
+    }
+    config = LayerFrameConfig()
+                 .Keyframe()
+                 .Id(0)
+                 .S(0)
+                 .T(0)
+                 .Update(0)
+                 .Update(1)
+                 .Update(2)
+                 .Update(3)
+                 .Update(4)
+                 .Update(5)
+                 .Update(6)
+                 .Update(7);
+  }
+
+  absl::optional<GenericFrameInfo> frame_info;
+  if (config.Id() < 0 || config.Id() >= int{ABSL_ARRAYSIZE(kDtis)}) {
+    RTC_LOG(LS_ERROR) << "Unexpected config id " << config.Id();
+    return frame_info;
+  }
+  frame_info.emplace();
+  frame_info->spatial_id = config.SpatialId();
+  frame_info->temporal_id = config.TemporalId();
+  frame_info->encoder_buffers = config.Buffers();
+  frame_info->decode_target_indications.assign(std::begin(kDtis[config.Id()]),
+                                               std::end(kDtis[config.Id()]));
+  if (config.TemporalId() == 0) {
+    frame_info->part_of_chain = {config.SpatialId() == 0,
+                                 config.SpatialId() <= 1, true};
+  } else {
+    frame_info->part_of_chain = {false, false, false};
+  }
+  return frame_info;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_l3t3.h

@@ -0,0 +1,49 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T3_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T3_H_
+
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+// https://aomediacodec.github.io/av1-rtp-spec/#a63-l3t3-full-svc
+class ScalabilityStructureL3T3 : public ScalableVideoController {
+ public:
+  ~ScalabilityStructureL3T3() override;
+
+  StreamLayersConfig StreamConfig() const override;
+  FrameDependencyStructure DependencyStructure() const override;
+
+  std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override;
+  absl::optional<GenericFrameInfo> OnEncodeDone(
+      LayerFrameConfig config) override;
+
+ private:
+  enum FramePattern {
+    kKeyFrame,
+    kDeltaFrameT2A,
+    kDeltaFrameT1,
+    kDeltaFrameT2B,
+    kDeltaFrameT0,
+  };
+  LayerFrameConfig KeyFrameConfig() const;
+
+  FramePattern next_pattern_ = kKeyFrame;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_L3T3_H_

modules/video_coding/codecs/av1/scalability_structure_s2t1.cc

@@ -0,0 +1,93 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/scalability_structure_s2t1.h"
+
+#include <utility>
+#include <vector>
+
+#include "absl/base/macros.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+namespace {
+
+constexpr auto kNotPresent = DecodeTargetIndication::kNotPresent;
+constexpr auto kSwitch = DecodeTargetIndication::kSwitch;
+
+constexpr DecodeTargetIndication kDtis[2][2] = {
+    {kSwitch, kNotPresent},  // S0
+    {kNotPresent, kSwitch},  // S1
+};
+
+}  // namespace
+
+ScalabilityStructureS2T1::~ScalabilityStructureS2T1() = default;
+
+ScalableVideoController::StreamLayersConfig
+ScalabilityStructureS2T1::StreamConfig() const {
+  StreamLayersConfig result;
+  result.num_spatial_layers = 2;
+  result.num_temporal_layers = 1;
+  result.scaling_factor_num[0] = 1;
+  result.scaling_factor_den[0] = 2;
+  return result;
+}
+
+FrameDependencyStructure ScalabilityStructureS2T1::DependencyStructure() const {
+  FrameDependencyStructure structure;
+  structure.num_decode_targets = 2;
+  structure.num_chains = 2;
+  structure.decode_target_protected_by_chain = {0, 1};
+  structure.templates.resize(4);
+  structure.templates[0].S(0).Dtis("S-").ChainDiffs({2, 1}).FrameDiffs({2});
+  structure.templates[1].S(0).Dtis("S-").ChainDiffs({0, 0});
+  structure.templates[2].S(1).Dtis("-S").ChainDiffs({1, 2}).FrameDiffs({2});
+  structure.templates[3].S(1).Dtis("-S").ChainDiffs({1, 0});
+  return structure;
+}
+
+std::vector<ScalableVideoController::LayerFrameConfig>
+ScalabilityStructureS2T1::NextFrameConfig(bool restart) {
+  std::vector<LayerFrameConfig> result(2);
+  // Buffer0 keeps latest S0T0 frame, Buffer1 keeps latest S1T0 frame.
+  if (restart || keyframe_) {
+    result[0].S(0).Keyframe().Update(0);
+    result[1].S(1).Keyframe().Update(1);
+    keyframe_ = false;
+  } else {
+    result[0].S(0).ReferenceAndUpdate(0);
+    result[1].S(1).ReferenceAndUpdate(1);
+  }
+  return result;
+}
+
+absl::optional<GenericFrameInfo> ScalabilityStructureS2T1::OnEncodeDone(
+    LayerFrameConfig config) {
+  absl::optional<GenericFrameInfo> frame_info;
+  if (config.SpatialId() < 0 ||
+      config.SpatialId() >= int{ABSL_ARRAYSIZE(kDtis)}) {
+    RTC_LOG(LS_ERROR) << "Unexpected spatial id " << config.SpatialId();
+    return frame_info;
+  }
+  frame_info.emplace();
+  frame_info->spatial_id = config.SpatialId();
+  frame_info->temporal_id = config.TemporalId();
+  frame_info->encoder_buffers = std::move(config.Buffers());
+  frame_info->decode_target_indications.assign(
+      std::begin(kDtis[config.SpatialId()]),
+      std::end(kDtis[config.SpatialId()]));
+  frame_info->part_of_chain = {config.SpatialId() == 0,
+                               config.SpatialId() == 1};
+  return frame_info;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalability_structure_s2t1.h

@@ -0,0 +1,41 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_S2T1_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_S2T1_H_
+
+#include <vector>
+
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+// S1  0--0--0-
+//             ...
+// S0  0--0--0-
+class ScalabilityStructureS2T1 : public ScalableVideoController {
+ public:
+  ~ScalabilityStructureS2T1() override;
+
+  StreamLayersConfig StreamConfig() const override;
+  FrameDependencyStructure DependencyStructure() const override;
+
+  std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override;
+  absl::optional<GenericFrameInfo> OnEncodeDone(
+      LayerFrameConfig config) override;
+
+ private:
+  bool keyframe_ = true;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABILITY_STRUCTURE_S2T1_H_

modules/video_coding/codecs/av1/scalability_structure_unittest.cc

@@ -0,0 +1,295 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <memory>
+#include <ostream>
+#include <string>
+
+#include "absl/types/optional.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "api/video/video_frame_type.h"
+#include "modules/video_coding/chain_diff_calculator.h"
+#include "modules/video_coding/codecs/av1/create_scalability_structure.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+#include "modules/video_coding/frame_dependencies_calculator.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace {
+
+using ::testing::AllOf;
+using ::testing::Contains;
+using ::testing::Each;
+using ::testing::Field;
+using ::testing::Ge;
+using ::testing::IsEmpty;
+using ::testing::Le;
+using ::testing::Lt;
+using ::testing::Not;
+using ::testing::SizeIs;
+using ::testing::TestWithParam;
+using ::testing::Values;
+
+struct SvcTestParam {
+  friend std::ostream& operator<<(std::ostream& os, const SvcTestParam& param) {
+    return os << param.name;
+  }
+
+  std::string name;
+  int num_temporal_units;
+};
+
+class ScalabilityStructureTest : public TestWithParam<SvcTestParam> {
+ public:
+  std::vector<GenericFrameInfo> GenerateAllFrames() {
+    std::vector<GenericFrameInfo> frames;
+
+    FrameDependenciesCalculator frame_deps_calculator;
+    ChainDiffCalculator chain_diff_calculator;
+    std::unique_ptr<ScalableVideoController> structure_controller =
+        CreateScalabilityStructure(GetParam().name);
+    FrameDependencyStructure structure =
+        structure_controller->DependencyStructure();
+    for (int i = 0; i < GetParam().num_temporal_units; ++i) {
+      for (auto& layer_frame :
+           structure_controller->NextFrameConfig(/*reset=*/false)) {
+        int64_t frame_id = static_cast<int64_t>(frames.size());
+        bool is_keyframe = layer_frame.IsKeyframe();
+        absl::optional<GenericFrameInfo> frame_info =
+            structure_controller->OnEncodeDone(std::move(layer_frame));
+        EXPECT_TRUE(frame_info.has_value());
+        if (is_keyframe) {
+          chain_diff_calculator.Reset(frame_info->part_of_chain);
+        }
+        frame_info->chain_diffs =
+            chain_diff_calculator.From(frame_id, frame_info->part_of_chain);
+        for (int64_t base_frame_id : frame_deps_calculator.FromBuffersUsage(
+                 is_keyframe ? VideoFrameType::kVideoFrameKey
+                             : VideoFrameType::kVideoFrameDelta,
+                 frame_id, frame_info->encoder_buffers)) {
+          EXPECT_LT(base_frame_id, frame_id);
+          EXPECT_GE(base_frame_id, 0);
+          frame_info->frame_diffs.push_back(frame_id - base_frame_id);
+        }
+
+        frames.push_back(*std::move(frame_info));
+      }
+    }
+    return frames;
+  }
+};
+
+TEST_P(ScalabilityStructureTest,
+       NumberOfDecodeTargetsAndChainsAreInRangeAndConsistent) {
+  FrameDependencyStructure structure =
+      CreateScalabilityStructure(GetParam().name)->DependencyStructure();
+  EXPECT_GT(structure.num_decode_targets, 0);
+  EXPECT_LE(structure.num_decode_targets,
+            DependencyDescriptor::kMaxDecodeTargets);
+  EXPECT_GE(structure.num_chains, 0);
+  EXPECT_LE(structure.num_chains, structure.num_decode_targets);
+  if (structure.num_chains == 0) {
+    EXPECT_THAT(structure.decode_target_protected_by_chain, IsEmpty());
+  } else {
+    EXPECT_THAT(structure.decode_target_protected_by_chain,
+                AllOf(SizeIs(structure.num_decode_targets), Each(Ge(0)),
+                      Each(Lt(structure.num_chains))));
+  }
+  EXPECT_THAT(structure.templates,
+              SizeIs(Lt(size_t{DependencyDescriptor::kMaxTemplates})));
+}
+
+TEST_P(ScalabilityStructureTest, TemplatesAreSortedByLayerId) {
+  FrameDependencyStructure structure =
+      CreateScalabilityStructure(GetParam().name)->DependencyStructure();
+  ASSERT_THAT(structure.templates, Not(IsEmpty()));
+  const auto& first_templates = structure.templates.front();
+  EXPECT_EQ(first_templates.spatial_id, 0);
+  EXPECT_EQ(first_templates.temporal_id, 0);
+  for (size_t i = 1; i < structure.templates.size(); ++i) {
+    const auto& prev_template = structure.templates[i - 1];
+    const auto& next_template = structure.templates[i];
+    if (next_template.spatial_id == prev_template.spatial_id &&
+        next_template.temporal_id == prev_template.temporal_id) {
+      // Same layer, next_layer_idc == 0
+    } else if (next_template.spatial_id == prev_template.spatial_id &&
+               next_template.temporal_id == prev_template.temporal_id + 1) {
+      // Next temporal layer, next_layer_idc == 1
+    } else if (next_template.spatial_id == prev_template.spatial_id + 1 &&
+               next_template.temporal_id == 0) {
+      // Next spatial layer, next_layer_idc == 2
+    } else {
+      // everything else is invalid.
+      ADD_FAILURE() << "Invalid templates order. Template #" << i
+                    << " with layer (" << next_template.spatial_id << ","
+                    << next_template.temporal_id
+                    << ") follows template with layer ("
+                    << prev_template.spatial_id << ","
+                    << prev_template.temporal_id << ").";
+    }
+  }
+}
+
+TEST_P(ScalabilityStructureTest, TemplatesMatchNumberOfDecodeTargetsAndChains) {
+  FrameDependencyStructure structure =
+      CreateScalabilityStructure(GetParam().name)->DependencyStructure();
+  EXPECT_THAT(
+      structure.templates,
+      Each(AllOf(Field(&FrameDependencyTemplate::decode_target_indications,
+                       SizeIs(structure.num_decode_targets)),
+                 Field(&FrameDependencyTemplate::chain_diffs,
+                       SizeIs(structure.num_chains)))));
+}
+
+TEST_P(ScalabilityStructureTest, FrameInfoMatchesFrameDependencyStructure) {
+  FrameDependencyStructure structure =
+      CreateScalabilityStructure(GetParam().name)->DependencyStructure();
+  std::vector<GenericFrameInfo> frame_infos = GenerateAllFrames();
+  for (size_t frame_id = 0; frame_id < frame_infos.size(); ++frame_id) {
+    const auto& frame = frame_infos[frame_id];
+    EXPECT_GE(frame.spatial_id, 0) << " for frame " << frame_id;
+    EXPECT_GE(frame.temporal_id, 0) << " for frame " << frame_id;
+    EXPECT_THAT(frame.decode_target_indications,
+                SizeIs(structure.num_decode_targets))
+        << " for frame " << frame_id;
+    EXPECT_THAT(frame.part_of_chain, SizeIs(structure.num_chains))
+        << " for frame " << frame_id;
+  }
+}
+
+TEST_P(ScalabilityStructureTest, ThereIsAPerfectTemplateForEachFrame) {
+  FrameDependencyStructure structure =
+      CreateScalabilityStructure(GetParam().name)->DependencyStructure();
+  std::vector<GenericFrameInfo> frame_infos = GenerateAllFrames();
+  for (size_t frame_id = 0; frame_id < frame_infos.size(); ++frame_id) {
+    EXPECT_THAT(structure.templates, Contains(frame_infos[frame_id]))
+        << " for frame " << frame_id;
+  }
+}
+
+TEST_P(ScalabilityStructureTest, FrameDependsOnSameOrLowerLayer) {
+  std::vector<GenericFrameInfo> frame_infos = GenerateAllFrames();
+  int64_t num_frames = frame_infos.size();
+
+  for (int64_t frame_id = 0; frame_id < num_frames; ++frame_id) {
+    const auto& frame = frame_infos[frame_id];
+    for (int frame_diff : frame.frame_diffs) {
+      int64_t base_frame_id = frame_id - frame_diff;
+      const auto& base_frame = frame_infos[base_frame_id];
+      EXPECT_GE(frame.spatial_id, base_frame.spatial_id)
+          << "Frame " << frame_id << " depends on frame " << base_frame_id;
+      EXPECT_GE(frame.temporal_id, base_frame.temporal_id)
+          << "Frame " << frame_id << " depends on frame " << base_frame_id;
+    }
+  }
+}
+
+TEST_P(ScalabilityStructureTest, NoFrameDependsOnDiscardableOrNotPresent) {
+  std::vector<GenericFrameInfo> frame_infos = GenerateAllFrames();
+  int64_t num_frames = frame_infos.size();
+  FrameDependencyStructure structure =
+      CreateScalabilityStructure(GetParam().name)->DependencyStructure();
+
+  for (int dt = 0; dt < structure.num_decode_targets; ++dt) {
+    for (int64_t frame_id = 0; frame_id < num_frames; ++frame_id) {
+      const auto& frame = frame_infos[frame_id];
+      if (frame.decode_target_indications[dt] ==
+          DecodeTargetIndication::kNotPresent) {
+        continue;
+      }
+      for (int frame_diff : frame.frame_diffs) {
+        int64_t base_frame_id = frame_id - frame_diff;
+        const auto& base_frame = frame_infos[base_frame_id];
+        EXPECT_NE(base_frame.decode_target_indications[dt],
+                  DecodeTargetIndication::kNotPresent)
+            << "Frame " << frame_id << " depends on frame " << base_frame_id
+            << " that is not part of decode target#" << dt;
+        EXPECT_NE(base_frame.decode_target_indications[dt],
+                  DecodeTargetIndication::kDiscardable)
+            << "Frame " << frame_id << " depends on frame " << base_frame_id
+            << " that is discardable for decode target#" << dt;
+      }
+    }
+  }
+}
+
+TEST_P(ScalabilityStructureTest, NoFrameDependsThroughSwitchIndication) {
+  FrameDependencyStructure structure =
+      CreateScalabilityStructure(GetParam().name)->DependencyStructure();
+  std::vector<GenericFrameInfo> frame_infos = GenerateAllFrames();
+  int64_t num_frames = frame_infos.size();
+  std::vector<std::set<int64_t>> full_deps(num_frames);
+
+  // For each frame calculate set of all frames it depends on, both directly and
+  // indirectly.
+  for (int64_t frame_id = 0; frame_id < num_frames; ++frame_id) {
+    std::set<int64_t> all_base_frames;
+    for (int frame_diff : frame_infos[frame_id].frame_diffs) {
+      int64_t base_frame_id = frame_id - frame_diff;
+      all_base_frames.insert(base_frame_id);
+      const auto& indirect = full_deps[base_frame_id];
+      all_base_frames.insert(indirect.begin(), indirect.end());
+    }
+    full_deps[frame_id] = std::move(all_base_frames);
+  }
+
+  // Now check the switch indication: frames after the switch indication mustn't
+  // depend on any addition frames before the switch indications.
+  for (int dt = 0; dt < structure.num_decode_targets; ++dt) {
+    for (int64_t switch_frame_id = 0; switch_frame_id < num_frames;
+         ++switch_frame_id) {
+      if (frame_infos[switch_frame_id].decode_target_indications[dt] !=
+          DecodeTargetIndication::kSwitch) {
+        continue;
+      }
+      for (int64_t later_frame_id = switch_frame_id + 1;
+           later_frame_id < num_frames; ++later_frame_id) {
+        if (frame_infos[later_frame_id].decode_target_indications[dt] ==
+            DecodeTargetIndication::kNotPresent) {
+          continue;
+        }
+        for (int frame_diff : frame_infos[later_frame_id].frame_diffs) {
+          int64_t early_frame_id = later_frame_id - frame_diff;
+          if (early_frame_id < switch_frame_id) {
+            EXPECT_THAT(full_deps[switch_frame_id], Contains(early_frame_id))
+                << "For decode target #" << dt << " frame " << later_frame_id
+                << " depends on the frame " << early_frame_id
+                << " that switch indication frame " << switch_frame_id
+                << " doesn't directly on indirectly depend on.";
+          }
+        }
+      }
+    }
+  }
+}
+
+INSTANTIATE_TEST_SUITE_P(
+    Svc,
+    ScalabilityStructureTest,
+    Values(SvcTestParam{"L1T2", /*num_temporal_units=*/4},
+           SvcTestParam{"L1T3", /*num_temporal_units=*/8},
+           SvcTestParam{"L2T1", /*num_temporal_units=*/3},
+           SvcTestParam{"L2T1_KEY", /*num_temporal_units=*/3},
+           SvcTestParam{"L3T1", /*num_temporal_units=*/3},
+           SvcTestParam{"L3T3", /*num_temporal_units=*/8},
+           SvcTestParam{"S2T1", /*num_temporal_units=*/3},
+           SvcTestParam{"L2T2", /*num_temporal_units=*/4},
+           SvcTestParam{"L2T2_KEY", /*num_temporal_units=*/4},
+           SvcTestParam{"L2T2_KEY_SHIFT", /*num_temporal_units=*/4}),
+    [](const testing::TestParamInfo<SvcTestParam>& info) {
+      return info.param.name;
+    });
+
+}  // namespace
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalable_video_controller.h

@@ -0,0 +1,141 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABLE_VIDEO_CONTROLLER_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABLE_VIDEO_CONTROLLER_H_
+
+#include <vector>
+
+#include "absl/container/inlined_vector.h"
+#include "absl/types/optional.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "api/video/video_bitrate_allocation.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+
+namespace webrtc {
+
+// Controls how video should be encoded to be scalable. Outputs results as
+// buffer usage configuration for encoder and enough details to communicate the
+// scalability structure via dependency descriptor rtp header extension.
+class ScalableVideoController {
+ public:
+  struct StreamLayersConfig {
+    int num_spatial_layers = 1;
+    int num_temporal_layers = 1;
+    // Spatial layers scaling. Frames with spatial_id = i expected to be encoded
+    // with original_resolution * scaling_factor_num[i] / scaling_factor_den[i].
+    int scaling_factor_num[DependencyDescriptor::kMaxSpatialIds] = {1, 1, 1, 1};
+    int scaling_factor_den[DependencyDescriptor::kMaxSpatialIds] = {1, 1, 1, 1};
+  };
+  class LayerFrameConfig {
+   public:
+    // Builders/setters.
+    LayerFrameConfig& Id(int value);
+    LayerFrameConfig& Keyframe();
+    LayerFrameConfig& S(int value);
+    LayerFrameConfig& T(int value);
+    LayerFrameConfig& Reference(int buffer_id);
+    LayerFrameConfig& Update(int buffer_id);
+    LayerFrameConfig& ReferenceAndUpdate(int buffer_id);
+
+    // Getters.
+    int Id() const { return id_; }
+    bool IsKeyframe() const { return is_keyframe_; }
+    int SpatialId() const { return spatial_id_; }
+    int TemporalId() const { return temporal_id_; }
+    const absl::InlinedVector<CodecBufferUsage, kMaxEncoderBuffers>& Buffers()
+        const {
+      return buffers_;
+    }
+
+   private:
+    // Id to match configuration returned by NextFrameConfig with
+    // (possibly modified) configuration passed back via OnEncoderDone.
+    // The meaning of the id is an implementation detail of
+    // the ScalableVideoController.
+    int id_ = 0;
+
+    // Indication frame should be encoded as a key frame. In particular when
+    // `is_keyframe=true` property `CodecBufferUsage::referenced` should be
+    // ignored and treated as false.
+    bool is_keyframe_ = false;
+
+    int spatial_id_ = 0;
+    int temporal_id_ = 0;
+    // Describes how encoder which buffers encoder allowed to reference and
+    // which buffers encoder should update.
+    absl::InlinedVector<CodecBufferUsage, kMaxEncoderBuffers> buffers_;
+  };
+
+  virtual ~ScalableVideoController() = default;
+
+  // Returns video structure description for encoder to configure itself.
+  virtual StreamLayersConfig StreamConfig() const = 0;
+
+  // Returns video structure description in format compatible with
+  // dependency descriptor rtp header extension.
+  virtual FrameDependencyStructure DependencyStructure() const = 0;
+
+  // Notifies Controller with updated bitrates per layer. In particular notifies
+  // when certain layers should be disabled.
+  // Controller shouldn't produce LayerFrameConfig for disabled layers.
+  // TODO(bugs.webrtc.org/11404): Make pure virtual when implemented by all
+  // structures.
+  virtual void OnRatesUpdated(const VideoBitrateAllocation& bitrates) {}
+
+  // When `restart` is true, first `LayerFrameConfig` should have `is_keyframe`
+  // set to true.
+  // Returned vector shouldn't be empty.
+  virtual std::vector<LayerFrameConfig> NextFrameConfig(bool restart) = 0;
+
+  // Returns configuration to pass to EncoderCallback.
+  virtual absl::optional<GenericFrameInfo> OnEncodeDone(
+      LayerFrameConfig config) = 0;
+};
+
+// Below are implementation details.
+inline ScalableVideoController::LayerFrameConfig&
+ScalableVideoController::LayerFrameConfig::Id(int value) {
+  id_ = value;
+  return *this;
+}
+inline ScalableVideoController::LayerFrameConfig&
+ScalableVideoController::LayerFrameConfig::Keyframe() {
+  is_keyframe_ = true;
+  return *this;
+}
+inline ScalableVideoController::LayerFrameConfig&
+ScalableVideoController::LayerFrameConfig::S(int value) {
+  spatial_id_ = value;
+  return *this;
+}
+inline ScalableVideoController::LayerFrameConfig&
+ScalableVideoController::LayerFrameConfig::T(int value) {
+  temporal_id_ = value;
+  return *this;
+}
+inline ScalableVideoController::LayerFrameConfig&
+ScalableVideoController::LayerFrameConfig::Reference(int buffer_id) {
+  buffers_.emplace_back(buffer_id, /*referenced=*/true, /*updated=*/false);
+  return *this;
+}
+inline ScalableVideoController::LayerFrameConfig&
+ScalableVideoController::LayerFrameConfig::Update(int buffer_id) {
+  buffers_.emplace_back(buffer_id, /*referenced=*/false, /*updated=*/true);
+  return *this;
+}
+inline ScalableVideoController::LayerFrameConfig&
+ScalableVideoController::LayerFrameConfig::ReferenceAndUpdate(int buffer_id) {
+  buffers_.emplace_back(buffer_id, /*referenced=*/true, /*updated=*/true);
+  return *this;
+}
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABLE_VIDEO_CONTROLLER_H_

modules/video_coding/codecs/av1/scalable_video_controller_no_layering.cc

@@ -0,0 +1,67 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "modules/video_coding/codecs/av1/scalable_video_controller_no_layering.h"
+
+#include <utility>
+#include <vector>
+
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+
+ScalableVideoControllerNoLayering::~ScalableVideoControllerNoLayering() =
+    default;
+
+ScalableVideoController::StreamLayersConfig
+ScalableVideoControllerNoLayering::StreamConfig() const {
+  StreamLayersConfig result;
+  result.num_spatial_layers = 1;
+  result.num_temporal_layers = 1;
+  return result;
+}
+
+FrameDependencyStructure
+ScalableVideoControllerNoLayering::DependencyStructure() const {
+  FrameDependencyStructure structure;
+  structure.num_decode_targets = 1;
+  FrameDependencyTemplate a_template;
+  a_template.decode_target_indications = {DecodeTargetIndication::kSwitch};
+  structure.templates.push_back(a_template);
+  return structure;
+}
+
+std::vector<ScalableVideoController::LayerFrameConfig>
+ScalableVideoControllerNoLayering::NextFrameConfig(bool restart) {
+  std::vector<LayerFrameConfig> result(1);
+  if (restart || start_) {
+    result[0].Id(0).Keyframe().Update(0);
+  } else {
+    result[0].Id(0).ReferenceAndUpdate(0);
+  }
+  start_ = false;
+  return result;
+}
+
+absl::optional<GenericFrameInfo>
+ScalableVideoControllerNoLayering::OnEncodeDone(LayerFrameConfig config) {
+  RTC_DCHECK_EQ(config.Id(), 0);
+  absl::optional<GenericFrameInfo> frame_info(absl::in_place);
+  frame_info->encoder_buffers = config.Buffers();
+  if (config.IsKeyframe()) {
+    for (auto& buffer : frame_info->encoder_buffers) {
+      buffer.referenced = false;
+    }
+  }
+  frame_info->decode_target_indications = {DecodeTargetIndication::kSwitch};
+  return frame_info;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/av1/scalable_video_controller_no_layering.h

@@ -0,0 +1,38 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#ifndef MODULES_VIDEO_CODING_CODECS_AV1_SCALABLE_VIDEO_CONTROLLER_NO_LAYERING_H_
+#define MODULES_VIDEO_CODING_CODECS_AV1_SCALABLE_VIDEO_CONTROLLER_NO_LAYERING_H_
+
+#include <vector>
+
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "common_video/generic_frame_descriptor/generic_frame_info.h"
+#include "modules/video_coding/codecs/av1/scalable_video_controller.h"
+
+namespace webrtc {
+
+class ScalableVideoControllerNoLayering : public ScalableVideoController {
+ public:
+  ~ScalableVideoControllerNoLayering() override;
+
+  StreamLayersConfig StreamConfig() const override;
+  FrameDependencyStructure DependencyStructure() const override;
+
+  std::vector<LayerFrameConfig> NextFrameConfig(bool restart) override;
+  absl::optional<GenericFrameInfo> OnEncodeDone(
+      LayerFrameConfig config) override;
+
+ private:
+  bool start_ = true;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_CODECS_AV1_SCALABLE_VIDEO_CONTROLLER_NO_LAYERING_H_

modules/video_coding/codecs/h264/h264_decoder_impl.cc

@@ -32,7 +32,6 @@ extern "C" {
 #include "common_video/include/video_frame_buffer.h"
 #include "modules/video_coding/codecs/h264/h264_color_space.h"
 #include "rtc_base/checks.h"
-#include "rtc_base/critical_section.h"
 #include "rtc_base/keep_ref_until_done.h"
 #include "rtc_base/logging.h"
 #include "system_wrappers/include/field_trial.h"

modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h

@@ -21,7 +21,7 @@
 #include "api/video_codecs/video_encoder_factory.h"
 #include "modules/video_coding/codecs/multiplex/multiplex_encoded_image_packer.h"
 #include "modules/video_coding/include/video_codec_interface.h"
-#include "rtc_base/critical_section.h"
+#include "rtc_base/synchronization/mutex.h"
 
 namespace webrtc {
 
@@ -71,7 +71,7 @@ class MultiplexEncoderAdapter : public VideoEncoder {
   EncodedImageCallback* encoded_complete_callback_;
 
   std::map<uint32_t /* timestamp */, MultiplexImage> stashed_images_
-      RTC_GUARDED_BY(crit_);
+      RTC_GUARDED_BY(mutex_);
 
   uint16_t picture_index_ = 0;
   std::vector<uint8_t> multiplex_dummy_planes_;
@@ -79,7 +79,7 @@ class MultiplexEncoderAdapter : public VideoEncoder {
   int key_frame_interval_;
   EncodedImage combined_image_;
 
-  rtc::CriticalSection crit_;
+  Mutex mutex_;
 
   const bool supports_augmented_data_;
   int augmenting_data_size_ = 0;

modules/video_coding/codecs/multiplex/multiplex_encoded_image_packer.h

@@ -16,6 +16,7 @@
 #include <vector>
 
 #include "api/video/encoded_image.h"
+#include "api/video_codecs/video_codec.h"
 
 namespace webrtc {
 

modules/video_coding/codecs/multiplex/multiplex_encoder_adapter.cc

@@ -180,7 +180,7 @@ int MultiplexEncoderAdapter::Encode(
   }
 
   {
-    rtc::CritScope cs(&crit_);
+    MutexLock lock(&mutex_);
     stashed_images_.emplace(
         std::piecewise_construct,
         std::forward_as_tuple(input_image.timestamp()),
@@ -273,7 +273,7 @@ int MultiplexEncoderAdapter::Release() {
   }
   encoders_.clear();
   adapter_callbacks_.clear();
-  rtc::CritScope cs(&crit_);
+  MutexLock lock(&mutex_);
   stashed_images_.clear();
 
   return WEBRTC_VIDEO_CODEC_OK;
@@ -298,7 +298,7 @@ EncodedImageCallback::Result MultiplexEncoderAdapter::OnEncodedImage(
   // If we don't already own the buffer, make a copy.
   image_component.encoded_image.Retain();
 
-  rtc::CritScope cs(&crit_);
+  MutexLock lock(&mutex_);
   const auto& stashed_image_itr =
       stashed_images_.find(encodedImage.Timestamp());
   const auto& stashed_image_next_itr = std::next(stashed_image_itr, 1);

modules/video_coding/codecs/test/encoded_video_frame_producer.cc

@@ -0,0 +1,78 @@
+/*
+ *  Copyright 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/codecs/test/encoded_video_frame_producer.h"
+
+#include <memory>
+#include <vector>
+
+#include "api/test/create_frame_generator.h"
+#include "api/test/frame_generator_interface.h"
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "api/video/video_frame.h"
+#include "api/video/video_frame_type.h"
+#include "api/video_codecs/video_encoder.h"
+#include "modules/video_coding/include/video_codec_interface.h"
+#include "modules/video_coding/include/video_error_codes.h"
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+namespace {
+
+class EncoderCallback : public EncodedImageCallback {
+ public:
+  explicit EncoderCallback(
+      std::vector<EncodedVideoFrameProducer::EncodedFrame>& output_frames)
+      : output_frames_(output_frames) {}
+
+ private:
+  Result OnEncodedImage(
+      const EncodedImage& encoded_image,
+      const CodecSpecificInfo* codec_specific_info,
+      const RTPFragmentationHeader* /*fragmentation*/) override {
+    output_frames_.push_back({encoded_image, *codec_specific_info});
+    return Result(Result::Error::OK);
+  }
+
+  std::vector<EncodedVideoFrameProducer::EncodedFrame>& output_frames_;
+};
+
+}  // namespace
+
+std::vector<EncodedVideoFrameProducer::EncodedFrame>
+EncodedVideoFrameProducer::Encode() {
+  std::unique_ptr<test::FrameGeneratorInterface> frame_buffer_generator =
+      test::CreateSquareFrameGenerator(
+          resolution_.Width(), resolution_.Height(),
+          test::FrameGeneratorInterface::OutputType::kI420, absl::nullopt);
+
+  std::vector<EncodedFrame> encoded_frames;
+  EncoderCallback encoder_callback(encoded_frames);
+  RTC_CHECK_EQ(encoder_.RegisterEncodeCompleteCallback(&encoder_callback),
+               WEBRTC_VIDEO_CODEC_OK);
+
+  uint32_t rtp_tick = 90000 / framerate_fps_;
+  std::vector<VideoFrameType> frame_types = {VideoFrameType::kVideoFrameDelta};
+  for (int i = 0; i < num_input_frames_; ++i) {
+    VideoFrame frame =
+        VideoFrame::Builder()
+            .set_video_frame_buffer(frame_buffer_generator->NextFrame().buffer)
+            .set_timestamp_rtp(rtp_timestamp_)
+            .build();
+    rtp_timestamp_ += rtp_tick;
+    RTC_CHECK_EQ(encoder_.Encode(frame, &frame_types), WEBRTC_VIDEO_CODEC_OK);
+  }
+
+  RTC_CHECK_EQ(encoder_.RegisterEncodeCompleteCallback(nullptr),
+               WEBRTC_VIDEO_CODEC_OK);
+  return encoded_frames;
+}
+
+}  // namespace webrtc

modules/video_coding/codecs/test/encoded_video_frame_producer.h

@@ -0,0 +1,83 @@
+/*
+ *  Copyright 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_CODECS_TEST_ENCODED_VIDEO_FRAME_PRODUCER_H_
+#define MODULES_VIDEO_CODING_CODECS_TEST_ENCODED_VIDEO_FRAME_PRODUCER_H_
+
+#include <stdint.h>
+
+#include <vector>
+
+#include "api/transport/rtp/dependency_descriptor.h"
+#include "api/video/encoded_image.h"
+#include "api/video_codecs/video_encoder.h"
+#include "modules/video_coding/include/video_codec_interface.h"
+
+namespace webrtc {
+
+// Wrapper around VideoEncoder::Encode for convenient input (generates frames)
+// and output (returns encoded frames instead of passing them to callback)
+class EncodedVideoFrameProducer {
+ public:
+  struct EncodedFrame {
+    EncodedImage encoded_image;
+    CodecSpecificInfo codec_specific_info;
+  };
+
+  // `encoder` should be initialized, but shouldn't have `EncoderCallback` set.
+  explicit EncodedVideoFrameProducer(VideoEncoder& encoder)
+      : encoder_(encoder) {}
+  EncodedVideoFrameProducer(const EncodedVideoFrameProducer&) = delete;
+  EncodedVideoFrameProducer& operator=(const EncodedVideoFrameProducer&) =
+      delete;
+
+  // Number of the input frames to pass to the encoder.
+  EncodedVideoFrameProducer& SetNumInputFrames(int value);
+  // Resolution of the input frames.
+  EncodedVideoFrameProducer& SetResolution(RenderResolution value);
+
+  EncodedVideoFrameProducer& SetFramerateFps(int value);
+
+  // Generates input video frames and encodes them with `encoder` provided in
+  // the constructor. Returns frame passed to the `OnEncodedImage` by wraping
+  // `EncodedImageCallback` underneath.
+  std::vector<EncodedFrame> Encode();
+
+ private:
+  VideoEncoder& encoder_;
+
+  uint32_t rtp_timestamp_ = 1000;
+  int num_input_frames_ = 1;
+  int framerate_fps_ = 30;
+  RenderResolution resolution_ = {320, 180};
+};
+
+inline EncodedVideoFrameProducer& EncodedVideoFrameProducer::SetNumInputFrames(
+    int value) {
+  RTC_DCHECK_GT(value, 0);
+  num_input_frames_ = value;
+  return *this;
+}
+
+inline EncodedVideoFrameProducer& EncodedVideoFrameProducer::SetResolution(
+    RenderResolution value) {
+  resolution_ = value;
+  return *this;
+}
+
+inline EncodedVideoFrameProducer& EncodedVideoFrameProducer::SetFramerateFps(
+    int value) {
+  RTC_DCHECK_GT(value, 0);
+  framerate_fps_ = value;
+  return *this;
+}
+
+}  // namespace webrtc
+#endif  // MODULES_VIDEO_CODING_CODECS_TEST_ENCODED_VIDEO_FRAME_PRODUCER_H_

modules/video_coding/codecs/test/video_codec_unittest.cc

@@ -37,7 +37,7 @@ VideoCodecUnitTest::FakeEncodeCompleteCallback::OnEncodedImage(
     const EncodedImage& frame,
     const CodecSpecificInfo* codec_specific_info,
     const RTPFragmentationHeader* fragmentation) {
-  rtc::CritScope lock(&test_->encoded_frame_section_);
+  MutexLock lock(&test_->encoded_frame_section_);
   test_->encoded_frames_.push_back(frame);
   RTC_DCHECK(codec_specific_info);
   test_->codec_specific_infos_.push_back(*codec_specific_info);
@@ -58,7 +58,7 @@ void VideoCodecUnitTest::FakeDecodeCompleteCallback::Decoded(
     VideoFrame& frame,
     absl::optional<int32_t> decode_time_ms,
     absl::optional<uint8_t> qp) {
-  rtc::CritScope lock(&test_->decoded_frame_section_);
+  MutexLock lock(&test_->decoded_frame_section_);
   test_->decoded_frame_.emplace(frame);
   test_->decoded_qp_ = qp;
   test_->decoded_frame_event_.Set();
@@ -126,7 +126,7 @@ bool VideoCodecUnitTest::WaitForEncodedFrame(
 }
 
 void VideoCodecUnitTest::SetWaitForEncodedFramesThreshold(size_t num_frames) {
-  rtc::CritScope lock(&encoded_frame_section_);
+  MutexLock lock(&encoded_frame_section_);
   wait_for_encoded_frames_threshold_ = num_frames;
 }
 
@@ -136,7 +136,7 @@ bool VideoCodecUnitTest::WaitForEncodedFrames(
   EXPECT_TRUE(encoded_frame_event_.Wait(kEncodeTimeoutMs))
       << "Timed out while waiting for encoded frame.";
   // This becomes unsafe if there are multiple threads waiting for frames.
-  rtc::CritScope lock(&encoded_frame_section_);
+  MutexLock lock(&encoded_frame_section_);
   EXPECT_FALSE(encoded_frames_.empty());
   EXPECT_FALSE(codec_specific_infos_.empty());
   EXPECT_EQ(encoded_frames_.size(), codec_specific_infos_.size());
@@ -157,7 +157,7 @@ bool VideoCodecUnitTest::WaitForDecodedFrame(std::unique_ptr<VideoFrame>* frame,
   bool ret = decoded_frame_event_.Wait(kDecodeTimeoutMs);
   EXPECT_TRUE(ret) << "Timed out while waiting for a decoded frame.";
   // This becomes unsafe if there are multiple threads waiting for frames.
-  rtc::CritScope lock(&decoded_frame_section_);
+  MutexLock lock(&decoded_frame_section_);
   EXPECT_TRUE(decoded_frame_);
   if (decoded_frame_) {
     frame->reset(new VideoFrame(std::move(*decoded_frame_)));
@@ -170,7 +170,7 @@ bool VideoCodecUnitTest::WaitForDecodedFrame(std::unique_ptr<VideoFrame>* frame,
 }
 
 size_t VideoCodecUnitTest::GetNumEncodedFrames() {
-  rtc::CritScope lock(&encoded_frame_section_);
+  MutexLock lock(&encoded_frame_section_);
   return encoded_frames_.size();
 }
 

modules/video_coding/codecs/test/video_codec_unittest.h

@@ -20,8 +20,8 @@
 #include "modules/video_coding/include/video_codec_interface.h"
 #include "modules/video_coding/utility/vp8_header_parser.h"
 #include "modules/video_coding/utility/vp9_uncompressed_header_parser.h"
-#include "rtc_base/critical_section.h"
 #include "rtc_base/event.h"
+#include "rtc_base/synchronization/mutex.h"
 #include "rtc_base/thread_annotations.h"
 #include "test/gtest.h"
 
@@ -108,7 +108,7 @@ class VideoCodecUnitTest : public ::testing::Test {
   FakeDecodeCompleteCallback decode_complete_callback_;
 
   rtc::Event encoded_frame_event_;
-  rtc::CriticalSection encoded_frame_section_;
+  Mutex encoded_frame_section_;
   size_t wait_for_encoded_frames_threshold_;
   std::vector<EncodedImage> encoded_frames_
       RTC_GUARDED_BY(encoded_frame_section_);
@@ -116,7 +116,7 @@ class VideoCodecUnitTest : public ::testing::Test {
       RTC_GUARDED_BY(encoded_frame_section_);
 
   rtc::Event decoded_frame_event_;
-  rtc::CriticalSection decoded_frame_section_;
+  Mutex decoded_frame_section_;
   absl::optional<VideoFrame> decoded_frame_
       RTC_GUARDED_BY(decoded_frame_section_);
   absl::optional<uint8_t> decoded_qp_ RTC_GUARDED_BY(decoded_frame_section_);

modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc

@@ -205,6 +205,9 @@ void VideoCodecTestFixtureImpl::Config::SetCodecSettings(
       codec_settings.VP9()->numberOfSpatialLayers =
           static_cast<uint8_t>(num_spatial_layers);
       break;
+    case kVideoCodecAV1:
+      codec_settings.qpMax = 63;
+      break;
     case kVideoCodecH264:
       codec_settings.H264()->frameDroppingOn = frame_dropper_on;
       codec_settings.H264()->keyFrameInterval = kBaseKeyFrameInterval;

modules/video_coding/codecs/test/videocodec_test_libaom.cc

@@ -0,0 +1,97 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <memory>
+#include <vector>
+
+#include "api/test/create_videocodec_test_fixture.h"
+#include "api/test/video/function_video_encoder_factory.h"
+#include "api/video_codecs/sdp_video_format.h"
+#include "media/base/media_constants.h"
+#include "media/engine/internal_decoder_factory.h"
+#include "media/engine/internal_encoder_factory.h"
+#include "media/engine/simulcast_encoder_adapter.h"
+#include "test/gtest.h"
+#include "test/testsupport/file_utils.h"
+
+namespace webrtc {
+namespace test {
+namespace {
+// Test clips settings.
+constexpr int kCifWidth = 352;
+constexpr int kCifHeight = 288;
+constexpr int kNumFramesLong = 300;
+
+VideoCodecTestFixture::Config CreateConfig(std::string filename) {
+  VideoCodecTestFixture::Config config;
+  config.filename = filename;
+  config.filepath = ResourcePath(config.filename, "yuv");
+  config.num_frames = kNumFramesLong;
+  config.use_single_core = true;
+  return config;
+}
+
+TEST(VideoCodecTestLibaom, HighBitrateAV1) {
+  auto config = CreateConfig("foreman_cif");
+  config.SetCodecSettings(cricket::kAv1CodecName, 1, 1, 1, false, true, true,
+                          kCifWidth, kCifHeight);
+  config.num_frames = kNumFramesLong;
+  auto fixture = CreateVideoCodecTestFixture(config);
+
+  std::vector<RateProfile> rate_profiles = {{500, 30, 0}};
+
+  std::vector<RateControlThresholds> rc_thresholds = {
+      {12, 1, 0, 1, 0.3, 0.1, 0, 1}};
+
+  std::vector<QualityThresholds> quality_thresholds = {{37, 34, 0.94, 0.92}};
+
+  fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
+}
+
+TEST(VideoCodecTestLibaom, VeryLowBitrateAV1) {
+  auto config = CreateConfig("foreman_cif");
+  config.SetCodecSettings(cricket::kAv1CodecName, 1, 1, 1, false, true, true,
+                          kCifWidth, kCifHeight);
+  auto fixture = CreateVideoCodecTestFixture(config);
+
+  std::vector<RateProfile> rate_profiles = {{50, 30, 0}};
+
+  std::vector<RateControlThresholds> rc_thresholds = {
+      {15, 8, 75, 2, 2, 2, 2, 1}};
+
+  std::vector<QualityThresholds> quality_thresholds = {{28, 25, 0.70, 0.62}};
+
+  fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
+}
+
+#if !defined(WEBRTC_ANDROID)
+constexpr int kHdWidth = 1280;
+constexpr int kHdHeight = 720;
+TEST(VideoCodecTestLibaom, HdAV1) {
+  auto config = CreateConfig("ConferenceMotion_1280_720_50");
+  config.SetCodecSettings(cricket::kAv1CodecName, 1, 1, 1, false, true, true,
+                          kHdWidth, kHdHeight);
+  config.num_frames = kNumFramesLong;
+  auto fixture = CreateVideoCodecTestFixture(config);
+
+  std::vector<RateProfile> rate_profiles = {{1000, 50, 0}};
+
+  std::vector<RateControlThresholds> rc_thresholds = {
+      {13, 3, 0, 1, 0.3, 0.1, 0, 1}};
+
+  std::vector<QualityThresholds> quality_thresholds = {{36, 32, 0.93, 0.87}};
+
+  fixture->RunTest(rate_profiles, &rc_thresholds, &quality_thresholds, nullptr);
+}
+#endif
+
+}  // namespace
+}  // namespace test
+}  // namespace webrtc

modules/video_coding/codecs/vp8/default_temporal_layers.cc

@@ -608,58 +608,52 @@ FrameDependencyStructure DefaultTemporalLayers::GetTemplateStructure(
   FrameDependencyStructure template_structure;
   template_structure.num_decode_targets = num_layers;
 
-  using Builder = GenericFrameInfo::Builder;
   switch (num_layers) {
     case 1: {
-      template_structure.templates = {
-          Builder().T(0).Dtis("S").Build(),
-          Builder().T(0).Dtis("S").Fdiffs({1}).Build(),
-      };
+      template_structure.templates.resize(2);
+      template_structure.templates[0].T(0).Dtis("S");
+      template_structure.templates[1].T(0).Dtis("S").FrameDiffs({1});
       return template_structure;
     }
     case 2: {
-      template_structure.templates = {
-          Builder().T(0).Dtis("SS").Build(),
-          Builder().T(0).Dtis("SS").Fdiffs({2}).Build(),
-          Builder().T(0).Dtis("SR").Fdiffs({2}).Build(),
-          Builder().T(1).Dtis("-S").Fdiffs({1}).Build(),
-          Builder().T(1).Dtis("-D").Fdiffs({1, 2}).Build(),
-      };
+      template_structure.templates.resize(5);
+      template_structure.templates[0].T(0).Dtis("SS");
+      template_structure.templates[1].T(0).Dtis("SS").FrameDiffs({2});
+      template_structure.templates[2].T(0).Dtis("SR").FrameDiffs({2});
+      template_structure.templates[3].T(1).Dtis("-S").FrameDiffs({1});
+      template_structure.templates[4].T(1).Dtis("-D").FrameDiffs({2, 1});
       return template_structure;
     }
     case 3: {
       if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) {
-        template_structure.templates = {
-            Builder().T(0).Dtis("SSS").Build(),
-            Builder().T(0).Dtis("SSS").Fdiffs({4}).Build(),
-            Builder().T(1).Dtis("-DR").Fdiffs({2}).Build(),
-            Builder().T(2).Dtis("--S").Fdiffs({1}).Build(),
-            Builder().T(2).Dtis("--D").Fdiffs({1, 2}).Build(),
-        };
+        template_structure.templates.resize(5);
+        template_structure.templates[0].T(0).Dtis("SSS");
+        template_structure.templates[1].T(0).Dtis("SSS").FrameDiffs({4});
+        template_structure.templates[2].T(1).Dtis("-DR").FrameDiffs({2});
+        template_structure.templates[3].T(2).Dtis("--S").FrameDiffs({1});
+        template_structure.templates[4].T(2).Dtis("--D").FrameDiffs({2, 1});
       } else {
-        template_structure.templates = {
-            Builder().T(0).Dtis("SSS").Build(),
-            Builder().T(0).Dtis("SSS").Fdiffs({4}).Build(),
-            Builder().T(0).Dtis("SRR").Fdiffs({4}).Build(),
-            Builder().T(1).Dtis("-SS").Fdiffs({2}).Build(),
-            Builder().T(1).Dtis("-DS").Fdiffs({2, 4}).Build(),
-            Builder().T(2).Dtis("--D").Fdiffs({1}).Build(),
-            Builder().T(2).Dtis("--D").Fdiffs({1, 3}).Build(),
-        };
+        template_structure.templates.resize(7);
+        template_structure.templates[0].T(0).Dtis("SSS");
+        template_structure.templates[1].T(0).Dtis("SSS").FrameDiffs({4});
+        template_structure.templates[2].T(0).Dtis("SRR").FrameDiffs({4});
+        template_structure.templates[3].T(1).Dtis("-SS").FrameDiffs({2});
+        template_structure.templates[4].T(1).Dtis("-DS").FrameDiffs({4, 2});
+        template_structure.templates[5].T(2).Dtis("--D").FrameDiffs({1});
+        template_structure.templates[6].T(2).Dtis("--D").FrameDiffs({3, 1});
       }
       return template_structure;
     }
     case 4: {
-      template_structure.templates = {
-          Builder().T(0).Dtis("SSSS").Build(),
-          Builder().T(0).Dtis("SSSS").Fdiffs({8}).Build(),
-          Builder().T(1).Dtis("-SRR").Fdiffs({4}).Build(),
-          Builder().T(1).Dtis("-SRR").Fdiffs({4, 8}).Build(),
-          Builder().T(2).Dtis("--SR").Fdiffs({2}).Build(),
-          Builder().T(2).Dtis("--SR").Fdiffs({2, 4}).Build(),
-          Builder().T(3).Dtis("---D").Fdiffs({1}).Build(),
-          Builder().T(3).Dtis("---D").Fdiffs({1, 3}).Build(),
-      };
+      template_structure.templates.resize(8);
+      template_structure.templates[0].T(0).Dtis("SSSS");
+      template_structure.templates[1].T(0).Dtis("SSSS").FrameDiffs({8});
+      template_structure.templates[2].T(1).Dtis("-SRR").FrameDiffs({4});
+      template_structure.templates[3].T(1).Dtis("-SRR").FrameDiffs({4, 8});
+      template_structure.templates[4].T(2).Dtis("--SR").FrameDiffs({2});
+      template_structure.templates[5].T(2).Dtis("--SR").FrameDiffs({2, 4});
+      template_structure.templates[6].T(3).Dtis("---D").FrameDiffs({1});
+      template_structure.templates[7].T(3).Dtis("---D").FrameDiffs({1, 3});
       return template_structure;
     }
     default:

modules/video_coding/codecs/vp8/default_temporal_layers.h

@@ -75,7 +75,7 @@ class DefaultTemporalLayers final : public Vp8FrameBufferController {
     DependencyInfo(absl::string_view indication_symbols,
                    Vp8FrameConfig frame_config)
         : decode_target_indications(
-              GenericFrameInfo::DecodeTargetInfo(indication_symbols)),
+              webrtc_impl::StringToDecodeTargetIndications(indication_symbols)),
           frame_config(frame_config) {}
 
     absl::InlinedVector<DecodeTargetIndication, 10> decode_target_indications;

modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc

@@ -44,26 +44,48 @@ constexpr int kVp8ErrorPropagationTh = 30;
 constexpr long kDecodeDeadlineRealtime = 1;  // NOLINT
 
 const char kVp8PostProcArmFieldTrial[] = "WebRTC-VP8-Postproc-Config-Arm";
+const char kVp8PostProcFieldTrial[] = "WebRTC-VP8-Postproc-Config";
 
-void GetPostProcParamsFromFieldTrialGroup(
-    LibvpxVp8Decoder::DeblockParams* deblock_params) {
-  std::string group =
-      webrtc::field_trial::FindFullName(kVp8PostProcArmFieldTrial);
-  if (group.empty())
-    return;
+#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
+    defined(WEBRTC_ANDROID)
+constexpr bool kIsArm = true;
+#else
+constexpr bool kIsArm = false;
+#endif
+
+absl::optional<LibvpxVp8Decoder::DeblockParams> DefaultDeblockParams() {
+  if (kIsArm) {
+    // For ARM, this is only called when deblocking is explicitly enabled, and
+    // the default strength is set by the ctor.
+    return LibvpxVp8Decoder::DeblockParams();
+  }
+  // For non-arm, don't use the explicit deblocking settings by default.
+  return absl::nullopt;
+}
+
+absl::optional<LibvpxVp8Decoder::DeblockParams>
+GetPostProcParamsFromFieldTrialGroup() {
+  std::string group = webrtc::field_trial::FindFullName(
+      kIsArm ? kVp8PostProcArmFieldTrial : kVp8PostProcFieldTrial);
+  if (group.empty()) {
+    return DefaultDeblockParams();
+  }
 
   LibvpxVp8Decoder::DeblockParams params;
   if (sscanf(group.c_str(), "Enabled-%d,%d,%d", &params.max_level,
-             &params.min_qp, &params.degrade_qp) != 3)
-    return;
+             &params.min_qp, &params.degrade_qp) != 3) {
+    return DefaultDeblockParams();
+  }
 
-  if (params.max_level < 0 || params.max_level > 16)
-    return;
+  if (params.max_level < 0 || params.max_level > 16) {
+    return DefaultDeblockParams();
+  }
 
-  if (params.min_qp < 0 || params.degrade_qp <= params.min_qp)
-    return;
+  if (params.min_qp < 0 || params.degrade_qp <= params.min_qp) {
+    return DefaultDeblockParams();
+  }
 
-  *deblock_params = params;
+  return params;
 }
 
 }  // namespace
@@ -97,8 +119,9 @@ class LibvpxVp8Decoder::QpSmoother {
 };
 
 LibvpxVp8Decoder::LibvpxVp8Decoder()
-    : use_postproc_arm_(
-          webrtc::field_trial::IsEnabled(kVp8PostProcArmFieldTrial)),
+    : use_postproc_(
+          kIsArm ? webrtc::field_trial::IsEnabled(kVp8PostProcArmFieldTrial)
+                 : true),
       buffer_pool_(false, 300 /* max_number_of_buffers*/),
       decode_complete_callback_(NULL),
       inited_(false),
@@ -107,10 +130,9 @@ LibvpxVp8Decoder::LibvpxVp8Decoder()
       last_frame_width_(0),
       last_frame_height_(0),
       key_frame_required_(true),
-      qp_smoother_(use_postproc_arm_ ? new QpSmoother() : nullptr) {
-  if (use_postproc_arm_)
-    GetPostProcParamsFromFieldTrialGroup(&deblock_);
-}
+      deblock_params_(use_postproc_ ? GetPostProcParamsFromFieldTrialGroup()
+                                    : absl::nullopt),
+      qp_smoother_(use_postproc_ ? new QpSmoother() : nullptr) {}
 
 LibvpxVp8Decoder::~LibvpxVp8Decoder() {
   inited_ = true;  // in order to do the actual release
@@ -131,12 +153,7 @@ int LibvpxVp8Decoder::InitDecode(const VideoCodec* inst, int number_of_cores) {
   cfg.threads = 1;
   cfg.h = cfg.w = 0;  // set after decode
 
-#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
-    defined(WEBRTC_ANDROID)
-  vpx_codec_flags_t flags = use_postproc_arm_ ? VPX_CODEC_USE_POSTPROC : 0;
-#else
-  vpx_codec_flags_t flags = VPX_CODEC_USE_POSTPROC;
-#endif
+  vpx_codec_flags_t flags = use_postproc_ ? VPX_CODEC_USE_POSTPROC : 0;
 
   if (vpx_codec_dec_init(decoder_, vpx_codec_vp8_dx(), &cfg, flags)) {
     delete decoder_;
@@ -174,43 +191,47 @@ int LibvpxVp8Decoder::Decode(const EncodedImage& input_image,
   }
 
 // Post process configurations.
-#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
-    defined(WEBRTC_ANDROID)
-  if (use_postproc_arm_) {
+  if (use_postproc_) {
     vp8_postproc_cfg_t ppcfg;
+    // MFQE enabled to reduce key frame popping.
     ppcfg.post_proc_flag = VP8_MFQE;
-    // For low resolutions, use stronger deblocking filter.
-    int last_width_x_height = last_frame_width_ * last_frame_height_;
-    if (last_width_x_height > 0 && last_width_x_height <= 320 * 240) {
-      // Enable the deblock and demacroblocker based on qp thresholds.
-      RTC_DCHECK(qp_smoother_);
-      int qp = qp_smoother_->GetAvg();
-      if (qp > deblock_.min_qp) {
-        int level = deblock_.max_level;
-        if (qp < deblock_.degrade_qp) {
-          // Use lower level.
-          level = deblock_.max_level * (qp - deblock_.min_qp) /
-                  (deblock_.degrade_qp - deblock_.min_qp);
-        }
-        // Deblocking level only affects VP8_DEMACROBLOCK.
-        ppcfg.deblocking_level = std::max(level, 1);
-        ppcfg.post_proc_flag |= VP8_DEBLOCK | VP8_DEMACROBLOCK;
-      }
+
+    if (kIsArm) {
+      RTC_DCHECK(deblock_params_.has_value());
     }
+    if (deblock_params_.has_value()) {
+      // For low resolutions, use stronger deblocking filter.
+      int last_width_x_height = last_frame_width_ * last_frame_height_;
+      if (last_width_x_height > 0 && last_width_x_height <= 320 * 240) {
+        // Enable the deblock and demacroblocker based on qp thresholds.
+        RTC_DCHECK(qp_smoother_);
+        int qp = qp_smoother_->GetAvg();
+        if (qp > deblock_params_->min_qp) {
+          int level = deblock_params_->max_level;
+          if (qp < deblock_params_->degrade_qp) {
+            // Use lower level.
+            level = deblock_params_->max_level *
+                    (qp - deblock_params_->min_qp) /
+                    (deblock_params_->degrade_qp - deblock_params_->min_qp);
+          }
+          // Deblocking level only affects VP8_DEMACROBLOCK.
+          ppcfg.deblocking_level = std::max(level, 1);
+          ppcfg.post_proc_flag |= VP8_DEBLOCK | VP8_DEMACROBLOCK;
+        }
+      }
+    } else {
+      // Non-arm with no explicit deblock params set.
+      ppcfg.post_proc_flag |= VP8_DEBLOCK;
+      // For VGA resolutions and lower, enable the demacroblocker postproc.
+      if (last_frame_width_ * last_frame_height_ <= 640 * 360) {
+        ppcfg.post_proc_flag |= VP8_DEMACROBLOCK;
+      }
+      // Strength of deblocking filter. Valid range:[0,16]
+      ppcfg.deblocking_level = 3;
+    }
+
     vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg);
   }
-#else
-  vp8_postproc_cfg_t ppcfg;
-  // MFQE enabled to reduce key frame popping.
-  ppcfg.post_proc_flag = VP8_MFQE | VP8_DEBLOCK;
-  // For VGA resolutions and lower, enable the demacroblocker postproc.
-  if (last_frame_width_ * last_frame_height_ <= 640 * 360) {
-    ppcfg.post_proc_flag |= VP8_DEMACROBLOCK;
-  }
-  // Strength of deblocking filter. Valid range:[0,16]
-  ppcfg.deblocking_level = 3;
-  vpx_codec_control(decoder_, VP8_SET_POSTPROC, &ppcfg);
-#endif
 
   // Always start with a complete key frame.
   if (key_frame_required_) {

modules/video_coding/codecs/vp8/libvpx_vp8_decoder.h

@@ -13,6 +13,7 @@
 
 #include <memory>
 
+#include "absl/types/optional.h"
 #include "api/video/encoded_image.h"
 #include "api/video_codecs/video_decoder.h"
 #include "common_video/include/i420_buffer_pool.h"
@@ -51,7 +52,7 @@ class LibvpxVp8Decoder : public VideoDecoder {
                   uint32_t timeStamp,
                   int qp,
                   const webrtc::ColorSpace* explicit_color_space);
-  const bool use_postproc_arm_;
+  const bool use_postproc_;
 
   I420BufferPool buffer_pool_;
   DecodedImageCallback* decode_complete_callback_;
@@ -61,7 +62,7 @@ class LibvpxVp8Decoder : public VideoDecoder {
   int last_frame_width_;
   int last_frame_height_;
   bool key_frame_required_;
-  DeblockParams deblock_;
+  const absl::optional<DeblockParams> deblock_params_;
   const std::unique_ptr<QpSmoother> qp_smoother_;
 };
 

modules/video_coding/codecs/vp8/screenshare_layers.cc

@@ -36,6 +36,7 @@ constexpr int kMinTimeBetweenSyncs = kOneSecond90Khz * 2;
 constexpr int kMaxTimeBetweenSyncs = kOneSecond90Khz * 4;
 constexpr int kQpDeltaThresholdForSync = 8;
 constexpr int kMinBitrateKbpsForQpBoost = 500;
+constexpr auto kSwitch = DecodeTargetIndication::kSwitch;
 }  // namespace
 
 const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
@@ -319,8 +320,7 @@ void ScreenshareLayers::OnEncodeDone(size_t stream_index,
   if (number_of_temporal_layers_ == 1) {
     vp8_info.temporalIdx = kNoTemporalIdx;
     vp8_info.layerSync = false;
-    generic_frame_info.decode_target_indications =
-        GenericFrameInfo::DecodeTargetInfo("S");
+    generic_frame_info.decode_target_indications = {kSwitch};
     generic_frame_info.encoder_buffers.emplace_back(
         0, /*referenced=*/!is_keyframe, /*updated=*/true);
   } else {
@@ -344,8 +344,7 @@ void ScreenshareLayers::OnEncodeDone(size_t stream_index,
       active_layer_ = 1;
       info->template_structure =
           GetTemplateStructure(number_of_temporal_layers_);
-      generic_frame_info.decode_target_indications =
-          GenericFrameInfo::DecodeTargetInfo("SS");
+      generic_frame_info.decode_target_indications = {kSwitch, kSwitch};
     } else if (active_layer_ >= 0 && layers_[active_layer_].state ==
                                          TemporalLayer::State::kKeyFrame) {
       layers_[active_layer_].state = TemporalLayer::State::kNormal;
@@ -429,21 +428,18 @@ FrameDependencyStructure ScreenshareLayers::GetTemplateStructure(
   FrameDependencyStructure template_structure;
   template_structure.num_decode_targets = num_layers;
 
-  using Builder = GenericFrameInfo::Builder;
   switch (num_layers) {
     case 1: {
-      template_structure.templates = {
-          Builder().T(0).Dtis("S").Build(),
-          Builder().T(0).Dtis("S").Fdiffs({1}).Build(),
-      };
+      template_structure.templates.resize(2);
+      template_structure.templates[0].T(0).Dtis("S");
+      template_structure.templates[1].T(0).Dtis("S").FrameDiffs({1});
       return template_structure;
     }
     case 2: {
-      template_structure.templates = {
-          Builder().T(0).Dtis("SS").Build(),
-          Builder().T(0).Dtis("SS").Fdiffs({1}).Build(),
-          Builder().T(1).Dtis("-S").Fdiffs({1}).Build(),
-      };
+      template_structure.templates.resize(3);
+      template_structure.templates[0].T(0).Dtis("SS");
+      template_structure.templates[1].T(0).Dtis("SS").FrameDiffs({1});
+      template_structure.templates[2].T(1).Dtis("-S").FrameDiffs({1});
       return template_structure;
     }
     default:

modules/video_coding/codecs/vp8/screenshare_layers.h

@@ -78,7 +78,7 @@ class ScreenshareLayers final : public Vp8FrameBufferController {
     DependencyInfo(absl::string_view indication_symbols,
                    Vp8FrameConfig frame_config)
         : decode_target_indications(
-              GenericFrameInfo::DecodeTargetInfo(indication_symbols)),
+              webrtc_impl::StringToDecodeTargetIndications(indication_symbols)),
           frame_config(frame_config) {}
 
     absl::InlinedVector<DecodeTargetIndication, 10> decode_target_indications;

modules/video_coding/codecs/vp8/test/mock_libvpx_interface.h

@@ -19,74 +19,86 @@ namespace webrtc {
 
 class MockLibvpxVp8Interface : public LibvpxInterface {
  public:
-  MOCK_CONST_METHOD5(img_alloc,
-                     vpx_image_t*(vpx_image_t*,
-                                  vpx_img_fmt_t,
-                                  unsigned int,
-                                  unsigned int,
-                                  unsigned int));
-  MOCK_CONST_METHOD6(img_wrap,
-                     vpx_image_t*(vpx_image_t*,
-                                  vpx_img_fmt_t,
-                                  unsigned int,
-                                  unsigned int,
-                                  unsigned int,
-                                  unsigned char*));
-  MOCK_CONST_METHOD1(img_free, void(vpx_image_t* img));
-  MOCK_CONST_METHOD2(codec_enc_config_set,
-                     vpx_codec_err_t(vpx_codec_ctx_t*,
-                                     const vpx_codec_enc_cfg_t*));
-  MOCK_CONST_METHOD3(codec_enc_config_default,
-                     vpx_codec_err_t(vpx_codec_iface_t*,
-                                     vpx_codec_enc_cfg_t*,
-                                     unsigned int));
-  MOCK_CONST_METHOD4(codec_enc_init,
-                     vpx_codec_err_t(vpx_codec_ctx_t*,
-                                     vpx_codec_iface_t*,
-                                     const vpx_codec_enc_cfg_t*,
-                                     vpx_codec_flags_t));
-  MOCK_CONST_METHOD6(codec_enc_init_multi,
-                     vpx_codec_err_t(vpx_codec_ctx_t*,
-                                     vpx_codec_iface_t*,
-                                     vpx_codec_enc_cfg_t*,
-                                     int,
-                                     vpx_codec_flags_t,
-                                     vpx_rational_t*));
-  MOCK_CONST_METHOD1(codec_destroy, vpx_codec_err_t(vpx_codec_ctx_t*));
-  MOCK_CONST_METHOD3(codec_control,
-                     vpx_codec_err_t(vpx_codec_ctx_t*,
-                                     vp8e_enc_control_id,
-                                     uint32_t));
-  MOCK_CONST_METHOD3(codec_control,
-                     vpx_codec_err_t(vpx_codec_ctx_t*,
-                                     vp8e_enc_control_id,
-                                     int));
-  MOCK_CONST_METHOD3(codec_control,
-                     vpx_codec_err_t(vpx_codec_ctx_t*,
-                                     vp8e_enc_control_id,
-                                     int*));
-  MOCK_CONST_METHOD3(codec_control,
-                     vpx_codec_err_t(vpx_codec_ctx_t*,
-                                     vp8e_enc_control_id,
-                                     vpx_roi_map*));
-  MOCK_CONST_METHOD3(codec_control,
-                     vpx_codec_err_t(vpx_codec_ctx_t*,
-                                     vp8e_enc_control_id,
-                                     vpx_active_map*));
-  MOCK_CONST_METHOD3(codec_control,
-                     vpx_codec_err_t(vpx_codec_ctx_t*,
-                                     vp8e_enc_control_id,
-                                     vpx_scaling_mode*));
-  MOCK_CONST_METHOD6(codec_encode,
-                     vpx_codec_err_t(vpx_codec_ctx_t*,
-                                     const vpx_image_t*,
-                                     vpx_codec_pts_t,
-                                     uint64_t,
-                                     vpx_enc_frame_flags_t,
-                                     uint64_t));
-  MOCK_CONST_METHOD2(codec_get_cx_data,
-                     const vpx_codec_cx_pkt_t*(vpx_codec_ctx_t*,
-                                               vpx_codec_iter_t*));
+  MOCK_METHOD(
+      vpx_image_t*,
+      img_alloc,
+      (vpx_image_t*, vpx_img_fmt_t, unsigned int, unsigned int, unsigned int),
+      (const, override));
+  MOCK_METHOD(vpx_image_t*,
+              img_wrap,
+              (vpx_image_t*,
+               vpx_img_fmt_t,
+               unsigned int,
+               unsigned int,
+               unsigned int,
+               unsigned char*),
+              (const, override));
+  MOCK_METHOD(void, img_free, (vpx_image_t * img), (const, override));
+  MOCK_METHOD(vpx_codec_err_t,
+              codec_enc_config_set,
+              (vpx_codec_ctx_t*, const vpx_codec_enc_cfg_t*),
+              (const, override));
+  MOCK_METHOD(vpx_codec_err_t,
+              codec_enc_config_default,
+              (vpx_codec_iface_t*, vpx_codec_enc_cfg_t*, unsigned int),
+              (const, override));
+  MOCK_METHOD(vpx_codec_err_t,
+              codec_enc_init,
+              (vpx_codec_ctx_t*,
+               vpx_codec_iface_t*,
+               const vpx_codec_enc_cfg_t*,
+               vpx_codec_flags_t),
+              (const, override));
+  MOCK_METHOD(vpx_codec_err_t,
+              codec_enc_init_multi,
+              (vpx_codec_ctx_t*,
+               vpx_codec_iface_t*,
+               vpx_codec_enc_cfg_t*,
+               int,
+               vpx_codec_flags_t,
+               vpx_rational_t*),
+              (const, override));
+  MOCK_METHOD(vpx_codec_err_t,
+              codec_destroy,
+              (vpx_codec_ctx_t*),
+              (const, override));
+  MOCK_METHOD(vpx_codec_err_t,
+              codec_control,
+              (vpx_codec_ctx_t*, vp8e_enc_control_id, uint32_t),
+              (const, override));
+  MOCK_METHOD(vpx_codec_err_t,
+              codec_control,
+              (vpx_codec_ctx_t*, vp8e_enc_control_id, int),
+              (const, override));
+  MOCK_METHOD(vpx_codec_err_t,
+              codec_control,
+              (vpx_codec_ctx_t*, vp8e_enc_control_id, int*),
+              (const, override));
+  MOCK_METHOD(vpx_codec_err_t,
+              codec_control,
+              (vpx_codec_ctx_t*, vp8e_enc_control_id, vpx_roi_map*),
+              (const, override));
+  MOCK_METHOD(vpx_codec_err_t,
+              codec_control,
+              (vpx_codec_ctx_t*, vp8e_enc_control_id, vpx_active_map*),
+              (const, override));
+  MOCK_METHOD(vpx_codec_err_t,
+              codec_control,
+              (vpx_codec_ctx_t*, vp8e_enc_control_id, vpx_scaling_mode*),
+              (const, override));
+  MOCK_METHOD(vpx_codec_err_t,
+              codec_encode,
+              (vpx_codec_ctx_t*,
+               const vpx_image_t*,
+               vpx_codec_pts_t,
+               uint64_t,
+               vpx_enc_frame_flags_t,
+               uint64_t),
+              (const, override));
+  MOCK_METHOD(const vpx_codec_cx_pkt_t*,
+              codec_get_cx_data,
+              (vpx_codec_ctx_t*, vpx_codec_iter_t*),
+              (const, override));
 };
 
 }  // namespace webrtc

modules/video_coding/codecs/vp9/include/vp9.h

@@ -25,6 +25,10 @@ namespace webrtc {
 // negotiate in SDP, in order of preference.
 std::vector<SdpVideoFormat> SupportedVP9Codecs();
 
+// Returns a vector with all supported internal VP9 decode profiles in order of
+// preference. These will be availble for receive-only connections.
+std::vector<SdpVideoFormat> SupportedVP9DecoderCodecs();
+
 class VP9Encoder : public VideoEncoder {
  public:
   // Deprecated. Returns default implementation using VP9 Profile 0.

modules/video_coding/codecs/vp9/include/vp9_globals.h

@@ -30,8 +30,8 @@ const size_t kMaxVp9RefPics = 3;
 const size_t kMaxVp9FramesInGof = 0xFF;  // 8 bits
 const size_t kMaxVp9NumberOfSpatialLayers = 8;
 
-const size_t kMinVp9SpatialLayerWidth = 320;
-const size_t kMinVp9SpatialLayerHeight = 180;
+const size_t kMinVp9SpatialLayerWidth = 240;
+const size_t kMinVp9SpatialLayerHeight = 135;
 
 enum TemporalStructureMode {
   kTemporalStructureMode1,  // 1 temporal layer structure - i.e., IPPP...

modules/video_coding/codecs/vp9/svc_config.cc

@@ -16,6 +16,7 @@
 
 #include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
 #include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
 
 namespace webrtc {
 
@@ -74,11 +75,23 @@ std::vector<SpatialLayer> ConfigureSvcNormalVideo(size_t input_width,
   const size_t num_layers_fit_vert = static_cast<size_t>(
       std::floor(1 + std::max(0.0f, std::log2(1.0f * input_height /
                                               kMinVp9SpatialLayerHeight))));
-  num_spatial_layers =
-      std::min({num_spatial_layers, num_layers_fit_horz, num_layers_fit_vert});
+  const size_t limited_num_spatial_layers =
+      std::min(num_layers_fit_horz, num_layers_fit_vert);
+  if (limited_num_spatial_layers < num_spatial_layers) {
+    RTC_LOG(LS_WARNING) << "Reducing number of spatial layers from "
+                        << num_spatial_layers << " to "
+                        << limited_num_spatial_layers
+                        << " due to low input resolution.";
+    num_spatial_layers = limited_num_spatial_layers;
+  }
   // First active layer must be configured.
   num_spatial_layers = std::max(num_spatial_layers, first_active_layer + 1);
 
+  // Ensure top layer is even enough.
+  int required_divisiblity = 1 << (num_spatial_layers - first_active_layer - 1);
+  input_width = input_width - input_width % required_divisiblity;
+  input_height = input_height - input_height % required_divisiblity;
+
   for (size_t sl_idx = first_active_layer; sl_idx < num_spatial_layers;
        ++sl_idx) {
     SpatialLayer spatial_layer = {0};
@@ -108,6 +121,19 @@ std::vector<SpatialLayer> ConfigureSvcNormalVideo(size_t input_width,
     spatial_layers.push_back(spatial_layer);
   }
 
+  // A workaround for sitiation when single HD layer is left with minBitrate
+  // about 500kbps. This would mean that there will always be at least 500kbps
+  // allocated to video regardless of how low is the actual BWE.
+  // Also, boost maxBitrate for the first layer to account for lost ability to
+  // predict from previous layers.
+  if (first_active_layer > 0) {
+    spatial_layers[0].minBitrate = kMinVp9SvcBitrateKbps;
+    // TODO(ilnik): tune this value or come up with a different formula to
+    // ensure that all singlecast configurations look good and not too much
+    // bitrate is added.
+    spatial_layers[0].maxBitrate *= 1.1;
+  }
+
   return spatial_layers;
 }
 

modules/video_coding/codecs/vp9/svc_config_unittest.cc

@@ -41,6 +41,32 @@ TEST(SvcConfig, AlwaysSendsAtLeastOneLayer) {
   EXPECT_EQ(spatial_layers.back().width, kMinVp9SpatialLayerWidth);
 }
 
+TEST(SvcConfig, EnforcesMinimalRequiredParity) {
+  const size_t max_num_spatial_layers = 3;
+  const size_t kOddSize = 1023;
+
+  std::vector<SpatialLayer> spatial_layers =
+      GetSvcConfig(kOddSize, kOddSize, 30,
+                   /*first_active_layer=*/1, max_num_spatial_layers, 1, false);
+  // Since there are 2 layers total (1, 2), divisiblity by 2 is required.
+  EXPECT_EQ(spatial_layers.back().width, kOddSize - 1);
+  EXPECT_EQ(spatial_layers.back().width, kOddSize - 1);
+
+  spatial_layers =
+      GetSvcConfig(kOddSize, kOddSize, 30,
+                   /*first_active_layer=*/0, max_num_spatial_layers, 1, false);
+  // Since there are 3 layers total (0, 1, 2), divisiblity by 4 is required.
+  EXPECT_EQ(spatial_layers.back().width, kOddSize - 3);
+  EXPECT_EQ(spatial_layers.back().width, kOddSize - 3);
+
+  spatial_layers =
+      GetSvcConfig(kOddSize, kOddSize, 30,
+                   /*first_active_layer=*/2, max_num_spatial_layers, 1, false);
+  // Since there is only 1 layer active (2), divisiblity by 1 is required.
+  EXPECT_EQ(spatial_layers.back().width, kOddSize);
+  EXPECT_EQ(spatial_layers.back().width, kOddSize);
+}
+
 TEST(SvcConfig, SkipsInactiveLayers) {
   const size_t num_spatial_layers = 4;
   const size_t first_active_layer = 2;

modules/video_coding/codecs/vp9/svc_rate_allocator.cc

@@ -140,7 +140,8 @@ DataRate FindLayerTogglingThreshold(const VideoCodec& codec,
       }
     }
     upper_bound += DataRate::KilobitsPerSec(
-        codec.spatialLayers[num_active_layers - 1].minBitrate);
+        codec.spatialLayers[first_active_layer + num_active_layers - 1]
+            .minBitrate);
 
     // Do a binary search until upper and lower bound is the highest bitrate for
     // |num_active_layers| - 1 layers and lowest bitrate for |num_active_layers|

modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc

@@ -16,6 +16,7 @@
 #include "common_video/libyuv/include/webrtc_libyuv.h"
 #include "media/base/vp9_profile.h"
 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
+#include "modules/video_coding/codecs/test/encoded_video_frame_producer.h"
 #include "modules/video_coding/codecs/test/video_codec_unittest.h"
 #include "modules/video_coding/codecs/vp9/include/vp9.h"
 #include "modules/video_coding/codecs/vp9/svc_config.h"
@@ -25,20 +26,33 @@
 #include "test/video_codec_settings.h"
 
 namespace webrtc {
+namespace {
 
 using ::testing::ElementsAreArray;
+using ::testing::SizeIs;
+using ::testing::UnorderedElementsAreArray;
 using EncoderInfo = webrtc::VideoEncoder::EncoderInfo;
 using FramerateFractions =
     absl::InlinedVector<uint8_t, webrtc::kMaxTemporalStreams>;
 
-namespace {
-const size_t kWidth = 1280;
-const size_t kHeight = 720;
+constexpr size_t kWidth = 1280;
+constexpr size_t kHeight = 720;
 
 const VideoEncoder::Capabilities kCapabilities(false);
 const VideoEncoder::Settings kSettings(kCapabilities,
                                        /*number_of_cores=*/1,
                                        /*max_payload_size=*/0);
+
+VideoCodec DefaultCodecSettings() {
+  VideoCodec codec_settings;
+  webrtc::test::CodecSettings(kVideoCodecVP9, &codec_settings);
+  codec_settings.width = kWidth;
+  codec_settings.height = kHeight;
+  codec_settings.VP9()->numberOfTemporalLayers = 1;
+  codec_settings.VP9()->numberOfSpatialLayers = 1;
+  return codec_settings;
+}
+
 }  // namespace
 
 class TestVp9Impl : public VideoCodecUnitTest {
@@ -59,53 +73,6 @@ class TestVp9Impl : public VideoCodecUnitTest {
     codec_settings->VP9()->numberOfSpatialLayers = 1;
   }
 
-  void ExpectFrameWith(uint8_t temporal_idx) {
-    EncodedImage encoded_frame;
-    CodecSpecificInfo codec_specific_info;
-    ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
-    EXPECT_EQ(temporal_idx, codec_specific_info.codecSpecific.VP9.temporal_idx);
-  }
-
-  void ExpectFrameWith(size_t num_spatial_layers,
-                       uint8_t temporal_idx,
-                       bool temporal_up_switch,
-                       uint8_t num_ref_pics,
-                       const std::vector<uint8_t>& p_diff) {
-    std::vector<EncodedImage> encoded_frame;
-    std::vector<CodecSpecificInfo> codec_specific;
-    ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific));
-    for (size_t spatial_idx = 0; spatial_idx < num_spatial_layers;
-         ++spatial_idx) {
-      const CodecSpecificInfoVP9& vp9 =
-          codec_specific[spatial_idx].codecSpecific.VP9;
-      if (vp9.temporal_idx == kNoTemporalIdx) {
-        EXPECT_EQ(temporal_idx, 0);
-      } else {
-        EXPECT_EQ(vp9.temporal_idx, temporal_idx);
-      }
-      if (num_spatial_layers == 1) {
-        EXPECT_FALSE(encoded_frame[spatial_idx].SpatialIndex());
-      } else {
-        EXPECT_EQ(encoded_frame[spatial_idx].SpatialIndex(),
-                  static_cast<int>(spatial_idx));
-      }
-      EXPECT_EQ(vp9.temporal_up_switch, temporal_up_switch);
-
-      // Ensure there are no duplicates in reference list.
-      std::vector<uint8_t> vp9_p_diff(vp9.p_diff,
-                                      vp9.p_diff + vp9.num_ref_pics);
-      std::sort(vp9_p_diff.begin(), vp9_p_diff.end());
-      EXPECT_EQ(std::unique(vp9_p_diff.begin(), vp9_p_diff.end()),
-                vp9_p_diff.end());
-
-      for (size_t ref_pic_num = 0; ref_pic_num < num_ref_pics; ++ref_pic_num) {
-        EXPECT_NE(
-            std::find(p_diff.begin(), p_diff.end(), vp9.p_diff[ref_pic_num]),
-            p_diff.end());
-      }
-    }
-  }
-
   void ConfigureSvc(size_t num_spatial_layers, size_t num_temporal_layers = 1) {
     codec_settings_.VP9()->numberOfSpatialLayers =
         static_cast<unsigned char>(num_spatial_layers);
@@ -187,57 +154,61 @@ TEST_F(TestVp9Impl, DecodedQpEqualsEncodedQp) {
   EXPECT_EQ(encoded_frame.qp_, *decoded_qp);
 }
 
-TEST_F(TestVp9Impl, ParserQpEqualsEncodedQp) {
-  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
-  EncodedImage encoded_frame;
-  CodecSpecificInfo codec_specific_info;
-  ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
+TEST(Vp9ImplTest, ParserQpEqualsEncodedQp) {
+  std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
+  VideoCodec codec_settings = DefaultCodecSettings();
+  encoder->InitEncode(&codec_settings, kSettings);
 
+  std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
+      EncodedVideoFrameProducer(*encoder)
+          .SetNumInputFrames(1)
+          .SetResolution({kWidth, kHeight})
+          .Encode();
+  ASSERT_THAT(frames, SizeIs(1));
+  const auto& encoded_frame = frames.front().encoded_image;
   int qp = 0;
   ASSERT_TRUE(vp9::GetQp(encoded_frame.data(), encoded_frame.size(), &qp));
   EXPECT_EQ(encoded_frame.qp_, qp);
 }
 
-TEST_F(TestVp9Impl, EncoderWith2TemporalLayers) {
-  // Override default settings.
-  codec_settings_.VP9()->numberOfTemporalLayers = 2;
+TEST(Vp9ImplTest, EncoderWith2TemporalLayers) {
+  std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
+  VideoCodec codec_settings = DefaultCodecSettings();
+  codec_settings.VP9()->numberOfTemporalLayers = 2;
   // Tl0PidIdx is only used in non-flexible mode.
-  codec_settings_.VP9()->flexibleMode = false;
-  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
-            encoder_->InitEncode(&codec_settings_, kSettings));
+  codec_settings.VP9()->flexibleMode = false;
+  EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings),
+            WEBRTC_VIDEO_CODEC_OK);
 
-  // Temporal layer 0.
-  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
-  EncodedImage encoded_frame;
-  CodecSpecificInfo codec_specific_info;
-  ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
-  EXPECT_EQ(0, codec_specific_info.codecSpecific.VP9.temporal_idx);
+  std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
+      EncodedVideoFrameProducer(*encoder)
+          .SetNumInputFrames(4)
+          .SetResolution({kWidth, kHeight})
+          .Encode();
 
-  // Temporal layer 1.
-  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
-  ExpectFrameWith(1);
-
-  // Temporal layer 0.
-  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
-  ExpectFrameWith(0);
-
-  // Temporal layer 1.
-  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
-  ExpectFrameWith(1);
+  ASSERT_THAT(frames, SizeIs(4));
+  EXPECT_EQ(frames[0].codec_specific_info.codecSpecific.VP9.temporal_idx, 0);
+  EXPECT_EQ(frames[1].codec_specific_info.codecSpecific.VP9.temporal_idx, 1);
+  EXPECT_EQ(frames[2].codec_specific_info.codecSpecific.VP9.temporal_idx, 0);
+  EXPECT_EQ(frames[3].codec_specific_info.codecSpecific.VP9.temporal_idx, 1);
 }
 
-TEST_F(TestVp9Impl, EncoderWith2SpatialLayers) {
-  codec_settings_.VP9()->numberOfSpatialLayers = 2;
-  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
-            encoder_->InitEncode(&codec_settings_, kSettings));
+TEST(Vp9ImplTest, EncoderWith2SpatialLayers) {
+  std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
+  VideoCodec codec_settings = DefaultCodecSettings();
+  codec_settings.VP9()->numberOfSpatialLayers = 2;
+  EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings),
+            WEBRTC_VIDEO_CODEC_OK);
 
-  SetWaitForEncodedFramesThreshold(2);
-  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
-  std::vector<EncodedImage> encoded_frame;
-  std::vector<CodecSpecificInfo> codec_info;
-  ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_info));
-  EXPECT_EQ(encoded_frame[0].SpatialIndex(), 0);
-  EXPECT_EQ(encoded_frame[1].SpatialIndex(), 1);
+  std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
+      EncodedVideoFrameProducer(*encoder)
+          .SetNumInputFrames(1)
+          .SetResolution({kWidth, kHeight})
+          .Encode();
+
+  ASSERT_THAT(frames, SizeIs(2));
+  EXPECT_EQ(frames[0].encoded_image.SpatialIndex(), 0);
+  EXPECT_EQ(frames[1].encoded_image.SpatialIndex(), 1);
 }
 
 TEST_F(TestVp9Impl, EncoderExplicitLayering) {
@@ -1421,29 +1392,34 @@ TEST_F(TestVp9Impl, EncoderInfoFpsAllocationFlexibleMode) {
               ::testing::ElementsAreArray(expected_fps_allocation));
 }
 
-class TestVp9ImplWithLayering
-    : public TestVp9Impl,
-      public ::testing::WithParamInterface<::testing::tuple<uint8_t, uint8_t>> {
+class Vp9ImplWithLayeringTest
+    : public ::testing::TestWithParam<std::tuple<int, int, bool>> {
  protected:
-  TestVp9ImplWithLayering()
-      : num_spatial_layers_(::testing::get<0>(GetParam())),
-        num_temporal_layers_(::testing::get<1>(GetParam())) {}
+  Vp9ImplWithLayeringTest()
+      : num_spatial_layers_(std::get<0>(GetParam())),
+        num_temporal_layers_(std::get<1>(GetParam())),
+        override_field_trials_(std::get<2>(GetParam())
+                                   ? "WebRTC-Vp9ExternalRefCtrl/Enabled/"
+                                   : "") {}
 
   const uint8_t num_spatial_layers_;
   const uint8_t num_temporal_layers_;
+  const test::ScopedFieldTrials override_field_trials_;
 };
 
-TEST_P(TestVp9ImplWithLayering, FlexibleMode) {
+TEST_P(Vp9ImplWithLayeringTest, FlexibleMode) {
   // In flexible mode encoder wrapper obtains actual list of references from
   // encoder and writes it into RTP payload descriptor. Check that reference
   // list in payload descriptor matches the predefined one, which is used
   // in non-flexible mode.
-  codec_settings_.VP9()->flexibleMode = true;
-  codec_settings_.VP9()->frameDroppingOn = false;
-  codec_settings_.VP9()->numberOfSpatialLayers = num_spatial_layers_;
-  codec_settings_.VP9()->numberOfTemporalLayers = num_temporal_layers_;
-  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
-            encoder_->InitEncode(&codec_settings_, kSettings));
+  std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
+  VideoCodec codec_settings = DefaultCodecSettings();
+  codec_settings.VP9()->flexibleMode = true;
+  codec_settings.VP9()->frameDroppingOn = false;
+  codec_settings.VP9()->numberOfSpatialLayers = num_spatial_layers_;
+  codec_settings.VP9()->numberOfTemporalLayers = num_temporal_layers_;
+  EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings),
+            WEBRTC_VIDEO_CODEC_OK);
 
   GofInfoVP9 gof;
   if (num_temporal_layers_ == 1) {
@@ -1456,65 +1432,48 @@ TEST_P(TestVp9ImplWithLayering, FlexibleMode) {
 
   // Encode at least (num_frames_in_gof + 1) frames to verify references
   // of non-key frame with gof_idx = 0.
-  for (size_t frame_num = 0; frame_num < gof.num_frames_in_gof + 1;
-       ++frame_num) {
-    SetWaitForEncodedFramesThreshold(num_spatial_layers_);
-    EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
-              encoder_->Encode(NextInputFrame(), nullptr));
+  int num_input_frames = gof.num_frames_in_gof + 1;
+  std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
+      EncodedVideoFrameProducer(*encoder)
+          .SetNumInputFrames(num_input_frames)
+          .SetResolution({kWidth, kHeight})
+          .Encode();
+  ASSERT_THAT(frames, SizeIs(num_input_frames * num_spatial_layers_));
 
-    const bool is_key_frame = frame_num == 0;
-    const size_t gof_idx = frame_num % gof.num_frames_in_gof;
-    const std::vector<uint8_t> p_diff(std::begin(gof.pid_diff[gof_idx]),
-                                      std::end(gof.pid_diff[gof_idx]));
+  for (size_t i = 0; i < frames.size(); ++i) {
+    const EncodedVideoFrameProducer::EncodedFrame& frame = frames[i];
+    const size_t picture_idx = i / num_spatial_layers_;
+    const size_t gof_idx = picture_idx % gof.num_frames_in_gof;
 
-    ExpectFrameWith(num_spatial_layers_, gof.temporal_idx[gof_idx],
-                    gof.temporal_up_switch[gof_idx],
-                    is_key_frame ? 0 : gof.num_ref_pics[gof_idx], p_diff);
-  }
-}
-
-TEST_P(TestVp9ImplWithLayering, ExternalRefControl) {
-  test::ScopedFieldTrials override_field_trials(
-      "WebRTC-Vp9ExternalRefCtrl/Enabled/");
-  codec_settings_.VP9()->flexibleMode = true;
-  codec_settings_.VP9()->frameDroppingOn = false;
-  codec_settings_.VP9()->numberOfSpatialLayers = num_spatial_layers_;
-  codec_settings_.VP9()->numberOfTemporalLayers = num_temporal_layers_;
-  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
-            encoder_->InitEncode(&codec_settings_, kSettings));
-
-  GofInfoVP9 gof;
-  if (num_temporal_layers_ == 1) {
-    gof.SetGofInfoVP9(kTemporalStructureMode1);
-  } else if (num_temporal_layers_ == 2) {
-    gof.SetGofInfoVP9(kTemporalStructureMode2);
-  } else if (num_temporal_layers_ == 3) {
-    gof.SetGofInfoVP9(kTemporalStructureMode3);
-  }
-
-  // Encode at least (num_frames_in_gof + 1) frames to verify references
-  // of non-key frame with gof_idx = 0.
-  for (size_t frame_num = 0; frame_num < gof.num_frames_in_gof + 1;
-       ++frame_num) {
-    SetWaitForEncodedFramesThreshold(num_spatial_layers_);
-    EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
-              encoder_->Encode(NextInputFrame(), nullptr));
-
-    const bool is_key_frame = frame_num == 0;
-    const size_t gof_idx = frame_num % gof.num_frames_in_gof;
-    const std::vector<uint8_t> p_diff(std::begin(gof.pid_diff[gof_idx]),
-                                      std::end(gof.pid_diff[gof_idx]));
-
-    ExpectFrameWith(num_spatial_layers_, gof.temporal_idx[gof_idx],
-                    gof.temporal_up_switch[gof_idx],
-                    is_key_frame ? 0 : gof.num_ref_pics[gof_idx], p_diff);
+    const CodecSpecificInfoVP9& vp9 =
+        frame.codec_specific_info.codecSpecific.VP9;
+    EXPECT_EQ(frame.encoded_image.SpatialIndex(),
+              num_spatial_layers_ == 1
+                  ? absl::nullopt
+                  : absl::optional<int>(i % num_spatial_layers_))
+        << "Frame " << i;
+    EXPECT_EQ(vp9.temporal_idx, num_temporal_layers_ == 1
+                                    ? kNoTemporalIdx
+                                    : gof.temporal_idx[gof_idx])
+        << "Frame " << i;
+    EXPECT_EQ(vp9.temporal_up_switch, gof.temporal_up_switch[gof_idx])
+        << "Frame " << i;
+    if (picture_idx == 0) {
+      EXPECT_EQ(vp9.num_ref_pics, 0) << "Frame " << i;
+    } else {
+      EXPECT_THAT(rtc::MakeArrayView(vp9.p_diff, vp9.num_ref_pics),
+                  UnorderedElementsAreArray(gof.pid_diff[gof_idx],
+                                            gof.num_ref_pics[gof_idx]))
+          << "Frame " << i;
+    }
   }
 }
 
 INSTANTIATE_TEST_SUITE_P(All,
-                         TestVp9ImplWithLayering,
+                         Vp9ImplWithLayeringTest,
                          ::testing::Combine(::testing::Values(1, 2, 3),
-                                            ::testing::Values(1, 2, 3)));
+                                            ::testing::Values(1, 2, 3),
+                                            ::testing::Bool()));
 
 class TestVp9ImplFrameDropping : public TestVp9Impl {
  protected:
@@ -1774,4 +1733,12 @@ TEST_F(TestVp9Impl, ReenablingUpperLayerAfterKFWithInterlayerPredIsEnabled) {
   EXPECT_EQ(encoded_frames[0]._frameType, VideoFrameType::kVideoFrameDelta);
 }
 
+TEST_F(TestVp9Impl, HandlesEmptyInitDecode) {
+  std::unique_ptr<VideoDecoder> decoder = CreateDecoder();
+  // Check that nullptr settings are ok for decoder.
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
+            decoder->InitDecode(/*codec_settings=*/nullptr, 1));
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder->Release());
+}
+
 }  // namespace webrtc

modules/video_coding/codecs/vp9/vp9.cc

@@ -39,6 +39,22 @@ std::vector<SdpVideoFormat> SupportedVP9Codecs() {
         cricket::kVp9CodecName,
         {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile2)}}));
   }
+
+  return supported_formats;
+#else
+  return std::vector<SdpVideoFormat>();
+#endif
+}
+
+std::vector<SdpVideoFormat> SupportedVP9DecoderCodecs() {
+#ifdef RTC_ENABLE_VP9
+  std::vector<SdpVideoFormat> supported_formats = SupportedVP9Codecs();
+  // The WebRTC internal decoder supports VP9 profile 1. However, there's
+  // currently no way of sending VP9 profile 1 using the internal encoder.
+  // It would require extended support for I444, I422, and I440 buffers.
+  supported_formats.push_back(SdpVideoFormat(
+      cricket::kVp9CodecName,
+      {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile1)}}));
   return supported_formats;
 #else
   return std::vector<SdpVideoFormat>();

modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.cc

@@ -58,7 +58,7 @@ Vp9FrameBufferPool::GetFrameBuffer(size_t min_size) {
   RTC_DCHECK_GT(min_size, 0);
   rtc::scoped_refptr<Vp9FrameBuffer> available_buffer = nullptr;
   {
-    rtc::CritScope cs(&buffers_lock_);
+    MutexLock lock(&buffers_lock_);
     // Do we have a buffer we can recycle?
     for (const auto& buffer : allocated_buffers_) {
       if (buffer->HasOneRef()) {
@@ -91,7 +91,7 @@ Vp9FrameBufferPool::GetFrameBuffer(size_t min_size) {
 
 int Vp9FrameBufferPool::GetNumBuffersInUse() const {
   int num_buffers_in_use = 0;
-  rtc::CritScope cs(&buffers_lock_);
+  MutexLock lock(&buffers_lock_);
   for (const auto& buffer : allocated_buffers_) {
     if (!buffer->HasOneRef())
       ++num_buffers_in_use;
@@ -100,7 +100,7 @@ int Vp9FrameBufferPool::GetNumBuffersInUse() const {
 }
 
 bool Vp9FrameBufferPool::Resize(size_t max_number_of_buffers) {
-  rtc::CritScope cs(&buffers_lock_);
+  MutexLock lock(&buffers_lock_);
   size_t used_buffers_count = 0;
   for (const auto& buffer : allocated_buffers_) {
     // If the buffer is in use, the ref count will be >= 2, one from the list we
@@ -130,7 +130,7 @@ bool Vp9FrameBufferPool::Resize(size_t max_number_of_buffers) {
 }
 
 void Vp9FrameBufferPool::ClearPool() {
-  rtc::CritScope cs(&buffers_lock_);
+  MutexLock lock(&buffers_lock_);
   allocated_buffers_.clear();
 }
 

modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h

@@ -18,8 +18,8 @@
 
 #include "api/scoped_refptr.h"
 #include "rtc_base/buffer.h"
-#include "rtc_base/critical_section.h"
 #include "rtc_base/ref_count.h"
+#include "rtc_base/synchronization/mutex.h"
 
 struct vpx_codec_ctx;
 struct vpx_codec_frame_buffer;
@@ -119,7 +119,7 @@ class Vp9FrameBufferPool {
 
  private:
   // Protects |allocated_buffers_|.
-  rtc::CriticalSection buffers_lock_;
+  mutable Mutex buffers_lock_;
   // All buffers, in use or ready to be recycled.
   std::vector<rtc::scoped_refptr<Vp9FrameBuffer>> allocated_buffers_
       RTC_GUARDED_BY(buffers_lock_);

modules/video_coding/codecs/vp9/vp9_impl.cc

@@ -25,6 +25,7 @@
 #include "common_video/libyuv/include/webrtc_libyuv.h"
 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
 #include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
+#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/experiments/rate_control_settings.h"
 #include "rtc_base/keep_ref_until_done.h"
@@ -45,14 +46,21 @@ namespace {
 uint8_t kRefBufIdx[4] = {0, 0, 0, 1};
 uint8_t kUpdBufIdx[4] = {0, 0, 1, 0};
 
-int kMaxNumTiles4kVideo = 8;
-
 // Maximum allowed PID difference for differnet per-layer frame-rate case.
 const int kMaxAllowedPidDiff = 30;
 
 constexpr double kLowRateFactor = 1.0;
 constexpr double kHighRateFactor = 2.0;
 
+// TODO(ilink): Tune these thresholds further.
+// Selected using ConverenceMotion_1280_720_50.yuv clip.
+// No toggling observed on any link capacity from 100-2000kbps.
+// HD was reached consistently when link capacity was 1500kbps.
+// Set resolutions are a bit more conservative than svc_config.cc sets, e.g.
+// for 300kbps resolution converged to 270p instead of 360p.
+constexpr int kLowVp9QpThreshold = 149;
+constexpr int kHighVp9QpThreshold = 205;
+
 // These settings correspond to the settings in vpx_codec_enc_cfg.
 struct Vp9RateSettings {
   uint32_t rc_undershoot_pct;
@@ -249,6 +257,8 @@ VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec)
           "WebRTC-VP9VariableFramerateScreenshare")),
       variable_framerate_controller_(
           variable_framerate_experiment_.framerate_limit),
+      quality_scaler_experiment_(
+          ParseQualityScalerConfig("WebRTC-VP9QualityScaler")),
       num_steady_state_frames_(0),
       config_changed_(true) {
   codec_ = {};
@@ -399,7 +409,6 @@ bool VP9EncoderImpl::SetSvcRates(
       expect_no_more_active_layers = seen_active_layer;
     }
   }
-  RTC_DCHECK_GT(num_active_spatial_layers_, 0);
 
   if (higher_layers_enabled && !force_key_frame_) {
     // Prohibit drop of all layers for the next frame, so newly enabled
@@ -517,6 +526,11 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst,
       config_->g_profile = 0;
       config_->g_input_bit_depth = 8;
       break;
+    case VP9Profile::kProfile1:
+      // Encoding of profile 1 is not implemented. It would require extended
+      // support for I444, I422, and I440 buffers.
+      RTC_NOTREACHED();
+      break;
     case VP9Profile::kProfile2:
       img_fmt = VPX_IMG_FMT_I42016;
       bits_for_storage = 16;
@@ -562,7 +576,13 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst,
   // put some key-frames at will even in VPX_KF_DISABLED kf_mode.
   config_->kf_max_dist = inst->VP9().keyFrameInterval;
   config_->kf_min_dist = config_->kf_max_dist;
-  config_->rc_resize_allowed = inst->VP9().automaticResizeOn ? 1 : 0;
+  if (quality_scaler_experiment_.enabled) {
+    // In that experiment webrtc wide quality scaler is used instead of libvpx
+    // internal scaler.
+    config_->rc_resize_allowed = 0;
+  } else {
+    config_->rc_resize_allowed = inst->VP9().automaticResizeOn ? 1 : 0;
+  }
   // Determine number of threads based on the image size and #cores.
   config_->g_threads =
       NumberOfThreads(config_->g_w, config_->g_h, settings.number_of_cores);
@@ -587,13 +607,6 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst,
       (num_spatial_layers_ > 1 &&
        codec_.mode == VideoCodecMode::kScreensharing) ||
       inter_layer_pred_ == InterLayerPredMode::kOn;
-  // TODO(ilnik): Remove this workaround once external reference control works
-  // nicely with simulcast SVC mode.
-  // Simlucast SVC mode is currently only used in some tests and is impossible
-  // to trigger for users without using some field trials.
-  if (inter_layer_pred_ == InterLayerPredMode::kOff) {
-    external_ref_control_ = false;
-  }
 
   if (num_temporal_layers_ == 1) {
     gof_.SetGofInfoVP9(kTemporalStructureMode1);
@@ -987,6 +1000,10 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image,
       raw_->stride[VPX_PLANE_V] = i420_buffer->StrideV();
       break;
     }
+    case VP9Profile::kProfile1: {
+      RTC_NOTREACHED();
+      break;
+    }
     case VP9Profile::kProfile2: {
       // We can inject kI010 frames directly for encode. All other formats
       // should be converted to it.
@@ -1560,7 +1577,12 @@ VideoEncoder::EncoderInfo VP9EncoderImpl::GetEncoderInfo() const {
   EncoderInfo info;
   info.supports_native_handle = false;
   info.implementation_name = "libvpx";
-  info.scaling_settings = VideoEncoder::ScalingSettings::kOff;
+  if (quality_scaler_experiment_.enabled) {
+    info.scaling_settings = VideoEncoder::ScalingSettings(
+        quality_scaler_experiment_.low_qp, quality_scaler_experiment_.high_qp);
+  } else {
+    info.scaling_settings = VideoEncoder::ScalingSettings::kOff;
+  }
   info.has_trusted_rate_controller = trusted_rate_controller_;
   info.is_hardware_accelerated = false;
   info.has_internal_source = false;
@@ -1633,6 +1655,24 @@ VP9EncoderImpl::ParseVariableFramerateConfig(std::string group_name) {
   return config;
 }
 
+// static
+VP9EncoderImpl::QualityScalerExperiment
+VP9EncoderImpl::ParseQualityScalerConfig(std::string group_name) {
+  FieldTrialFlag disabled = FieldTrialFlag("Disabled");
+  FieldTrialParameter<int> low_qp("low_qp", kLowVp9QpThreshold);
+  FieldTrialParameter<int> high_qp("hihg_qp", kHighVp9QpThreshold);
+  ParseFieldTrial({&disabled, &low_qp, &high_qp},
+                  field_trial::FindFullName(group_name));
+  QualityScalerExperiment config;
+  config.enabled = !disabled.Get();
+  RTC_LOG(LS_INFO) << "Webrtc quality scaler for vp9 is "
+                   << (config.enabled ? "enabled." : "disabled");
+  config.low_qp = low_qp.Get();
+  config.high_qp = high_qp.Get();
+
+  return config;
+}
+
 VP9DecoderImpl::VP9DecoderImpl()
     : decode_complete_callback_(nullptr),
       inited_(false),
@@ -1672,14 +1712,32 @@ int VP9DecoderImpl::InitDecode(const VideoCodec* inst, int number_of_cores) {
   //    errors earlier than the multi-threads version.
   //  - Make peak CPU usage under control (not depending on input)
   cfg.threads = 1;
-  (void)kMaxNumTiles4kVideo;  // unused
 #else
-  // We want to use multithreading when decoding high resolution videos. But,
-  // since we don't know resolution of input stream at this stage, we always
-  // enable it.
-  cfg.threads = std::min(number_of_cores, kMaxNumTiles4kVideo);
+  if (!inst) {
+    // No config provided - don't know resolution to decode yet.
+    // Set thread count to one in the meantime.
+    cfg.threads = 1;
+  } else {
+    // We want to use multithreading when decoding high resolution videos. But
+    // not too many in order to avoid overhead when many stream are decoded
+    // concurrently.
+    // Set 2 thread as target for 1280x720 pixel count, and then scale up
+    // linearly from there - but cap at physical core count.
+    // For common resolutions this results in:
+    // 1 for 360p
+    // 2 for 720p
+    // 4 for 1080p
+    // 8 for 1440p
+    // 18 for 4K
+    int num_threads =
+        std::max(1, 2 * (inst->width * inst->height) / (1280 * 720));
+    cfg.threads = std::min(number_of_cores, num_threads);
+    current_codec_ = *inst;
+  }
 #endif
 
+  num_cores_ = number_of_cores;
+
   vpx_codec_flags_t flags = 0;
   if (vpx_codec_dec_init(decoder_, vpx_codec_vp9_dx(), &cfg, flags)) {
     return WEBRTC_VIDEO_CODEC_MEMORY;
@@ -1697,6 +1755,15 @@ int VP9DecoderImpl::InitDecode(const VideoCodec* inst, int number_of_cores) {
       return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
     }
   }
+
+  vpx_codec_err_t status =
+      vpx_codec_control(decoder_, VP9D_SET_LOOP_FILTER_OPT, 1);
+  if (status != VPX_CODEC_OK) {
+    RTC_LOG(LS_ERROR) << "Failed to enable VP9D_SET_LOOP_FILTER_OPT. "
+                      << vpx_codec_error(decoder_);
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  }
+
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
@@ -1709,6 +1776,29 @@ int VP9DecoderImpl::Decode(const EncodedImage& input_image,
   if (decode_complete_callback_ == nullptr) {
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
   }
+
+  if (input_image._frameType == VideoFrameType::kVideoFrameKey) {
+    absl::optional<vp9::FrameInfo> frame_info =
+        vp9::ParseIntraFrameInfo(input_image.data(), input_image.size());
+    if (frame_info) {
+      if (frame_info->frame_width != current_codec_.width ||
+          frame_info->frame_height != current_codec_.height) {
+        // Resolution has changed, tear down and re-init a new decoder in
+        // order to get correct sizing.
+        Release();
+        current_codec_.width = frame_info->frame_width;
+        current_codec_.height = frame_info->frame_height;
+        int reinit_status = InitDecode(&current_codec_, num_cores_);
+        if (reinit_status != WEBRTC_VIDEO_CODEC_OK) {
+          RTC_LOG(LS_WARNING) << "Failed to re-init decoder.";
+          return reinit_status;
+        }
+      }
+    } else {
+      RTC_LOG(LS_WARNING) << "Failed to parse VP9 header from key-frame.";
+    }
+  }
+
   // Always start with a complete key frame.
   if (key_frame_required_) {
     if (input_image._frameType != VideoFrameType::kVideoFrameKey)

modules/video_coding/codecs/vp9/vp9_impl.h

@@ -175,6 +175,15 @@ class VP9EncoderImpl : public VP9Encoder {
   static VariableFramerateExperiment ParseVariableFramerateConfig(
       std::string group_name);
   FramerateController variable_framerate_controller_;
+
+  const struct QualityScalerExperiment {
+    int low_qp;
+    int high_qp;
+    bool enabled;
+  } quality_scaler_experiment_;
+  static QualityScalerExperiment ParseQualityScalerConfig(
+      std::string group_name);
+
   int num_steady_state_frames_;
   // Only set config when this flag is set.
   bool config_changed_;
@@ -210,6 +219,8 @@ class VP9DecoderImpl : public VP9Decoder {
   bool inited_;
   vpx_codec_ctx_t* decoder_;
   bool key_frame_required_;
+  VideoCodec current_codec_;
+  int num_cores_;
 };
 }  // namespace webrtc