Restrict usage of resolution bitrate limits to singlecast

Bug: none
Change-Id: I4d0726d45a517b51eae124dc23e533910ede7cc7
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/203262
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33061}

video/adaptation/BUILD.gn

@@ -75,6 +75,7 @@ if (rtc_include_tests) {
 
     defines = []
     sources = [
+      "bitrate_constraint_unittest.cc",
       "overuse_frame_detector_unittest.cc",
       "pixel_limit_resource_unittest.cc",
       "quality_scaler_resource_unittest.cc",

video/adaptation/bitrate_constraint.cc

@@ -10,13 +10,32 @@
 
 #include <string>
 #include <utility>
+#include <vector>
 
 #include "call/adaptation/video_stream_adapter.h"
 #include "rtc_base/synchronization/sequence_checker.h"
 #include "video/adaptation/bitrate_constraint.h"
+#include "video/adaptation/video_stream_encoder_resource_manager.h"
 
 namespace webrtc {
 
+namespace {
+bool IsSimulcast(const VideoEncoderConfig& encoder_config) {
+  const std::vector<VideoStream>& simulcast_layers =
+      encoder_config.simulcast_layers;
+
+  bool is_simulcast = simulcast_layers.size() > 1;
+  bool is_lowest_layer_active = simulcast_layers[0].active;
+  int num_active_layers =
+      std::count_if(simulcast_layers.begin(), simulcast_layers.end(),
+                    [](const VideoStream& layer) { return layer.active; });
+
+  // We can't distinguish between simulcast and singlecast when only the
+  // lowest spatial layer is active. Treat this case as simulcast.
+  return is_simulcast && (num_active_layers > 1 || is_lowest_layer_active);
+}
+}  // namespace
+
 BitrateConstraint::BitrateConstraint()
     : encoder_settings_(absl::nullopt),
       encoder_target_bitrate_bps_(absl::nullopt) {
@@ -42,19 +61,35 @@ bool BitrateConstraint::IsAdaptationUpAllowed(
   RTC_DCHECK_RUN_ON(&sequence_checker_);
   // Make sure bitrate limits are not violated.
   if (DidIncreaseResolution(restrictions_before, restrictions_after)) {
+    if (!encoder_settings_.has_value()) {
+      return true;
+    }
+
     uint32_t bitrate_bps = encoder_target_bitrate_bps_.value_or(0);
+    if (bitrate_bps == 0) {
+      return true;
+    }
+
+    if (IsSimulcast(encoder_settings_->encoder_config())) {
+      // Resolution bitrate limits usage is restricted to singlecast.
+      return true;
+    }
+
+    absl::optional<uint32_t> current_frame_size_px =
+        VideoStreamEncoderResourceManager::GetSingleActiveLayerPixels(
+            encoder_settings_->video_codec());
+    if (!current_frame_size_px.has_value()) {
+      return true;
+    }
+
     absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
-        encoder_settings_.has_value()
-            ? encoder_settings_->encoder_info()
-                  .GetEncoderBitrateLimitsForResolution(
-                      // Need some sort of expected resulting pixels to be used
-                      // instead of unrestricted.
-                      GetHigherResolutionThan(
-                          input_state.frame_size_pixels().value()))
-            : absl::nullopt;
-    if (bitrate_limits.has_value() && bitrate_bps != 0) {
-      RTC_DCHECK_GE(bitrate_limits->frame_size_pixels,
-                    input_state.frame_size_pixels().value());
+        encoder_settings_->encoder_info().GetEncoderBitrateLimitsForResolution(
+            // Need some sort of expected resulting pixels to be used
+            // instead of unrestricted.
+            GetHigherResolutionThan(*current_frame_size_px));
+
+    if (bitrate_limits.has_value()) {
+      RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, *current_frame_size_px);
       return bitrate_bps >=
              static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps);
     }

video/adaptation/bitrate_constraint_unittest.cc

@@ -0,0 +1,223 @@
+/*
+ *  Copyright 2021 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "video/adaptation/bitrate_constraint.h"
+
+#include <limits>
+#include <utility>
+#include <vector>
+
+#include "api/video_codecs/video_encoder.h"
+#include "call/adaptation/encoder_settings.h"
+#include "call/adaptation/video_source_restrictions.h"
+#include "call/adaptation/video_stream_input_state.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+using ResolutionBitrateLimits = VideoEncoder::ResolutionBitrateLimits;
+
+namespace {
+
+void FillCodecConfig(VideoCodec* video_codec,
+                     VideoEncoderConfig* encoder_config,
+                     int width_px,
+                     int height_px,
+                     std::vector<bool> active_flags) {
+  size_t num_layers = active_flags.size();
+  video_codec->codecType = kVideoCodecVP8;
+  video_codec->numberOfSimulcastStreams = num_layers;
+
+  encoder_config->number_of_streams = num_layers;
+  encoder_config->simulcast_layers.resize(num_layers);
+
+  for (size_t layer_idx = 0; layer_idx < num_layers; ++layer_idx) {
+    int layer_width_px = width_px >> (num_layers - 1 - layer_idx);
+    int layer_height_px = height_px >> (num_layers - 1 - layer_idx);
+
+    video_codec->simulcastStream[layer_idx].active = active_flags[layer_idx];
+    video_codec->simulcastStream[layer_idx].width = layer_width_px;
+    video_codec->simulcastStream[layer_idx].height = layer_height_px;
+
+    encoder_config->simulcast_layers[layer_idx].active =
+        active_flags[layer_idx];
+    encoder_config->simulcast_layers[layer_idx].width = layer_width_px;
+    encoder_config->simulcast_layers[layer_idx].height = layer_height_px;
+  }
+}
+
+VideoEncoder::EncoderInfo MakeEncoderInfo() {
+  VideoEncoder::EncoderInfo encoder_info;
+  encoder_info.resolution_bitrate_limits = std::vector<ResolutionBitrateLimits>(
+      {ResolutionBitrateLimits(640 * 360, 500000, 0, 5000000),
+       ResolutionBitrateLimits(1280 * 720, 1000000, 0, 5000000),
+       ResolutionBitrateLimits(1920 * 1080, 2000000, 0, 5000000)});
+  return encoder_info;
+}
+}  // namespace
+
+TEST(BitrateConstraintTest, AdaptUpAllowedAtSinglecastIfBitrateIsEnough) {
+  VideoCodec video_codec;
+  VideoEncoderConfig encoder_config;
+  FillCodecConfig(&video_codec, &encoder_config,
+                  /*width_px=*/640, /*height_px=*/360,
+                  /*active_flags=*/{true});
+
+  EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config),
+                                   video_codec);
+
+  BitrateConstraint bitrate_constraint;
+  bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings);
+  bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000);
+
+  VideoSourceRestrictions restrictions_before(
+      /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360,
+      /*max_frame_rate=*/30);
+  VideoSourceRestrictions restrictions_after(
+      /*max_pixels_per_frame=*/1280 * 720,
+      /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30);
+
+  EXPECT_TRUE(bitrate_constraint.IsAdaptationUpAllowed(
+      VideoStreamInputState(), restrictions_before, restrictions_after));
+}
+
+TEST(BitrateConstraintTest, AdaptUpDisallowedAtSinglecastIfBitrateIsNotEnough) {
+  VideoCodec video_codec;
+  VideoEncoderConfig encoder_config;
+  FillCodecConfig(&video_codec, &encoder_config,
+                  /*width_px=*/640, /*height_px=*/360,
+                  /*active_flags=*/{true});
+
+  EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config),
+                                   video_codec);
+
+  BitrateConstraint bitrate_constraint;
+  bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings);
+  // 1 bps less than needed for 720p.
+  bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000 - 1);
+
+  VideoSourceRestrictions restrictions_before(
+      /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360,
+      /*max_frame_rate=*/30);
+  VideoSourceRestrictions restrictions_after(
+      /*max_pixels_per_frame=*/1280 * 720,
+      /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30);
+
+  EXPECT_FALSE(bitrate_constraint.IsAdaptationUpAllowed(
+      VideoStreamInputState(), restrictions_before, restrictions_after));
+}
+
+TEST(BitrateConstraintTest,
+     AdaptUpAllowedAtSinglecastUpperLayerActiveIfBitrateIsEnough) {
+  VideoCodec video_codec;
+  VideoEncoderConfig encoder_config;
+  FillCodecConfig(&video_codec, &encoder_config,
+                  /*width_px=*/640, /*height_px=*/360,
+                  /*active_flags=*/{false, true});
+
+  EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config),
+                                   video_codec);
+
+  BitrateConstraint bitrate_constraint;
+  bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings);
+  bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000);
+
+  VideoSourceRestrictions restrictions_before(
+      /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360,
+      /*max_frame_rate=*/30);
+  VideoSourceRestrictions restrictions_after(
+      /*max_pixels_per_frame=*/1280 * 720,
+      /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30);
+
+  EXPECT_TRUE(bitrate_constraint.IsAdaptationUpAllowed(
+      VideoStreamInputState(), restrictions_before, restrictions_after));
+}
+
+TEST(BitrateConstraintTest,
+     AdaptUpDisallowedAtSinglecastUpperLayerActiveIfBitrateIsNotEnough) {
+  VideoCodec video_codec;
+  VideoEncoderConfig encoder_config;
+  FillCodecConfig(&video_codec, &encoder_config,
+                  /*width_px=*/640, /*height_px=*/360,
+                  /*active_flags=*/{false, true});
+
+  EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config),
+                                   video_codec);
+
+  BitrateConstraint bitrate_constraint;
+  bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings);
+  // 1 bps less than needed for 720p.
+  bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000 - 1);
+
+  VideoSourceRestrictions restrictions_before(
+      /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360,
+      /*max_frame_rate=*/30);
+  VideoSourceRestrictions restrictions_after(
+      /*max_pixels_per_frame=*/1280 * 720,
+      /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30);
+
+  EXPECT_FALSE(bitrate_constraint.IsAdaptationUpAllowed(
+      VideoStreamInputState(), restrictions_before, restrictions_after));
+}
+
+TEST(BitrateConstraintTest,
+     AdaptUpAllowedAtSinglecastLowestLayerActiveIfBitrateIsNotEnough) {
+  VideoCodec video_codec;
+  VideoEncoderConfig encoder_config;
+  FillCodecConfig(&video_codec, &encoder_config,
+                  /*width_px=*/640, /*height_px=*/360,
+                  /*active_flags=*/{true, false});
+
+  EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config),
+                                   video_codec);
+
+  BitrateConstraint bitrate_constraint;
+  bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings);
+  // 1 bps less than needed for 720p.
+  bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000 - 1);
+
+  VideoSourceRestrictions restrictions_before(
+      /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360,
+      /*max_frame_rate=*/30);
+  VideoSourceRestrictions restrictions_after(
+      /*max_pixels_per_frame=*/1280 * 720,
+      /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30);
+
+  EXPECT_TRUE(bitrate_constraint.IsAdaptationUpAllowed(
+      VideoStreamInputState(), restrictions_before, restrictions_after));
+}
+
+TEST(BitrateConstraintTest, AdaptUpAllowedAtSimulcastIfBitrateIsNotEnough) {
+  VideoCodec video_codec;
+  VideoEncoderConfig encoder_config;
+  FillCodecConfig(&video_codec, &encoder_config,
+                  /*width_px=*/640, /*height_px=*/360,
+                  /*active_flags=*/{true, true});
+
+  EncoderSettings encoder_settings(MakeEncoderInfo(), std::move(encoder_config),
+                                   video_codec);
+
+  BitrateConstraint bitrate_constraint;
+  bitrate_constraint.OnEncoderSettingsUpdated(encoder_settings);
+  // 1 bps less than needed for 720p.
+  bitrate_constraint.OnEncoderTargetBitrateUpdated(1000 * 1000 - 1);
+
+  VideoSourceRestrictions restrictions_before(
+      /*max_pixels_per_frame=*/640 * 360, /*target_pixels_per_frame=*/640 * 360,
+      /*max_frame_rate=*/30);
+  VideoSourceRestrictions restrictions_after(
+      /*max_pixels_per_frame=*/1280 * 720,
+      /*target_pixels_per_frame=*/1280 * 720, /*max_frame_rate=*/30);
+
+  EXPECT_TRUE(bitrate_constraint.IsAdaptationUpAllowed(
+      VideoStreamInputState(), restrictions_before, restrictions_after));
+}
+
+}  // namespace webrtc

video/adaptation/video_stream_encoder_resource_manager.cc

@@ -64,30 +64,6 @@ std::string ToString(VideoAdaptationReason reason) {
   RTC_CHECK_NOTREACHED();
 }
 
-absl::optional<uint32_t> GetSingleActiveStreamPixels(const VideoCodec& codec) {
-  int num_active = 0;
-  absl::optional<uint32_t> pixels;
-  if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
-    for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) {
-      if (codec.spatialLayers[i].active) {
-        ++num_active;
-        pixels = codec.spatialLayers[i].width * codec.spatialLayers[i].height;
-      }
-    }
-  } else {
-    for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
-      if (codec.simulcastStream[i].active) {
-        ++num_active;
-        pixels =
-            codec.simulcastStream[i].width * codec.simulcastStream[i].height;
-      }
-    }
-  }
-  if (num_active > 1)
-    return absl::nullopt;
-  return pixels;
-}
-
 std::vector<bool> GetActiveLayersFlags(const VideoCodec& codec) {
   std::vector<bool> flags;
   if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
@@ -188,7 +164,7 @@ class VideoStreamEncoderResourceManager::InitialFrameDropper {
     last_active_flags_ = active_flags;
     last_input_width_ = codec.width;
     last_input_height_ = codec.height;
-    single_active_stream_pixels_ = GetSingleActiveStreamPixels(codec);
+    single_active_stream_pixels_ = GetSingleActiveLayerPixels(codec);
   }
 
   void OnFrameDroppedDueToSize() { ++initial_framedrop_; }
@@ -703,4 +679,31 @@ void VideoStreamEncoderResourceManager::OnQualityRampUp() {
   stream_adapter_->ClearRestrictions();
   quality_rampup_experiment_.reset();
 }
+
+absl::optional<uint32_t>
+VideoStreamEncoderResourceManager::GetSingleActiveLayerPixels(
+    const VideoCodec& codec) {
+  int num_active = 0;
+  absl::optional<uint32_t> pixels;
+  if (codec.codecType == VideoCodecType::kVideoCodecVP9) {
+    for (int i = 0; i < codec.VP9().numberOfSpatialLayers; ++i) {
+      if (codec.spatialLayers[i].active) {
+        ++num_active;
+        pixels = codec.spatialLayers[i].width * codec.spatialLayers[i].height;
+      }
+    }
+  } else {
+    for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
+      if (codec.simulcastStream[i].active) {
+        ++num_active;
+        pixels =
+            codec.simulcastStream[i].width * codec.simulcastStream[i].height;
+      }
+    }
+  }
+  if (num_active > 1)
+    return absl::nullopt;
+  return pixels;
+}
+
 }  // namespace webrtc

video/adaptation/video_stream_encoder_resource_manager.h

@@ -146,6 +146,9 @@ class VideoStreamEncoderResourceManager
   // QualityRampUpExperimentListener implementation.
   void OnQualityRampUp() override;
 
+  static absl::optional<uint32_t> GetSingleActiveLayerPixels(
+      const VideoCodec& codec);
+
  private:
   class InitialFrameDropper;