VideoStreamEncoder report VideoLayersAllocation for simulcast

Adds support for Vp8 simulcast.

Bug: webrtc:12000
Change-Id: Ib24fd0542642b023ec35f7a7bdc4880d72365edf
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/187341
Commit-Queue: Per Kjellander <perkj@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32416}

api/video/BUILD.gn

@@ -21,7 +21,6 @@ rtc_library("video_rtp_headers") {
     "hdr_metadata.h",
     "video_content_type.cc",
     "video_content_type.h",
-    "video_layers_allocation.h",
     "video_rotation.h",
     "video_timing.cc",
     "video_timing.h",
@@ -182,6 +181,13 @@ rtc_library("video_bitrate_allocation") {
   absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
 }
 
+rtc_source_set("video_layers_allocation") {
+  visibility = [ "*" ]
+  sources = [ "video_layers_allocation.h" ]
+  deps = [ "../units:data_rate" ]
+  absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ]
+}
+
 rtc_library("video_bitrate_allocator") {
   visibility = [ "*" ]
   sources = [
@@ -264,6 +270,7 @@ rtc_source_set("video_stream_encoder") {
     ":video_bitrate_allocator_factory",
     ":video_codec_constants",
     ":video_frame",
+    ":video_layers_allocation",
     "..:rtp_parameters",
     "..:scoped_refptr",
     "../:fec_controller_api",

api/video/video_stream_encoder_interface.h

@@ -19,6 +19,7 @@
 #include "api/scoped_refptr.h"
 #include "api/units/data_rate.h"
 #include "api/video/video_bitrate_allocator.h"
+#include "api/video/video_layers_allocation.h"
 #include "api/video/video_sink_interface.h"
 #include "api/video/video_source_interface.h"
 #include "api/video_codecs/video_encoder.h"
@@ -52,6 +53,9 @@ class VideoStreamEncoderInterface : public rtc::VideoSinkInterface<VideoFrame> {
 
     virtual void OnBitrateAllocationUpdated(
         const VideoBitrateAllocation& allocation) = 0;
+
+    virtual void OnVideoLayersAllocationUpdated(
+        VideoLayersAllocation allocation) = 0;
   };
 
   // If the resource is overusing, the VideoStreamEncoder will try to reduce

modules/rtp_rtcp/BUILD.gn

@@ -110,6 +110,7 @@ rtc_library("rtp_rtcp_format") {
     "../../api/transport/rtp:dependency_descriptor",
     "../../api/units:time_delta",
     "../../api/video:video_frame",
+    "../../api/video:video_layers_allocation",
     "../../api/video:video_rtp_headers",
     "../../common_video",
     "../../rtc_base:checks",
@@ -278,6 +279,7 @@ rtc_library("rtp_rtcp") {
     "../../api/video:video_codec_constants",
     "../../api/video:video_frame",
     "../../api/video:video_frame_type",
+    "../../api/video:video_layers_allocation",
     "../../api/video:video_rtp_headers",
     "../../api/video_codecs:video_codecs_api",
     "../../call:rtp_interfaces",
@@ -537,6 +539,7 @@ if (rtc_include_tests) {
       "../../api/video:video_bitrate_allocator",
       "../../api/video:video_codec_constants",
       "../../api/video:video_frame",
+      "../../api/video:video_layers_allocation",
       "../../api/video:video_rtp_headers",
       "../../api/video_codecs:video_codecs_api",
       "../../call:rtp_receiver",

video/BUILD.gn

@@ -226,6 +226,7 @@ rtc_library("video_stream_encoder_impl") {
     "../api/video:video_bitrate_allocator_factory",
     "../api/video:video_codec_constants",
     "../api/video:video_frame",
+    "../api/video:video_layers_allocation",
     "../api/video:video_rtp_headers",
     "../api/video:video_stream_encoder",
     "../api/video_codecs:video_codecs_api",

video/video_send_stream_impl.cc

@@ -472,6 +472,20 @@ void VideoSendStreamImpl::OnBitrateAllocationUpdated(
   }
 }
 
+void VideoSendStreamImpl::OnVideoLayersAllocationUpdated(
+    VideoLayersAllocation allocation) {
+  if (!worker_queue_->IsCurrent()) {
+    auto ptr = weak_ptr_;
+    worker_queue_->PostTask([allocation = std::move(allocation), ptr] {
+      if (!ptr.get())
+        return;
+      ptr->OnVideoLayersAllocationUpdated(allocation);
+    });
+    return;
+  }
+  // TODO(bugs.webrtc.org/12000): Implement
+}
+
 void VideoSendStreamImpl::SignalEncoderActive() {
   RTC_DCHECK_RUN_ON(worker_queue_);
   if (rtp_video_sender_->IsActive()) {

video/video_send_stream_impl.h

@@ -121,6 +121,8 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
 
   void OnBitrateAllocationUpdated(
       const VideoBitrateAllocation& allocation) override;
+  void OnVideoLayersAllocationUpdated(
+      VideoLayersAllocation allocation) override;
 
   // Implements EncodedImageCallback. The implementation routes encoded frames
   // to the |payload_router_| and |config.pre_encode_callback| if set.

video/video_stream_encoder.cc

@@ -198,6 +198,73 @@ VideoBitrateAllocation UpdateAllocationFromEncoderInfo(
   return new_allocation;
 }
 
+// Converts a VideoBitrateAllocation that contains allocated bitrate per layer,
+// and an EncoderInfo that contains information about the actual encoder
+// structure used by a codec. Stream structures can be Ksvc, Full SVC, Simulcast
+// etc.
+VideoLayersAllocation CreateVideoLayersAllocation(
+    const VideoCodec& encoder_config,
+    const VideoEncoder::RateControlParameters& current_rate,
+    const VideoEncoder::EncoderInfo& encoder_info) {
+  const VideoBitrateAllocation& target_bitrate = current_rate.target_bitrate;
+  VideoLayersAllocation layers_allocation;
+  if (target_bitrate.get_sum_bps() == 0) {
+    return layers_allocation;
+  }
+
+  if (encoder_config.numberOfSimulcastStreams > 0) {
+    layers_allocation.resolution_and_frame_rate_is_valid = true;
+    for (int si = 0; si < encoder_config.numberOfSimulcastStreams; ++si) {
+      if (!target_bitrate.IsSpatialLayerUsed(si) ||
+          target_bitrate.GetSpatialLayerSum(si) == 0) {
+        break;
+      }
+      layers_allocation.active_spatial_layers.emplace_back();
+      VideoLayersAllocation::SpatialLayer& spatial_layer =
+          layers_allocation.active_spatial_layers.back();
+      spatial_layer.width = encoder_config.simulcastStream[si].width;
+      spatial_layer.height = encoder_config.simulcastStream[si].height;
+      spatial_layer.rtp_stream_index = si;
+      spatial_layer.spatial_id = 0;
+      auto frame_rate_fraction =
+          VideoEncoder::EncoderInfo::kMaxFramerateFraction;
+      if (encoder_info.fps_allocation[si].size() == 1) {
+        // One TL is signalled to be used by the encoder. Do not distribute
+        // bitrate allocation across TLs (use sum at tl:0).
+        spatial_layer.target_bitrate_per_temporal_layer.push_back(
+            DataRate::BitsPerSec(target_bitrate.GetSpatialLayerSum(si)));
+        frame_rate_fraction = encoder_info.fps_allocation[si][0];
+      } else {  // Temporal layers are supported.
+        uint32_t temporal_layer_bitrate_bps = 0;
+        for (size_t ti = 0;
+             ti < encoder_config.simulcastStream[si].numberOfTemporalLayers;
+             ++ti) {
+          if (!target_bitrate.HasBitrate(si, ti)) {
+            break;
+          }
+          if (ti < encoder_info.fps_allocation[si].size()) {
+            // Use frame rate of the top used temporal layer.
+            frame_rate_fraction = encoder_info.fps_allocation[si][ti];
+          }
+          temporal_layer_bitrate_bps += target_bitrate.GetBitrate(si, ti);
+          spatial_layer.target_bitrate_per_temporal_layer.push_back(
+              DataRate::BitsPerSec(temporal_layer_bitrate_bps));
+        }
+      }
+      // Encoder may drop frames internally if `maxFramerate` is set.
+      spatial_layer.frame_rate_fps = std::min(
+          static_cast<uint8_t>(encoder_config.simulcastStream[si].maxFramerate),
+          static_cast<uint8_t>(
+              (current_rate.framerate_fps * frame_rate_fraction) /
+              VideoEncoder::EncoderInfo::kMaxFramerateFraction));
+    }
+  } else {
+    // TODO(bugs.webrtc.org/12000): Implement support for kSVC and full SVC.
+  }
+
+  return layers_allocation;
+}
+
 }  //  namespace
 
 VideoStreamEncoder::EncoderRateSettings::EncoderRateSettings()
@@ -1124,6 +1191,12 @@ void VideoStreamEncoder::SetEncoderRates(
         rate_settings.rate_control.bitrate,
         static_cast<uint32_t>(rate_settings.rate_control.framerate_fps + 0.5));
     stream_resource_manager_.SetEncoderRates(rate_settings.rate_control);
+    if (settings_.allocation_cb_type ==
+        VideoStreamEncoderSettings::BitrateAllocationCallbackType::
+            kVideoLayersAllocation) {
+      sink_->OnVideoLayersAllocationUpdated(CreateVideoLayersAllocation(
+          send_codec_, rate_settings.rate_control, encoder_->GetEncoderInfo()));
+    }
   }
   if ((settings_.allocation_cb_type ==
        VideoStreamEncoderSettings::BitrateAllocationCallbackType::

video/video_stream_encoder_unittest.cc

@@ -1254,6 +1254,16 @@ class VideoStreamEncoderTest : public ::testing::Test {
       return number_of_bitrate_allocations_;
     }
 
+    VideoLayersAllocation GetLastVideoLayersAllocation() {
+      MutexLock lock(&mutex_);
+      return last_layers_allocation_;
+    }
+
+    int number_of_layers_allocations() const {
+      MutexLock lock(&mutex_);
+      return number_of_layers_allocations_;
+    }
+
    private:
     Result OnEncodedImage(
         const EncodedImage& encoded_image,
@@ -1296,6 +1306,24 @@ class VideoStreamEncoderTest : public ::testing::Test {
       last_bitrate_allocation_ = allocation;
     }
 
+    void OnVideoLayersAllocationUpdated(
+        VideoLayersAllocation allocation) override {
+      MutexLock lock(&mutex_);
+      ++number_of_layers_allocations_;
+      last_layers_allocation_ = allocation;
+      rtc::StringBuilder log;
+      for (const auto& layer : allocation.active_spatial_layers) {
+        log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
+            << "[";
+        for (const auto target_bitrate :
+             layer.target_bitrate_per_temporal_layer) {
+          log << target_bitrate.kbps() << ",";
+        }
+        log << "]";
+      }
+      RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
+    }
+
     TimeController* const time_controller_;
     mutable Mutex mutex_;
     TestEncoder* test_encoder_;
@@ -1313,6 +1341,8 @@ class VideoStreamEncoderTest : public ::testing::Test {
     int min_transmit_bitrate_bps_ = 0;
     VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
     int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
+    VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
+    int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
   };
 
   class VideoBitrateAllocatorProxyFactory
@@ -4013,6 +4043,100 @@ TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
   video_stream_encoder_->Stop();
 }
 
+TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForV8Simulcast) {
+  ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
+               VideoStreamEncoderSettings::BitrateAllocationCallbackType::
+                   kVideoLayersAllocation);
+
+  const int kDefaultFps = 30;
+
+  video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+      DataRate::BitsPerSec(kLowTargetBitrateBps),
+      DataRate::BitsPerSec(kLowTargetBitrateBps),
+      DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
+
+  video_source_.IncomingCapturedFrame(
+      CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
+  WaitForEncodedFrame(CurrentTimeMs());
+  EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
+  VideoLayersAllocation last_layer_allocation =
+      sink_.GetLastVideoLayersAllocation();
+  // kLowTargetBitrateBps is only enough for one spatial layer.
+  ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
+
+  VideoBitrateAllocation bitrate_allocation =
+      fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
+  // Check that encoder has been updated too, not just allocation observer.
+  EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
+  AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
+
+  // VideoLayersAllocation might be updated if frame rate change.
+  int number_of_layers_allocation = 1;
+  const int64_t start_time_ms = CurrentTimeMs();
+  while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
+    video_source_.IncomingCapturedFrame(
+        CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
+    WaitForEncodedFrame(CurrentTimeMs());
+    AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
+    if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
+      number_of_layers_allocation = sink_.number_of_layers_allocations();
+      VideoLayersAllocation new_allocation =
+          sink_.GetLastVideoLayersAllocation();
+      ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
+      EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
+                last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
+      EXPECT_EQ(new_allocation.active_spatial_layers[0]
+                    .target_bitrate_per_temporal_layer,
+                last_layer_allocation.active_spatial_layers[0]
+                    .target_bitrate_per_temporal_layer);
+      last_layer_allocation = new_allocation;
+    }
+  }
+  EXPECT_LE(sink_.number_of_layers_allocations(), 3);
+  video_stream_encoder_->Stop();
+}
+
+TEST_F(VideoStreamEncoderTest,
+       ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
+  ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
+               VideoStreamEncoderSettings::BitrateAllocationCallbackType::
+                   kVideoLayersAllocation);
+
+  video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+      DataRate::BitsPerSec(kLowTargetBitrateBps),
+      DataRate::BitsPerSec(kLowTargetBitrateBps),
+      DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
+
+  video_source_.IncomingCapturedFrame(
+      CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
+  WaitForEncodedFrame(CurrentTimeMs());
+  EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
+  VideoLayersAllocation last_layer_allocation =
+      sink_.GetLastVideoLayersAllocation();
+  // kLowTargetBitrateBps is only enough for one spatial layer.
+  ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
+  EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
+                .target_bitrate_per_temporal_layer[0],
+            DataRate::BitsPerSec(kLowTargetBitrateBps));
+
+  video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
+      DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
+      DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
+      DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
+  video_source_.IncomingCapturedFrame(
+      CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
+  WaitForEncodedFrame(CurrentTimeMs());
+
+  EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
+  last_layer_allocation = sink_.GetLastVideoLayersAllocation();
+  ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
+  EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
+                .target_bitrate_per_temporal_layer[0],
+            DataRate::Zero());
+
+  video_stream_encoder_->Stop();
+}
+
 TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
   // 2 TLs configured, temporal layers supported by encoder.
   const int kNumTemporalLayers = 2;