Add maxFramerate support to SimulcastEncoderAdapter

Bug: webrtc:11117
Change-Id: I134964e669804e1a3c5acb9b9c7ddb6c911cf610
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/162203
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Florent Castelli <orphis@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30107}

media/engine/simulcast_encoder_adapter.cc

@@ -318,15 +318,18 @@ int SimulcastEncoderAdapter::InitEncode(
       // Without simulcast, just pass through the encoder info from the one
       // active encoder.
       encoder->RegisterEncodeCompleteCallback(encoded_complete_callback_);
-      streaminfos_.emplace_back(std::move(encoder), nullptr, stream_codec.width,
-                                stream_codec.height, send_stream);
+      streaminfos_.emplace_back(
+          std::move(encoder), nullptr,
+          std::make_unique<FramerateController>(stream_codec.maxFramerate),
+          stream_codec.width, stream_codec.height, send_stream);
     } else {
       std::unique_ptr<EncodedImageCallback> callback(
           new AdapterEncodedImageCallback(this, i));
       encoder->RegisterEncodeCompleteCallback(callback.get());
-      streaminfos_.emplace_back(std::move(encoder), std::move(callback),
-                                stream_codec.width, stream_codec.height,
-                                send_stream);
+      streaminfos_.emplace_back(
+          std::move(encoder), std::move(callback),
+          std::make_unique<FramerateController>(stream_codec.maxFramerate),
+          stream_codec.width, stream_codec.height, send_stream);
     }
   }
 
@@ -377,13 +380,21 @@ int SimulcastEncoderAdapter::Encode(
       continue;
     }
 
+    const uint32_t frame_timestamp_ms =
+        1000 * input_image.timestamp() / 90000;  // kVideoPayloadTypeFrequency;
+
     std::vector<VideoFrameType> stream_frame_types;
     if (send_key_frame) {
       stream_frame_types.push_back(VideoFrameType::kVideoFrameKey);
       streaminfos_[stream_idx].key_frame_request = false;
     } else {
+      if (streaminfos_[stream_idx].framerate_controller->DropFrame(
+              frame_timestamp_ms)) {
+        continue;
+      }
       stream_frame_types.push_back(VideoFrameType::kVideoFrameDelta);
     }
+    streaminfos_[stream_idx].framerate_controller->AddFrame(frame_timestamp_ms);
 
     int dst_width = streaminfos_[stream_idx].width;
     int dst_height = streaminfos_[stream_idx].height;
@@ -504,6 +515,10 @@ void SimulcastEncoderAdapter::SetRates(
       }
     }
 
+    stream_parameters.framerate_fps = std::min<double>(
+        parameters.framerate_fps,
+        streaminfos_[stream_idx].framerate_controller->GetTargetRate());
+
     streaminfos_[stream_idx].encoder->SetRates(stream_parameters);
   }
 }
@@ -557,6 +572,7 @@ void SimulcastEncoderAdapter::PopulateStreamCodec(
   stream_codec->height = inst.simulcastStream[stream_index].height;
   stream_codec->maxBitrate = inst.simulcastStream[stream_index].maxBitrate;
   stream_codec->minBitrate = inst.simulcastStream[stream_index].minBitrate;
+  stream_codec->maxFramerate = inst.simulcastStream[stream_index].maxFramerate;
   stream_codec->qpMax = inst.simulcastStream[stream_index].qpMax;
   // Settings that are based on stream/resolution.
   if (stream_resolution == StreamResolution::LOWEST) {

media/engine/simulcast_encoder_adapter.h

@@ -23,6 +23,7 @@
 #include "api/video_codecs/sdp_video_format.h"
 #include "api/video_codecs/video_encoder.h"
 #include "modules/video_coding/include/video_codec_interface.h"
+#include "modules/video_coding/utility/framerate_controller.h"
 #include "rtc_base/atomic_ops.h"
 #include "rtc_base/synchronization/sequence_checker.h"
 #include "rtc_base/system/rtc_export.h"
@@ -78,17 +79,20 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
   struct StreamInfo {
     StreamInfo(std::unique_ptr<VideoEncoder> encoder,
                std::unique_ptr<EncodedImageCallback> callback,
+               std::unique_ptr<FramerateController> framerate_controller,
                uint16_t width,
                uint16_t height,
                bool send_stream)
         : encoder(std::move(encoder)),
           callback(std::move(callback)),
+          framerate_controller(std::move(framerate_controller)),
           width(width),
           height(height),
           key_frame_request(false),
           send_stream(send_stream) {}
     std::unique_ptr<VideoEncoder> encoder;
     std::unique_ptr<EncodedImageCallback> callback;
+    std::unique_ptr<FramerateController> framerate_controller;
     uint16_t width;
     uint16_t height;
     bool key_frame_request;

media/engine/simulcast_encoder_adapter_unittest.cc

@@ -1377,5 +1377,21 @@ TEST_F(TestSimulcastEncoderAdapterFake, SupportsFallback) {
   EXPECT_EQ(0, adapter_->Encode(input_frame, &frame_types));
 }
 
+TEST_F(TestSimulcastEncoderAdapterFake, SupportsPerSimulcastLayerMaxFramerate) {
+  SimulcastTestFixtureImpl::DefaultSettings(
+      &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
+      kVideoCodecVP8);
+  codec_.numberOfSimulcastStreams = 3;
+  codec_.simulcastStream[0].maxFramerate = 60;
+  codec_.simulcastStream[1].maxFramerate = 30;
+  codec_.simulcastStream[2].maxFramerate = 10;
+
+  EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
+  ASSERT_EQ(3u, helper_->factory()->encoders().size());
+  EXPECT_EQ(60u, helper_->factory()->encoders()[0]->codec().maxFramerate);
+  EXPECT_EQ(30u, helper_->factory()->encoders()[1]->codec().maxFramerate);
+  EXPECT_EQ(10u, helper_->factory()->encoders()[2]->codec().maxFramerate);
+}
+
 }  // namespace test
 }  // namespace webrtc