Reland of Add experimental simulcast screen content mode

The original CL was reverted because of a bug discovered by the
chromium bots. Description of that CL:

> Review-Url: https://codereview.webrtc.org/2636443002
> Cr-Commit-Position: refs/heads/master@{#16135}
> Committed: a28e971e3b

The first patch set of this CL is the same as r16135.
Subsequence patch sets are the fixes applied.
Some new test cases have been added, which reveal a few more bugs that
have also been fixed.

BUG=webrtc:4172

Review-Url: https://codereview.webrtc.org/2641133002
Cr-Commit-Position: refs/heads/master@{#16299}

webrtc/modules/video_coding/BUILD.gn

@@ -371,6 +371,7 @@ if (rtc_include_tests) {
       "utility/moving_average_unittest.cc",
       "utility/quality_scaler_unittest.cc",
       "utility/simulcast_rate_allocator_unittest.cc",
+      "video_codec_initializer_unittest.cc",
       "video_coding_robustness_unittest.cc",
       "video_packet_buffer_unittest.cc",
       "video_receiver_unittest.cc",

webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc

@@ -94,11 +94,11 @@ class RealTimeTemporalLayers : public TemporalLayers {
         timestamp_(0),
         last_base_layer_sync_(0),
         layer_ids_length_(0),
-        layer_ids_(NULL),
+        layer_ids_(nullptr),
         encode_flags_length_(0),
-        encode_flags_(NULL) {
+        encode_flags_(nullptr) {
     RTC_CHECK_GE(max_temporal_layers_, 1);
-    RTC_CHECK_GE(max_temporal_layers_, 3);
+    RTC_CHECK_LE(max_temporal_layers_, 3);
   }
 
   virtual ~RealTimeTemporalLayers() {}

webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc

@@ -30,6 +30,8 @@ static const int kQpDeltaThresholdForSync = 8;
 const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
 const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
 
+constexpr int ScreenshareLayers::kMaxNumTemporalLayers;
+
 // Since this is TL0 we only allow updating and predicting from the LAST
 // reference frame.
 const int ScreenshareLayers::kTl0Flags =
@@ -55,8 +57,14 @@ webrtc::TemporalLayers* ScreenshareTemporalLayersFactory::Create(
     int simulcast_id,
     int num_temporal_layers,
     uint8_t initial_tl0_pic_idx) const {
-  webrtc::TemporalLayers* tl = new webrtc::ScreenshareLayers(
-      num_temporal_layers, rand(), webrtc::Clock::GetRealTimeClock());
+  webrtc::TemporalLayers* tl;
+  if (simulcast_id == 0) {
+    tl = new webrtc::ScreenshareLayers(num_temporal_layers, rand(),
+                                       webrtc::Clock::GetRealTimeClock());
+  } else {
+    RealTimeTemporalLayersFactory rt_tl_factory;
+    tl = rt_tl_factory.Create(simulcast_id, num_temporal_layers, rand());
+  }
   if (listener_)
     listener_->OnTemporalLayersCreated(simulcast_id, tl);
   return tl;
@@ -66,7 +74,8 @@ ScreenshareLayers::ScreenshareLayers(int num_temporal_layers,
                                      uint8_t initial_tl0_pic_idx,
                                      Clock* clock)
     : clock_(clock),
-      number_of_temporal_layers_(num_temporal_layers),
+      number_of_temporal_layers_(
+          std::min(kMaxNumTemporalLayers, num_temporal_layers)),
       last_base_layer_sync_(false),
       tl0_pic_idx_(initial_tl0_pic_idx),
       active_layer_(-1),
@@ -78,8 +87,8 @@ ScreenshareLayers::ScreenshareLayers(int num_temporal_layers,
       max_debt_bytes_(0),
       encode_framerate_(1000.0f, 1000.0f),  // 1 second window, second scale.
       bitrate_updated_(false) {
-  RTC_CHECK_GT(num_temporal_layers, 0);
-  RTC_CHECK_LE(num_temporal_layers, 2);
+  RTC_CHECK_GT(number_of_temporal_layers_, 0);
+  RTC_CHECK_LE(number_of_temporal_layers_, kMaxNumTemporalLayers);
 }
 
 ScreenshareLayers::~ScreenshareLayers() {

webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h

@@ -84,7 +84,7 @@ class ScreenshareLayers : public TemporalLayers {
   RateStatistics encode_framerate_;
   bool bitrate_updated_;
 
-  static const int kMaxNumTemporalLayers = 2;
+  static constexpr int kMaxNumTemporalLayers = 2;
   struct TemporalLayer {
     TemporalLayer()
         : state(State::kNormal),

webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc

@@ -106,23 +106,24 @@ BitrateAllocation SimulcastRateAllocator::GetAllocation(
     const int num_temporal_streams = std::max<uint8_t>(
         1, codec_.numberOfSimulcastStreams == 0
                ? codec_.VP8().numberOfTemporalLayers
-               : codec_.simulcastStream[0].numberOfTemporalLayers);
+               : codec_.simulcastStream[simulcast_id].numberOfTemporalLayers);
 
     uint32_t max_bitrate_kbps;
-    if (num_spatial_streams == 1) {
-      max_bitrate_kbps = codec_.maxBitrate;
-
-      // TODO(holmer): This is a temporary hack for screensharing, where we
+    // Legacy temporal-layered only screenshare, or simulcast screenshare
+    // with legacy mode for simulcast stream 0.
+    if (codec_.mode == kScreensharing && codec_.targetBitrate > 0 &&
+        ((num_spatial_streams == 1 && num_temporal_streams == 2) ||  // Legacy.
+         (num_spatial_streams > 1 && simulcast_id == 0))) {  // Simulcast.
+      // TODO(holmer): This is a "temporary" hack for screensharing, where we
       // interpret the startBitrate as the encoder target bitrate. This is
       // to allow for a different max bitrate, so if the codec can't meet
       // the target we still allow it to overshoot up to the max before dropping
       // frames. This hack should be improved.
-      if (codec_.mode == kScreensharing && codec_.targetBitrate > 0 &&
-          num_temporal_streams == 2) {
-        int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate_kbps);
-        max_bitrate_kbps = std::min(codec_.maxBitrate, target_bitrate_kbps);
-        target_bitrate_kbps = tl0_bitrate;
-      }
+      int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate_kbps);
+      max_bitrate_kbps = std::min(codec_.maxBitrate, target_bitrate_kbps);
+      target_bitrate_kbps = tl0_bitrate;
+    } else if (num_spatial_streams == 1) {
+      max_bitrate_kbps = codec_.maxBitrate;
     } else {
       max_bitrate_kbps = codec_.simulcastStream[simulcast_id].maxBitrate;
     }

webrtc/modules/video_coding/video_codec_initializer.cc

@@ -38,8 +38,9 @@ bool VideoCodecInitializer::SetupCodec(
     case kVideoCodecVP8: {
       if (!codec->VP8()->tl_factory) {
         if (codec->mode == kScreensharing &&
-            codec->numberOfSimulcastStreams == 1 &&
-            codec->VP8()->numberOfTemporalLayers == 2) {
+            (codec->numberOfSimulcastStreams > 1 ||
+             (codec->numberOfSimulcastStreams == 1 &&
+              codec->VP8()->numberOfTemporalLayers == 2))) {
           // Conference mode temporal layering for screen content.
           tl_factory.reset(new ScreenshareTemporalLayersFactory());
         } else {
@@ -102,7 +103,7 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec(
       break;
     case VideoEncoderConfig::ContentType::kScreen:
       video_codec.mode = kScreensharing;
-      if (streams.size() == 1 &&
+      if (streams.size() >= 1 &&
           streams[0].temporal_layer_thresholds_bps.size() == 1) {
         video_codec.targetBitrate =
             streams[0].temporal_layer_thresholds_bps[0] / 1000;
@@ -180,8 +181,12 @@ VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec(
     RTC_DCHECK_GT(streams[i].width, 0);
     RTC_DCHECK_GT(streams[i].height, 0);
     RTC_DCHECK_GT(streams[i].max_framerate, 0);
-    // Different framerates not supported per stream at the moment.
-    RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate);
+    // Different framerates not supported per stream at the moment, unless it's
+    // screenshare where there is an exception and a simulcast encoder adapter,
+    // which supports different framerates, is used instead.
+    if (config.content_type != VideoEncoderConfig::ContentType::kScreen) {
+      RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate);
+    }
     RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0);
     RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps);
     RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps);

webrtc/modules/video_coding/video_codec_initializer_unittest.cc

@@ -0,0 +1,217 @@
+/*
+ *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/common_video/include/video_bitrate_allocator.h"
+#include "webrtc/common_types.h"
+#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
+#include "webrtc/modules/video_coding/include/video_codec_initializer.h"
+#include "webrtc/test/gtest.h"
+#include "webrtc/video_encoder.h"
+
+namespace webrtc {
+
+namespace {
+static const char* kVp8PayloadName = "VP8";
+static const int kVp8PayloadType = 100;
+static const int kDefaultWidth = 1280;
+static const int kDefaultHeight = 720;
+static const int kDefaultFrameRate = 30;
+static const uint32_t kDefaultMinBitrateBps = 60000;
+static const uint32_t kDefaultTargetBitrateBps = 2000000;
+static const uint32_t kDefaultMaxBitrateBps = 2000000;
+static const uint32_t kDefaultMinTransmitBitrateBps = 400000;
+static const int kDefaultMaxQp = 48;
+static const uint32_t kScreenshareTl0BitrateBps = 100000;
+static const uint32_t kScreenshareCodecTargetBitrateBps = 200000;
+static const uint32_t kScreenshareDefaultFramerate = 5;
+// Bitrates for the temporal layers of the higher screenshare simulcast stream.
+static const uint32_t kHighScreenshareTl0Bps = 800000;
+static const uint32_t kHighScreenshareTl1Bps = 1200000;
+}  // namespace
+
+/*
+ *   static bool SetupCodec(
+      const VideoEncoderConfig& config,
+      const VideoSendStream::Config::EncoderSettings settings,
+      const std::vector<VideoStream>& streams,
+      bool nack_enabled,
+      VideoCodec* codec,
+      std::unique_ptr<VideoBitrateAllocator>* bitrate_allocator);
+
+  // Create a bitrate allocator for the specified codec. |tl_factory| is
+  // optional, if it is populated, ownership of that instance will be
+  // transferred to the VideoBitrateAllocator instance.
+  static std::unique_ptr<VideoBitrateAllocator> CreateBitrateAllocator(
+      const VideoCodec& codec,
+      std::unique_ptr<TemporalLayersFactory> tl_factory);
+ */
+
+// TODO(sprang): Extend coverage to handle the rest of the codec initializer.
+class VideoCodecInitializerTest : public ::testing::Test {
+ public:
+  VideoCodecInitializerTest() : nack_enabled_(false) {}
+  virtual ~VideoCodecInitializerTest() {}
+
+ protected:
+  void SetUpFor(VideoCodecType type,
+                int num_spatial_streams,
+                int num_temporal_streams,
+                bool screenshare) {
+    config_ = VideoEncoderConfig();
+    if (screenshare) {
+      config_.min_transmit_bitrate_bps = kDefaultMinTransmitBitrateBps;
+      config_.content_type = VideoEncoderConfig::ContentType::kScreen;
+    }
+
+    if (type == VideoCodecType::kVideoCodecVP8) {
+      config_.number_of_streams = num_spatial_streams;
+      VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings();
+      vp8_settings.numberOfTemporalLayers = num_temporal_streams;
+      config_.encoder_specific_settings = new rtc::RefCountedObject<
+          webrtc::VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings);
+      settings_.payload_name = kVp8PayloadName;
+      settings_.payload_type = kVp8PayloadType;
+    } else {
+      ADD_FAILURE() << "Unexpected codec type: " << type;
+    }
+  }
+
+  bool InitializeCodec() {
+    codec_out_ = VideoCodec();
+    bitrate_allocator_out_.reset();
+    temporal_layers_.clear();
+    if (!VideoCodecInitializer::SetupCodec(config_, settings_, streams_,
+                                           nack_enabled_, &codec_out_,
+                                           &bitrate_allocator_out_)) {
+      return false;
+    }
+
+    // Make sure temporal layers instances have been created.
+    if (codec_out_.codecType == VideoCodecType::kVideoCodecVP8) {
+      if (!codec_out_.VP8()->tl_factory)
+        return false;
+
+      for (int i = 0; i < codec_out_.numberOfSimulcastStreams; ++i) {
+        temporal_layers_.emplace_back(codec_out_.VP8()->tl_factory->Create(
+            i, streams_[i].temporal_layer_thresholds_bps.size() + 1, 0));
+      }
+    }
+    return true;
+  }
+
+  VideoStream DefaultStream() {
+    VideoStream stream;
+    stream.width = kDefaultWidth;
+    stream.height = kDefaultHeight;
+    stream.max_framerate = kDefaultFrameRate;
+    stream.min_bitrate_bps = kDefaultMinBitrateBps;
+    stream.target_bitrate_bps = kDefaultTargetBitrateBps;
+    stream.max_bitrate_bps = kDefaultMaxBitrateBps;
+    stream.max_qp = kDefaultMaxQp;
+    return stream;
+  }
+
+  VideoStream DefaultScreenshareStream() {
+    VideoStream stream = DefaultStream();
+    stream.min_bitrate_bps = 30000;
+    stream.target_bitrate_bps = kScreenshareTl0BitrateBps;
+    stream.max_bitrate_bps = 1000000;
+    stream.max_framerate = kScreenshareDefaultFramerate;
+    stream.temporal_layer_thresholds_bps.push_back(kScreenshareTl0BitrateBps);
+    return stream;
+  }
+
+  // Input settings.
+  VideoEncoderConfig config_;
+  VideoSendStream::Config::EncoderSettings settings_;
+  std::vector<VideoStream> streams_;
+  bool nack_enabled_;
+
+  // Output.
+  VideoCodec codec_out_;
+  std::unique_ptr<VideoBitrateAllocator> bitrate_allocator_out_;
+  std::vector<std::unique_ptr<TemporalLayers>> temporal_layers_;
+};
+
+TEST_F(VideoCodecInitializerTest, SingleStreamVp8Screenshare) {
+  SetUpFor(VideoCodecType::kVideoCodecVP8, 1, 1, true);
+  streams_.push_back(DefaultStream());
+  EXPECT_TRUE(InitializeCodec());
+
+  BitrateAllocation bitrate_allocation = bitrate_allocator_out_->GetAllocation(
+      kDefaultTargetBitrateBps, kDefaultFrameRate);
+  EXPECT_EQ(1u, codec_out_.numberOfSimulcastStreams);
+  EXPECT_EQ(1u, codec_out_.VP8()->numberOfTemporalLayers);
+  EXPECT_EQ(kDefaultTargetBitrateBps, bitrate_allocation.get_sum_bps());
+}
+
+TEST_F(VideoCodecInitializerTest, TemporalLayeredVp8Screenshare) {
+  SetUpFor(VideoCodecType::kVideoCodecVP8, 1, 2, true);
+  streams_.push_back(DefaultScreenshareStream());
+  EXPECT_TRUE(InitializeCodec());
+
+  EXPECT_EQ(1u, codec_out_.numberOfSimulcastStreams);
+  EXPECT_EQ(2u, codec_out_.VP8()->numberOfTemporalLayers);
+  BitrateAllocation bitrate_allocation = bitrate_allocator_out_->GetAllocation(
+      kScreenshareCodecTargetBitrateBps, kScreenshareDefaultFramerate);
+  EXPECT_EQ(kScreenshareCodecTargetBitrateBps,
+            bitrate_allocation.get_sum_bps());
+  EXPECT_EQ(kScreenshareTl0BitrateBps, bitrate_allocation.GetBitrate(0, 0));
+}
+
+TEST_F(VideoCodecInitializerTest, SimlucastVp8Screenshare) {
+  SetUpFor(VideoCodecType::kVideoCodecVP8, 2, 1, true);
+  streams_.push_back(DefaultScreenshareStream());
+  VideoStream video_stream = DefaultStream();
+  video_stream.max_framerate = kScreenshareDefaultFramerate;
+  streams_.push_back(video_stream);
+  EXPECT_TRUE(InitializeCodec());
+
+  EXPECT_EQ(2u, codec_out_.numberOfSimulcastStreams);
+  EXPECT_EQ(1u, codec_out_.VP8()->numberOfTemporalLayers);
+  const uint32_t max_bitrate_bps =
+      streams_[0].target_bitrate_bps + streams_[1].max_bitrate_bps;
+  BitrateAllocation bitrate_allocation = bitrate_allocator_out_->GetAllocation(
+      max_bitrate_bps, kScreenshareDefaultFramerate);
+  EXPECT_EQ(max_bitrate_bps, bitrate_allocation.get_sum_bps());
+  EXPECT_EQ(static_cast<uint32_t>(streams_[0].target_bitrate_bps),
+            bitrate_allocation.GetSpatialLayerSum(0));
+  EXPECT_EQ(static_cast<uint32_t>(streams_[1].max_bitrate_bps),
+            bitrate_allocation.GetSpatialLayerSum(1));
+}
+
+TEST_F(VideoCodecInitializerTest, HighFpsSimlucastVp8Screenshare) {
+  // Two simulcast streams, the lower one using legacy settings (two temporal
+  // streams, 5fps), the higher one using 3 temporal streams and 30fps.
+  SetUpFor(VideoCodecType::kVideoCodecVP8, 2, 3, true);
+  streams_.push_back(DefaultScreenshareStream());
+  VideoStream video_stream = DefaultStream();
+  video_stream.temporal_layer_thresholds_bps.push_back(kHighScreenshareTl0Bps);
+  video_stream.temporal_layer_thresholds_bps.push_back(kHighScreenshareTl1Bps);
+  streams_.push_back(video_stream);
+  EXPECT_TRUE(InitializeCodec());
+
+  EXPECT_EQ(2u, codec_out_.numberOfSimulcastStreams);
+  EXPECT_EQ(3u, codec_out_.VP8()->numberOfTemporalLayers);
+  const uint32_t max_bitrate_bps =
+      streams_[0].target_bitrate_bps + streams_[1].max_bitrate_bps;
+  BitrateAllocation bitrate_allocation =
+      bitrate_allocator_out_->GetAllocation(max_bitrate_bps, kDefaultFrameRate);
+  EXPECT_EQ(max_bitrate_bps, bitrate_allocation.get_sum_bps());
+  EXPECT_EQ(static_cast<uint32_t>(streams_[0].target_bitrate_bps),
+            bitrate_allocation.GetSpatialLayerSum(0));
+  EXPECT_EQ(static_cast<uint32_t>(streams_[1].max_bitrate_bps),
+            bitrate_allocation.GetSpatialLayerSum(1));
+  EXPECT_EQ(kHighScreenshareTl0Bps, bitrate_allocation.GetBitrate(1, 0));
+  EXPECT_EQ(kHighScreenshareTl1Bps - kHighScreenshareTl0Bps,
+            bitrate_allocation.GetBitrate(1, 1));
+}
+
+}  // namespace webrtc