From 78ce619a0cd871a769b5c3892dc930df10dfa253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Spr=C3=A5ng?= Date: Mon, 12 Sep 2016 16:04:43 +0200 Subject: [PATCH] Extract simulcast rate allocation outside of video encoder. This is a first step to refactor this code. I'm deprecating https://codereview.webrtc.org/1913073002 and implementing this in smaller more isolated steps. BUG=webrtc:5206 R=asapersson@webrtc.org, kjellander@webrtc.org, noahric@chromium.org Review URL: https://codereview.webrtc.org/2288223002 . Cr-Commit-Position: refs/heads/master@{#14186} --- webrtc/modules/BUILD.gn | 1 + webrtc/modules/modules.gyp | 1 + webrtc/modules/video_coding/BUILD.gn | 2 + .../codecs/vp8/simulcast_encoder_adapter.cc | 118 +++------- .../codecs/vp8/simulcast_encoder_adapter.h | 17 +- .../video_coding/codecs/vp8/vp8_impl.cc | 57 +---- .../video_coding/codecs/vp8/vp8_impl.h | 6 +- .../utility/simulcast_rate_allocator.cc | 72 ++++++ .../utility/simulcast_rate_allocator.h | 37 +++ .../simulcast_rate_allocator_unittest.cc | 211 ++++++++++++++++++ .../utility/video_coding_utility.gyp | 2 + 11 files changed, 374 insertions(+), 150 deletions(-) create mode 100644 webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc create mode 100644 webrtc/modules/video_coding/utility/simulcast_rate_allocator.h create mode 100644 webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc diff --git a/webrtc/modules/BUILD.gn b/webrtc/modules/BUILD.gn index 022e445e48..f997e1eda9 100644 --- a/webrtc/modules/BUILD.gn +++ b/webrtc/modules/BUILD.gn @@ -461,6 +461,7 @@ if (rtc_include_tests) { "video_coding/utility/h264_bitstream_parser_unittest.cc", "video_coding/utility/ivf_file_writer_unittest.cc", "video_coding/utility/quality_scaler_unittest.cc", + "video_coding/utility/simulcast_rate_allocator_unittest.cc", "video_coding/video_coding_robustness_unittest.cc", "video_coding/video_packet_buffer_unittest.cc", "video_coding/video_receiver_unittest.cc", diff --git a/webrtc/modules/modules.gyp b/webrtc/modules/modules.gyp index 9e30154b72..2b552c0314 100644 --- a/webrtc/modules/modules.gyp +++ b/webrtc/modules/modules.gyp @@ -385,6 +385,7 @@ 'video_coding/utility/h264_bitstream_parser_unittest.cc', 'video_coding/utility/ivf_file_writer_unittest.cc', 'video_coding/utility/quality_scaler_unittest.cc', + 'video_coding/utility/simulcast_rate_allocator_unittest.cc', 'video_coding/video_coding_robustness_unittest.cc', 'video_coding/video_packet_buffer_unittest.cc', 'video_coding/video_receiver_unittest.cc', diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn index 4f3b8e50f0..b73b6e703f 100644 --- a/webrtc/modules/video_coding/BUILD.gn +++ b/webrtc/modules/video_coding/BUILD.gn @@ -108,6 +108,8 @@ rtc_source_set("video_coding_utility") { "utility/qp_parser.h", "utility/quality_scaler.cc", "utility/quality_scaler.h", + "utility/simulcast_rate_allocator.cc", + "utility/simulcast_rate_allocator.h", "utility/vp8_header_parser.cc", "utility/vp8_header_parser.h", ] diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc index 586a6491fa..766e517484 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc @@ -17,6 +17,7 @@ #include "webrtc/base/checks.h" #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h" +#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" #include "webrtc/system_wrappers/include/clock.h" namespace { @@ -26,14 +27,6 @@ const unsigned int kDefaultMaxQp = 56; // Max qp for lowest spatial resolution when doing simulcast. const unsigned int kLowestResMaxQp = 45; -uint32_t SumStreamTargetBitrate(int streams, const webrtc::VideoCodec& codec) { - uint32_t bitrate_sum = 0; - for (int i = 0; i < streams; ++i) { - bitrate_sum += codec.simulcastStream[i].targetBitrate; - } - return bitrate_sum; -} - uint32_t SumStreamMaxBitrate(int streams, const webrtc::VideoCodec& codec) { uint32_t bitrate_sum = 0; for (int i = 0; i < streams; ++i) { @@ -92,13 +85,8 @@ int VerifyCodec(const webrtc::VideoCodec* inst) { return WEBRTC_VIDEO_CODEC_OK; } -// TL1 FrameDropper's max time to drop frames. -const float kTl1MaxTimeToDropFrames = 20.0f; - struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory { - ScreenshareTemporalLayersFactory() - : tl1_frame_dropper_(kTl1MaxTimeToDropFrames) {} - + ScreenshareTemporalLayersFactory() {} virtual ~ScreenshareTemporalLayersFactory() {} virtual webrtc::TemporalLayers* Create(int num_temporal_layers, @@ -106,9 +94,6 @@ struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory { return new webrtc::ScreenshareLayers(num_temporal_layers, rand(), webrtc::Clock::GetRealTimeClock()); } - - mutable webrtc::FrameDropper tl0_frame_dropper_; - mutable webrtc::FrameDropper tl1_frame_dropper_; }; // An EncodedImageCallback implementation that forwards on calls to a @@ -139,9 +124,10 @@ namespace webrtc { SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory) : factory_(factory), - encoded_complete_callback_(NULL), + encoded_complete_callback_(nullptr), implementation_name_("SimulcastEncoderAdapter") { memset(&codec_, 0, sizeof(webrtc::VideoCodec)); + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); } SimulcastEncoderAdapter::~SimulcastEncoderAdapter() { @@ -189,6 +175,9 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst, } codec_ = *inst; + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); + std::vector start_bitrates = + rate_allocator_->GetAllocation(codec_.startBitrate); // Special mode when screensharing on a single stream. if (number_of_streams == 1 && inst->mode == kScreensharing) { @@ -200,15 +189,18 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst, // Create |number_of_streams| of encoder instances and init them. for (int i = 0; i < number_of_streams; ++i) { VideoCodec stream_codec; - bool send_stream = true; + uint32_t start_bitrate_kbps = start_bitrates[i]; if (!doing_simulcast) { stream_codec = codec_; stream_codec.numberOfSimulcastStreams = 1; } else { + // Cap start bitrate to the min bitrate in order to avoid strange codec + // behavior. Since sending sending will be false, this should not matter. + start_bitrate_kbps = + std::max(codec_.simulcastStream[i].minBitrate, start_bitrate_kbps); bool highest_resolution_stream = (i == (number_of_streams - 1)); - PopulateStreamCodec(&codec_, i, number_of_streams, - highest_resolution_stream, &stream_codec, - &send_stream); + PopulateStreamCodec(&codec_, i, start_bitrate_kbps, + highest_resolution_stream, &stream_codec); } // TODO(ronghuawu): Remove once this is handled in VP8EncoderImpl. @@ -225,7 +217,8 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst, EncodedImageCallback* callback = new AdapterEncodedImageCallback(this, i); encoder->RegisterEncodeCompleteCallback(callback); streaminfos_.push_back(StreamInfo(encoder, callback, stream_codec.width, - stream_codec.height, send_stream)); + stream_codec.height, + start_bitrate_kbps > 0)); if (i != 0) implementation_name += ", "; implementation_name += streaminfos_[i].encoder->ImplementationName(); @@ -363,6 +356,8 @@ int SimulcastEncoderAdapter::SetRates(uint32_t new_bitrate_kbit, if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate) { new_bitrate_kbit = codec_.maxBitrate; } + + std::vector stream_bitrates; if (new_bitrate_kbit > 0) { // Make sure the bitrate fits the configured min bitrates. 0 is a special // value that means paused, though, so leave it alone. @@ -373,19 +368,20 @@ int SimulcastEncoderAdapter::SetRates(uint32_t new_bitrate_kbit, new_bitrate_kbit < codec_.simulcastStream[0].minBitrate) { new_bitrate_kbit = codec_.simulcastStream[0].minBitrate; } + stream_bitrates = rate_allocator_->GetAllocation(new_bitrate_kbit); } codec_.maxFramerate = new_framerate; - bool send_stream = true; - uint32_t stream_bitrate = 0; + // Disable any stream not in the current allocation. + stream_bitrates.resize(streaminfos_.size(), 0U); + for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { - stream_bitrate = GetStreamBitrate(stream_idx, streaminfos_.size(), - new_bitrate_kbit, &send_stream); + uint32_t stream_bitrate_kbps = stream_bitrates[stream_idx]; // Need a key frame if we have not sent this stream before. - if (send_stream && !streaminfos_[stream_idx].send_stream) { + if (stream_bitrate_kbps > 0 && !streaminfos_[stream_idx].send_stream) { streaminfos_[stream_idx].key_frame_request = true; } - streaminfos_[stream_idx].send_stream = send_stream; + streaminfos_[stream_idx].send_stream = stream_bitrate_kbps > 0; // TODO(holmer): This is a temporary hack for screensharing, where we // interpret the startBitrate as the encoder target bitrate. This is @@ -395,14 +391,15 @@ int SimulcastEncoderAdapter::SetRates(uint32_t new_bitrate_kbit, if (codec_.targetBitrate > 0 && (codec_.codecSpecific.VP8.numberOfTemporalLayers == 2 || codec_.simulcastStream[0].numberOfTemporalLayers == 2)) { - stream_bitrate = std::min(codec_.maxBitrate, stream_bitrate); + stream_bitrate_kbps = std::min(codec_.maxBitrate, stream_bitrate_kbps); // TODO(ronghuawu): Can't change max bitrate via the VideoEncoder // interface. And VP8EncoderImpl doesn't take negative framerate. - // max_bitrate = std::min(codec_.maxBitrate, stream_bitrate); + // max_bitrate = std::min(codec_.maxBitrate, stream_bitrate_kbps); // new_framerate = -1; } - streaminfos_[stream_idx].encoder->SetRates(stream_bitrate, new_framerate); + streaminfos_[stream_idx].encoder->SetRates(stream_bitrate_kbps, + new_framerate); } return WEBRTC_VIDEO_CODEC_OK; @@ -422,61 +419,12 @@ EncodedImageCallback::Result SimulcastEncoderAdapter::OnEncodedImage( encodedImage, &stream_codec_specific, fragmentation); } -uint32_t SimulcastEncoderAdapter::GetStreamBitrate( - int stream_idx, - size_t total_number_of_streams, - uint32_t new_bitrate_kbit, - bool* send_stream) const { - if (total_number_of_streams == 1) { - *send_stream = true; - return new_bitrate_kbit; - } - - // The bitrate needed to start sending this stream is given by the - // minimum bitrate allowed for encoding this stream, plus the sum target - // rates of all lower streams. - uint32_t sum_target_lower_streams = - SumStreamTargetBitrate(stream_idx, codec_); - uint32_t bitrate_to_send_this_layer = - codec_.simulcastStream[stream_idx].minBitrate + sum_target_lower_streams; - if (new_bitrate_kbit >= bitrate_to_send_this_layer) { - // We have enough bandwidth to send this stream. - *send_stream = true; - // Bitrate for this stream is the new bitrate (|new_bitrate_kbit|) minus the - // sum target rates of the lower streams, and capped to a maximum bitrate. - // The maximum cap depends on whether we send the next higher stream. - // If we will be sending the next higher stream, |max_rate| is given by - // current stream's |targetBitrate|, otherwise it's capped by |maxBitrate|. - if (stream_idx < codec_.numberOfSimulcastStreams - 1) { - unsigned int max_rate = codec_.simulcastStream[stream_idx].maxBitrate; - if (new_bitrate_kbit >= - SumStreamTargetBitrate(stream_idx + 1, codec_) + - codec_.simulcastStream[stream_idx + 1].minBitrate) { - max_rate = codec_.simulcastStream[stream_idx].targetBitrate; - } - return std::min(new_bitrate_kbit - sum_target_lower_streams, max_rate); - } else { - // For the highest stream (highest resolution), the |targetBitRate| and - // |maxBitrate| are not used. Any excess bitrate (above the targets of - // all lower streams) is given to this (highest resolution) stream. - return new_bitrate_kbit - sum_target_lower_streams; - } - } else { - // Not enough bitrate for this stream. - // Return our max bitrate of |stream_idx| - 1, but we don't send it. We need - // to keep this resolution coding in order for the multi-encoder to work. - *send_stream = false; - return codec_.simulcastStream[stream_idx - 1].maxBitrate; - } -} - void SimulcastEncoderAdapter::PopulateStreamCodec( const webrtc::VideoCodec* inst, int stream_index, - size_t total_number_of_streams, + uint32_t start_bitrate_kbps, bool highest_resolution_stream, - webrtc::VideoCodec* stream_codec, - bool* send_stream) { + webrtc::VideoCodec* stream_codec) { *stream_codec = *inst; // Stream specific settings. @@ -505,9 +453,7 @@ void SimulcastEncoderAdapter::PopulateStreamCodec( } // TODO(ronghuawu): what to do with targetBitrate. - int stream_bitrate = GetStreamBitrate(stream_index, total_number_of_streams, - inst->startBitrate, send_stream); - stream_codec->startBitrate = stream_bitrate; + stream_codec->startBitrate = start_bitrate_kbps; } bool SimulcastEncoderAdapter::Initialized() const { diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h index be6aa59b6c..30d5abb6cc 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h @@ -20,6 +20,8 @@ namespace webrtc { +class SimulcastRateAllocator; + class VideoEncoderFactory { public: virtual VideoEncoder* Create() = 0; @@ -91,22 +93,12 @@ class SimulcastEncoderAdapter : public VP8Encoder { bool send_stream; }; - // Get the stream bitrate, for the stream |stream_idx|, given the bitrate - // |new_bitrate_kbit| and the actual configured stream count in - // |total_number_of_streams|. The function also returns whether there's enough - // bandwidth to send this stream via |send_stream|. - uint32_t GetStreamBitrate(int stream_idx, - size_t total_number_of_streams, - uint32_t new_bitrate_kbit, - bool* send_stream) const; - // Populate the codec settings for each stream. void PopulateStreamCodec(const webrtc::VideoCodec* inst, int stream_index, - size_t total_number_of_streams, + uint32_t start_bitrate_kbps, bool highest_resolution_stream, - webrtc::VideoCodec* stream_codec, - bool* send_stream); + webrtc::VideoCodec* stream_codec); bool Initialized() const; @@ -116,6 +108,7 @@ class SimulcastEncoderAdapter : public VP8Encoder { std::vector streaminfos_; EncodedImageCallback* encoded_complete_callback_; std::string implementation_name_; + std::unique_ptr rate_allocator_; }; } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index dc32d3af41..ef5e8e3d73 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -29,6 +29,7 @@ #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" #include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h" #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" +#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" #include "webrtc/system_wrappers/include/clock.h" namespace webrtc { @@ -59,46 +60,6 @@ int GCD(int a, int b) { return b; } -std::vector GetStreamBitratesKbps(const VideoCodec& codec, - int bitrate_to_allocate_kbps) { - if (codec.numberOfSimulcastStreams <= 1) { - return std::vector(1, bitrate_to_allocate_kbps); - } - - std::vector bitrates_kbps(codec.numberOfSimulcastStreams); - // Allocate min -> target bitrates as long as we have bitrate to spend. - size_t last_active_stream = 0; - for (size_t i = 0; i < static_cast(codec.numberOfSimulcastStreams) && - bitrate_to_allocate_kbps >= - static_cast(codec.simulcastStream[i].minBitrate); - ++i) { - last_active_stream = i; - int allocated_bitrate_kbps = - std::min(static_cast(codec.simulcastStream[i].targetBitrate), - bitrate_to_allocate_kbps); - bitrates_kbps[i] = allocated_bitrate_kbps; - bitrate_to_allocate_kbps -= allocated_bitrate_kbps; - } - - // Spend additional bits on the highest-quality active layer, up to max - // bitrate. - // TODO(pbos): Consider spending additional bits on last_active_stream-1 down - // to 0 and not just the top layer when we have additional bitrate to spend. - int allocated_bitrate_kbps = std::min( - static_cast(codec.simulcastStream[last_active_stream].maxBitrate - - bitrates_kbps[last_active_stream]), - bitrate_to_allocate_kbps); - bitrates_kbps[last_active_stream] += allocated_bitrate_kbps; - bitrate_to_allocate_kbps -= allocated_bitrate_kbps; - - // Make sure we can always send something. Suspending below min bitrate is - // controlled outside the codec implementation and is not overriden by this. - if (bitrates_kbps[0] < static_cast(codec.simulcastStream[0].minBitrate)) - bitrates_kbps[0] = static_cast(codec.simulcastStream[0].minBitrate); - - return bitrates_kbps; -} - uint32_t SumStreamMaxBitrate(int streams, const VideoCodec& codec) { uint32_t bitrate_sum = 0; for (int i = 0; i < streams; ++i) { @@ -149,10 +110,9 @@ VP8Decoder* VP8Decoder::Create() { return new VP8DecoderImpl(); } -const float kTl1MaxTimeToDropFrames = 20.0f; - VP8EncoderImpl::VP8EncoderImpl() - : encoded_complete_callback_(NULL), + : encoded_complete_callback_(nullptr), + rate_allocator_(new SimulcastRateAllocator(codec_)), inited_(false), timestamp_(0), feedback_mode_(false), @@ -162,8 +122,6 @@ VP8EncoderImpl::VP8EncoderImpl() token_partitions_(VP8_ONE_TOKENPARTITION), down_scale_requested_(false), down_scale_bitrate_(0), - tl0_frame_dropper_(), - tl1_frame_dropper_(kTl1MaxTimeToDropFrames), key_frame_request_(kMaxSimulcastStreams, false), quality_scaler_enabled_(false) { uint32_t seed = rtc::Time32(); @@ -273,8 +231,8 @@ int VP8EncoderImpl::SetRates(uint32_t new_bitrate_kbit, } } - std::vector stream_bitrates = - GetStreamBitratesKbps(codec_, new_bitrate_kbit); + std::vector stream_bitrates = + rate_allocator_->GetAllocation(new_bitrate_kbit); size_t stream_idx = encoders_.size() - 1; for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) { if (encoders_.size() > 1) @@ -403,6 +361,7 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst, timestamp_ = 0; codec_ = *inst; + rate_allocator_.reset(new SimulcastRateAllocator(codec_)); // Code expects simulcastStream resolutions to be correct, make sure they are // filled even when there are no simulcast layers. @@ -570,8 +529,8 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst, // Note the order we use is different from webm, we have lowest resolution // at position 0 and they have highest resolution at position 0. int stream_idx = encoders_.size() - 1; - std::vector stream_bitrates = - GetStreamBitratesKbps(codec_, inst->startBitrate); + std::vector stream_bitrates = + rate_allocator_->GetAllocation(inst->startBitrate); SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx); configurations_[0].rc_target_bitrate = stream_bitrates[stream_idx]; temporal_layers_[stream_idx]->ConfigureBitrates( diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h index ff47caafaa..00a711554d 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h @@ -13,6 +13,7 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_VP8_IMPL_H_ #define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_VP8_IMPL_H_ +#include #include // NOTE: This include order must remain to avoid compile errors, even though @@ -26,12 +27,12 @@ #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" #include "webrtc/modules/video_coding/codecs/vp8/reference_picture_selection.h" -#include "webrtc/modules/video_coding/utility/frame_dropper.h" #include "webrtc/modules/video_coding/utility/quality_scaler.h" #include "webrtc/video_frame.h" namespace webrtc { +class SimulcastRateAllocator; class TemporalLayers; class VP8EncoderImpl : public VP8Encoder { @@ -93,6 +94,7 @@ class VP8EncoderImpl : public VP8Encoder { EncodedImageCallback* encoded_complete_callback_; VideoCodec codec_; + std::unique_ptr rate_allocator_; bool inited_; int64_t timestamp_; bool feedback_mode_; @@ -104,8 +106,6 @@ class VP8EncoderImpl : public VP8Encoder { std::vector temporal_layers_; bool down_scale_requested_; uint32_t down_scale_bitrate_; - FrameDropper tl0_frame_dropper_; - FrameDropper tl1_frame_dropper_; std::vector picture_id_; std::vector last_key_frame_picture_id_; std::vector key_frame_request_; diff --git a/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc b/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc new file mode 100644 index 0000000000..7264541bb9 --- /dev/null +++ b/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" + +#include + +namespace webrtc { + +webrtc::SimulcastRateAllocator::SimulcastRateAllocator(const VideoCodec& codec) + : codec_(codec) {} + +std::vector webrtc::SimulcastRateAllocator::GetAllocation( + uint32_t bitrate_kbps) const { + // Always allocate enough bitrate for the minimum bitrate of the first layer. + // Suspending below min bitrate is controlled outside the codec implementation + // and is not overridden by this. + const uint32_t min_bitrate_bps = codec_.numberOfSimulcastStreams == 0 + ? codec_.minBitrate + : codec_.simulcastStream[0].minBitrate; + uint32_t left_to_allocate = std::max(min_bitrate_bps, bitrate_kbps); + if (codec_.maxBitrate) + left_to_allocate = std::min(left_to_allocate, codec_.maxBitrate); + + if (codec_.numberOfSimulcastStreams == 0) { + // No simulcast, just set the target as this has been capped already. + return std::vector(1, left_to_allocate); + } + + // Initialize bitrates with zeroes. + std::vector allocated_bitrates_bps(codec_.numberOfSimulcastStreams, + 0); + + // First try to allocate up to the target bitrate for each substream. + size_t layer = 0; + for (; layer < codec_.numberOfSimulcastStreams; ++layer) { + const SimulcastStream& stream = codec_.simulcastStream[layer]; + if (left_to_allocate < stream.minBitrate) + break; + uint32_t allocation = std::min(left_to_allocate, stream.targetBitrate); + allocated_bitrates_bps[layer] = allocation; + left_to_allocate -= allocation; + } + + // Next, try allocate remaining bitrate, up to max bitrate, in top layer. + // TODO(sprang): Allocate up to max bitrate for all layers once we have a + // better idea of possible performance implications. + if (left_to_allocate > 0) { + size_t active_layer = layer - 1; + const SimulcastStream& stream = codec_.simulcastStream[active_layer]; + uint32_t allocation = + std::min(left_to_allocate, + stream.maxBitrate - allocated_bitrates_bps[active_layer]); + left_to_allocate -= allocation; + allocated_bitrates_bps[active_layer] += allocation; + } + + return allocated_bitrates_bps; +} + +const VideoCodec& webrtc::SimulcastRateAllocator::GetCodec() const { + return codec_; +} + +} // namespace webrtc diff --git a/webrtc/modules/video_coding/utility/simulcast_rate_allocator.h b/webrtc/modules/video_coding/utility/simulcast_rate_allocator.h new file mode 100644 index 0000000000..fb000e21c1 --- /dev/null +++ b/webrtc/modules/video_coding/utility/simulcast_rate_allocator.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_SIMULCAST_RATE_ALLOCATOR_H_ +#define WEBRTC_MODULES_VIDEO_CODING_UTILITY_SIMULCAST_RATE_ALLOCATOR_H_ + +#include + +#include "webrtc/base/basictypes.h" +#include "webrtc/base/constructormagic.h" +#include "webrtc/video_encoder.h" + +namespace webrtc { + +class SimulcastRateAllocator { + public: + explicit SimulcastRateAllocator(const VideoCodec& codec); + + std::vector GetAllocation(uint32_t bitrate_kbps) const; + const VideoCodec& GetCodec() const; + + private: + const VideoCodec codec_; + + RTC_DISALLOW_COPY_AND_ASSIGN(SimulcastRateAllocator); +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_VIDEO_CODING_UTILITY_SIMULCAST_RATE_ALLOCATOR_H_ diff --git a/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc b/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc new file mode 100644 index 0000000000..a4af36f6d2 --- /dev/null +++ b/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" + +#include +#include + +#include "testing/gtest/include/gtest/gtest.h" + +namespace webrtc { +namespace { +constexpr uint32_t kMinBitrate = 50; +constexpr uint32_t kTargetBitrate = 100; +constexpr uint32_t kMaxBitrate = 1000; +} // namespace + +class SimulcastRateAllocatorTest : public ::testing::Test { + public: + SimulcastRateAllocatorTest() { + memset(&codec_, 0, sizeof(VideoCodec)); + codec_.minBitrate = kMinBitrate; + codec_.targetBitrate = kTargetBitrate; + codec_.maxBitrate = kMaxBitrate; + CreateAllocator(); + } + virtual ~SimulcastRateAllocatorTest() {} + + template + void ExpectEqual(uint32_t (&expected)[S], + const std::vector& actual) { + EXPECT_EQ(S, actual.size()); + for (size_t i = 0; i < S; ++i) + EXPECT_EQ(expected[i], actual[i]) << "Mismatch at index " << i; + } + + void CreateAllocator() { + allocator_.reset(new SimulcastRateAllocator(codec_)); + } + + protected: + VideoCodec codec_; + std::unique_ptr allocator_; +}; + +TEST_F(SimulcastRateAllocatorTest, NoSimulcastBelowMin) { + uint32_t expected[] = {codec_.minBitrate}; + ExpectEqual(expected, allocator_->GetAllocation(codec_.minBitrate - 1)); + ExpectEqual(expected, allocator_->GetAllocation(1)); + ExpectEqual(expected, allocator_->GetAllocation(0)); +} + +TEST_F(SimulcastRateAllocatorTest, NoSimulcastAboveMax) { + uint32_t expected[] = {codec_.maxBitrate}; + ExpectEqual(expected, allocator_->GetAllocation(codec_.maxBitrate + 1)); + ExpectEqual(expected, + allocator_->GetAllocation(std::numeric_limits::max())); +} + +TEST_F(SimulcastRateAllocatorTest, NoSimulcastNoMax) { + constexpr uint32_t kMax = std::numeric_limits::max(); + codec_.maxBitrate = 0; + CreateAllocator(); + + uint32_t expected[] = {kMax}; + ExpectEqual(expected, allocator_->GetAllocation(kMax)); +} + +TEST_F(SimulcastRateAllocatorTest, NoSimulcastWithinLimits) { + for (uint32_t bitrate = codec_.minBitrate; bitrate <= codec_.maxBitrate; + ++bitrate) { + uint32_t expected[] = {bitrate}; + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + } +} + +TEST_F(SimulcastRateAllocatorTest, SingleSimulcastBelowMin) { + // With simulcast, use the min bitrate from the ss spec instead of the global. + codec_.numberOfSimulcastStreams = 1; + const uint32_t kMin = codec_.minBitrate - 10; + codec_.simulcastStream[0].minBitrate = kMin; + codec_.simulcastStream[0].targetBitrate = kTargetBitrate; + CreateAllocator(); + + uint32_t expected[] = {kMin}; + ExpectEqual(expected, allocator_->GetAllocation(kMin - 1)); + ExpectEqual(expected, allocator_->GetAllocation(1)); + ExpectEqual(expected, allocator_->GetAllocation(0)); +} + +TEST_F(SimulcastRateAllocatorTest, SingleSimulcastAboveMax) { + codec_.numberOfSimulcastStreams = 1; + codec_.simulcastStream[0].minBitrate = kMinBitrate; + const uint32_t kMax = codec_.simulcastStream[0].maxBitrate + 1000; + codec_.simulcastStream[0].maxBitrate = kMax; + CreateAllocator(); + + uint32_t expected[] = {kMax}; + ExpectEqual(expected, allocator_->GetAllocation(kMax + 1)); + ExpectEqual(expected, + allocator_->GetAllocation(std::numeric_limits::max())); +} + +TEST_F(SimulcastRateAllocatorTest, SingleSimulcastWithinLimits) { + codec_.numberOfSimulcastStreams = 1; + codec_.simulcastStream[0].minBitrate = kMinBitrate; + codec_.simulcastStream[0].targetBitrate = kTargetBitrate; + codec_.simulcastStream[0].maxBitrate = kMaxBitrate; + CreateAllocator(); + + for (uint32_t bitrate = kMinBitrate; bitrate <= kMaxBitrate; ++bitrate) { + uint32_t expected[] = {bitrate}; + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + } +} + +TEST_F(SimulcastRateAllocatorTest, OneToThreeStreams) { + codec_.numberOfSimulcastStreams = 3; + codec_.maxBitrate = 0; + codec_.simulcastStream[0].minBitrate = 10; + codec_.simulcastStream[0].targetBitrate = 100; + codec_.simulcastStream[0].maxBitrate = 500; + codec_.simulcastStream[1].minBitrate = 50; + codec_.simulcastStream[1].targetBitrate = 500; + codec_.simulcastStream[1].maxBitrate = 1000; + codec_.simulcastStream[2].minBitrate = 2000; + codec_.simulcastStream[2].targetBitrate = 3000; + codec_.simulcastStream[2].maxBitrate = 4000; + CreateAllocator(); + + { + // Single stream, min bitrate. + const uint32_t bitrate = codec_.simulcastStream[0].minBitrate; + uint32_t expected[] = {bitrate, 0, 0}; + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + } + + { + // Single stream at target bitrate. + const uint32_t bitrate = codec_.simulcastStream[0].targetBitrate; + uint32_t expected[] = {bitrate, 0, 0}; + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + } + + { + // Bitrate above target for first stream, but below min for the next one. + const uint32_t bitrate = codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].minBitrate - 1; + uint32_t expected[] = {bitrate, 0, 0}; + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + } + + { + // Just enough for two streams. + const uint32_t bitrate = codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].minBitrate; + uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, + codec_.simulcastStream[1].minBitrate, 0}; + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + } + + { + // Second stream maxed out, but not enough for third. + const uint32_t bitrate = codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].maxBitrate; + uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, + codec_.simulcastStream[1].maxBitrate, 0}; + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + } + + { + // First two streams maxed out, but not enough for third. Nowhere to put + // remaining bits. + const uint32_t bitrate = codec_.simulcastStream[0].maxBitrate + + codec_.simulcastStream[1].maxBitrate + 499; + uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, + codec_.simulcastStream[1].maxBitrate, 0}; + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + } + + { + // Just enough for all three streams. + const uint32_t bitrate = codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].targetBitrate + + codec_.simulcastStream[2].minBitrate; + uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, + codec_.simulcastStream[1].targetBitrate, + codec_.simulcastStream[2].minBitrate}; + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + } + + { + // Third maxed out. + const uint32_t bitrate = codec_.simulcastStream[0].targetBitrate + + codec_.simulcastStream[1].targetBitrate + + codec_.simulcastStream[2].maxBitrate; + uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, + codec_.simulcastStream[1].targetBitrate, + codec_.simulcastStream[2].maxBitrate}; + ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + } +} + +} // namespace webrtc diff --git a/webrtc/modules/video_coding/utility/video_coding_utility.gyp b/webrtc/modules/video_coding/utility/video_coding_utility.gyp index eed3486f4a..c189abcfda 100644 --- a/webrtc/modules/video_coding/utility/video_coding_utility.gyp +++ b/webrtc/modules/video_coding/utility/video_coding_utility.gyp @@ -30,6 +30,8 @@ 'qp_parser.h', 'quality_scaler.cc', 'quality_scaler.h', + 'simulcast_rate_allocator.cc', + 'simulcast_rate_allocator.h', 'vp8_header_parser.cc', 'vp8_header_parser.h', ],