diff --git a/webrtc/api/android/jni/androidmediaencoder_jni.cc b/webrtc/api/android/jni/androidmediaencoder_jni.cc index 92b5aae65c..273fe9cfd5 100644 --- a/webrtc/api/android/jni/androidmediaencoder_jni.cc +++ b/webrtc/api/android/jni/androidmediaencoder_jni.cc @@ -112,7 +112,8 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder, int32_t Release() override; int32_t SetChannelParameters(uint32_t /* packet_loss */, int64_t /* rtt */) override; - int32_t SetRates(uint32_t new_bit_rate, uint32_t frame_rate) override; + int32_t SetRateAllocation(const webrtc::BitrateAllocation& rate_allocation, + uint32_t frame_rate) override; // rtc::MessageHandler implementation. void OnMessage(rtc::Message* msg) override; @@ -465,11 +466,12 @@ int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */, return WEBRTC_VIDEO_CODEC_OK; } -int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate, - uint32_t frame_rate) { +int32_t MediaCodecVideoEncoder::SetRateAllocation( + const webrtc::BitrateAllocation& rate_allocation, + uint32_t frame_rate) { return codec_thread_->Invoke( RTC_FROM_HERE, Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread, this, - new_bit_rate, frame_rate)); + rate_allocation.get_sum_kbps(), frame_rate)); } void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) { diff --git a/webrtc/common_types.cc b/webrtc/common_types.cc index e78a4aacca..c63bb99706 100644 --- a/webrtc/common_types.cc +++ b/webrtc/common_types.cc @@ -11,6 +11,7 @@ #include "webrtc/base/checks.h" #include "webrtc/common_types.h" +#include #include namespace webrtc { @@ -59,6 +60,7 @@ VideoCodec::VideoCodec() simulcastStream(), spatialLayers(), mode(kRealtimeVideo), + expect_encode_from_texture(false), codecSpecific() {} VideoCodecVP8* VideoCodec::VP8() { @@ -91,4 +93,89 @@ const VideoCodecH264& VideoCodec::H264() const { return codecSpecific.H264; } +static const char* kPayloadNameVp8 = "VP8"; +static const char* kPayloadNameVp9 = "VP9"; +static const char* kPayloadNameH264 = "H264"; +static const char* kPayloadNameI420 = "I420"; +static const char* kPayloadNameRED = "RED"; +static const char* kPayloadNameULPFEC = "ULPFEC"; +static const char* kPayloadNameGeneric = "Generic"; + +rtc::Optional CodecTypeToPayloadName(VideoCodecType type) { + switch (type) { + case kVideoCodecVP8: + return rtc::Optional(kPayloadNameVp8); + case kVideoCodecVP9: + return rtc::Optional(kPayloadNameVp9); + case kVideoCodecH264: + return rtc::Optional(kPayloadNameH264); + case kVideoCodecI420: + return rtc::Optional(kPayloadNameI420); + case kVideoCodecRED: + return rtc::Optional(kPayloadNameRED); + case kVideoCodecULPFEC: + return rtc::Optional(kPayloadNameULPFEC); + case kVideoCodecGeneric: + return rtc::Optional(kPayloadNameGeneric); + default: + return rtc::Optional(); + } +} + +rtc::Optional PayloadNameToCodecType(const std::string& name) { + if (name == kPayloadNameVp8) + return rtc::Optional(kVideoCodecVP8); + if (name == kPayloadNameVp9) + return rtc::Optional(kVideoCodecVP9); + if (name == kPayloadNameH264) + return rtc::Optional(kVideoCodecH264); + if (name == kPayloadNameI420) + return rtc::Optional(kVideoCodecI420); + if (name == kPayloadNameRED) + return rtc::Optional(kVideoCodecRED); + if (name == kPayloadNameULPFEC) + return rtc::Optional(kVideoCodecULPFEC); + if (name == kPayloadNameGeneric) + return rtc::Optional(kVideoCodecGeneric); + return rtc::Optional(); +} + +const size_t BitrateAllocation::kMaxBitrateBps = + std::numeric_limits::max(); + +BitrateAllocation::BitrateAllocation() : sum_(0), bitrates_{} {} + +bool BitrateAllocation::SetBitrate(size_t spatial_index, + size_t temporal_index, + uint32_t bitrate_bps) { + RTC_DCHECK_LT(spatial_index, static_cast(kMaxSpatialLayers)); + RTC_DCHECK_LT(temporal_index, static_cast(kMaxTemporalStreams)); + RTC_DCHECK_LE(bitrates_[spatial_index][temporal_index], sum_); + uint64_t new_bitrate_sum_bps = sum_; + new_bitrate_sum_bps -= bitrates_[spatial_index][temporal_index]; + new_bitrate_sum_bps += bitrate_bps; + if (new_bitrate_sum_bps > kMaxBitrateBps) + return false; + + bitrates_[spatial_index][temporal_index] = bitrate_bps; + sum_ = static_cast(new_bitrate_sum_bps); + return true; +} + +uint32_t BitrateAllocation::GetBitrate(size_t spatial_index, + size_t temporal_index) const { + RTC_DCHECK_LT(spatial_index, static_cast(kMaxSpatialLayers)); + RTC_DCHECK_LT(temporal_index, static_cast(kMaxTemporalStreams)); + return bitrates_[spatial_index][temporal_index]; +} + +// Get the sum of all the temporal layer for a specific spatial layer. +uint32_t BitrateAllocation::GetSpatialLayerSum(size_t spatial_index) const { + RTC_DCHECK_LT(spatial_index, static_cast(kMaxSpatialLayers)); + uint32_t sum = 0; + for (int i = 0; i < kMaxTemporalStreams; ++i) + sum += bitrates_[spatial_index][i]; + return sum; +} + } // namespace webrtc diff --git a/webrtc/common_types.h b/webrtc/common_types.h index 24ba00a845..bf90afa964 100644 --- a/webrtc/common_types.h +++ b/webrtc/common_types.h @@ -18,6 +18,7 @@ #include #include +#include "webrtc/base/optional.h" #include "webrtc/common_video/rotation.h" #include "webrtc/typedefs.h" @@ -522,7 +523,7 @@ struct VideoCodecVP8 { bool automaticResizeOn; bool frameDroppingOn; int keyFrameInterval; - const TemporalLayersFactory* tl_factory; + TemporalLayersFactory* tl_factory; }; // VP9 specific. @@ -563,6 +564,10 @@ enum VideoCodecType { kVideoCodecUnknown }; +// Translates from name of codec to codec type and vice versa. +rtc::Optional CodecTypeToPayloadName(VideoCodecType type); +rtc::Optional PayloadNameToCodecType(const std::string& name); + union VideoCodecUnion { VideoCodecVP8 VP8; VideoCodecVP9 VP9; @@ -639,6 +644,35 @@ class VideoCodec { VideoCodecUnion codecSpecific; }; +class BitrateAllocation { + public: + static const size_t kMaxBitrateBps; + BitrateAllocation(); + + bool SetBitrate(size_t spatial_index, + size_t temporal_index, + uint32_t bitrate_bps); + + uint32_t GetBitrate(size_t spatial_index, size_t temporal_index) const; + + // Get the sum of all the temporal layer for a specific spatial layer. + uint32_t GetSpatialLayerSum(size_t spatial_index) const; + + uint32_t get_sum_bps() const { return sum_; } // Sum of all bitrates. + uint32_t get_sum_kbps() const { return (sum_ + 500) / 1000; } + + inline bool operator==(const BitrateAllocation& other) const { + return memcmp(bitrates_, other.bitrates_, sizeof(bitrates_)) == 0; + } + inline bool operator!=(const BitrateAllocation& other) const { + return !(*this == other); + } + + private: + uint32_t sum_; + uint32_t bitrates_[kMaxSpatialLayers][kMaxTemporalStreams]; +}; + // Bandwidth over-use detector options. These are used to drive // experimentation with bandwidth estimation parameters. // See modules/remote_bitrate_estimator/overuse_detector.h diff --git a/webrtc/common_video/BUILD.gn b/webrtc/common_video/BUILD.gn index ed98e02611..34cb6d6a3d 100644 --- a/webrtc/common_video/BUILD.gn +++ b/webrtc/common_video/BUILD.gn @@ -35,6 +35,7 @@ rtc_static_library("common_video") { "include/frame_callback.h", "include/i420_buffer_pool.h", "include/incoming_video_stream.h", + "include/video_bitrate_allocator.h", "include/video_frame_buffer.h", "incoming_video_stream.cc", "libyuv/include/webrtc_libyuv.h", diff --git a/webrtc/common_video/common_video.gyp b/webrtc/common_video/common_video.gyp index 58b2462fa8..262b77a1ac 100644 --- a/webrtc/common_video/common_video.gyp +++ b/webrtc/common_video/common_video.gyp @@ -73,6 +73,7 @@ 'include/frame_callback.h', 'include/i420_buffer_pool.h', 'include/incoming_video_stream.h', + 'include/video_bitrate_allocator.h', 'include/video_frame_buffer.h', 'libyuv/include/webrtc_libyuv.h', 'libyuv/webrtc_libyuv.cc', diff --git a/webrtc/common_video/include/video_bitrate_allocator.h b/webrtc/common_video/include/video_bitrate_allocator.h new file mode 100644 index 0000000000..66ff077be5 --- /dev/null +++ b/webrtc/common_video/include/video_bitrate_allocator.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_COMMON_VIDEO_INCLUDE_VIDEO_BITRATE_ALLOCATOR_H_ +#define WEBRTC_COMMON_VIDEO_INCLUDE_VIDEO_BITRATE_ALLOCATOR_H_ + +#include "webrtc/common_types.h" + +namespace webrtc { + +class VideoBitrateAllocator { + public: + VideoBitrateAllocator() {} + virtual ~VideoBitrateAllocator() {} + + virtual BitrateAllocation GetAllocation(uint32_t total_bitrate, + uint32_t framerate) = 0; + virtual uint32_t GetPreferredBitrateBps(uint32_t framerate) = 0; +}; + +} // namespace webrtc + +#endif // WEBRTC_COMMON_VIDEO_INCLUDE_VIDEO_BITRATE_ALLOCATOR_H_ diff --git a/webrtc/media/engine/fakewebrtcvideoengine.h b/webrtc/media/engine/fakewebrtcvideoengine.h index a6cfeb3edf..cb77072e2b 100644 --- a/webrtc/media/engine/fakewebrtcvideoengine.h +++ b/webrtc/media/engine/fakewebrtcvideoengine.h @@ -132,9 +132,9 @@ class FakeWebRtcVideoEncoder : public webrtc::VideoEncoder { FakeWebRtcVideoEncoder() : init_encode_event_(false, false), num_frames_encoded_(0) {} - virtual int32_t InitEncode(const webrtc::VideoCodec* codecSettings, - int32_t numberOfCores, - size_t maxPayloadSize) { + int32_t InitEncode(const webrtc::VideoCodec* codecSettings, + int32_t numberOfCores, + size_t maxPayloadSize) override { rtc::CritScope lock(&crit_); codec_settings_ = *codecSettings; init_encode_event_.Set(); @@ -148,27 +148,28 @@ class FakeWebRtcVideoEncoder : public webrtc::VideoEncoder { return codec_settings_; } - virtual int32_t Encode(const webrtc::VideoFrame& inputImage, - const webrtc::CodecSpecificInfo* codecSpecificInfo, - const std::vector* frame_types) { + int32_t Encode(const webrtc::VideoFrame& inputImage, + const webrtc::CodecSpecificInfo* codecSpecificInfo, + const std::vector* frame_types) override { rtc::CritScope lock(&crit_); ++num_frames_encoded_; init_encode_event_.Set(); return WEBRTC_VIDEO_CODEC_OK; } - virtual int32_t RegisterEncodeCompleteCallback( - webrtc::EncodedImageCallback* callback) { + int32_t RegisterEncodeCompleteCallback( + webrtc::EncodedImageCallback* callback) override { return WEBRTC_VIDEO_CODEC_OK; } - virtual int32_t Release() { return WEBRTC_VIDEO_CODEC_OK; } + int32_t Release() override { return WEBRTC_VIDEO_CODEC_OK; } - virtual int32_t SetChannelParameters(uint32_t packetLoss, int64_t rtt) { + int32_t SetChannelParameters(uint32_t packetLoss, int64_t rtt) override { return WEBRTC_VIDEO_CODEC_OK; } - virtual int32_t SetRates(uint32_t newBitRate, uint32_t frameRate) { + int32_t SetRateAllocation(const webrtc::BitrateAllocation& allocation, + uint32_t framerate) override { return WEBRTC_VIDEO_CODEC_OK; } diff --git a/webrtc/modules/BUILD.gn b/webrtc/modules/BUILD.gn index 8321bcd501..54a1cdec3a 100644 --- a/webrtc/modules/BUILD.gn +++ b/webrtc/modules/BUILD.gn @@ -507,6 +507,7 @@ if (rtc_include_tests) { "video_coding/test/stream_generator.cc", "video_coding/test/stream_generator.h", "video_coding/timing_unittest.cc", + "video_coding/utility/default_video_bitrate_allocator_unittest.cc", "video_coding/utility/frame_dropper_unittest.cc", "video_coding/utility/ivf_file_writer_unittest.cc", "video_coding/utility/moving_average_unittest.cc", diff --git a/webrtc/modules/video_coding/BUILD.gn b/webrtc/modules/video_coding/BUILD.gn index 9bd746be75..8f413d9820 100644 --- a/webrtc/modules/video_coding/BUILD.gn +++ b/webrtc/modules/video_coding/BUILD.gn @@ -33,6 +33,7 @@ rtc_static_library("video_coding") { "h264_sps_pps_tracker.h", "histogram.cc", "histogram.h", + "include/video_codec_initializer.h", "include/video_coding.h", "include/video_coding_defines.h", "inter_frame_delay.cc", @@ -70,6 +71,7 @@ rtc_static_library("video_coding") { "timestamp_map.h", "timing.cc", "timing.h", + "video_codec_initializer.cc", "video_coding_impl.cc", "video_coding_impl.h", "video_receiver.cc", @@ -98,6 +100,8 @@ rtc_static_library("video_coding") { rtc_static_library("video_coding_utility") { sources = [ + "utility/default_video_bitrate_allocator.cc", + "utility/default_video_bitrate_allocator.h", "utility/frame_dropper.cc", "utility/frame_dropper.h", "utility/ivf_file_writer.cc", diff --git a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc index 5c0aa1bd45..d411b18bf8 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc +++ b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc @@ -152,6 +152,14 @@ static void RtpFragmentize(EncodedImage* encoded_image, H264EncoderImpl::H264EncoderImpl() : openh264_encoder_(nullptr), + width_(0), + height_(0), + max_frame_rate_(0.0f), + target_bps_(0), + max_bps_(0), + mode_(kRealtimeVideo), + frame_dropping_on_(false), + key_frame_interval_(0), number_of_cores_(0), encoded_image_callback_(nullptr), has_reported_init_(false), @@ -263,11 +271,13 @@ int32_t H264EncoderImpl::RegisterEncodeCompleteCallback( return WEBRTC_VIDEO_CODEC_OK; } -int32_t H264EncoderImpl::SetRates(uint32_t bitrate, uint32_t framerate) { - if (bitrate <= 0 || framerate <= 0) { +int32_t H264EncoderImpl::SetRateAllocation( + const BitrateAllocation& bitrate_allocation, + uint32_t framerate) { + if (bitrate_allocation.get_sum_bps() <= 0 || framerate <= 0) return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - } - target_bps_ = bitrate * 1000; + + target_bps_ = bitrate_allocation.get_sum_bps(); max_frame_rate_ = static_cast(framerate); quality_scaler_.ReportFramerate(framerate); diff --git a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h index 1e461b992f..ca28eb3250 100644 --- a/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h +++ b/webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.h @@ -44,7 +44,8 @@ class H264EncoderImpl : public H264Encoder { int32_t RegisterEncodeCompleteCallback( EncodedImageCallback* callback) override; - int32_t SetRates(uint32_t bitrate, uint32_t framerate) override; + int32_t SetRateAllocation(const BitrateAllocation& bitrate_allocation, + uint32_t framerate) override; // The result of encoding - an EncodedImage and RTPFragmentationHeader - are // passed to the encode complete callback. @@ -74,8 +75,8 @@ class H264EncoderImpl : public H264Encoder { int width_; int height_; float max_frame_rate_; - unsigned int target_bps_; - unsigned int max_bps_; + uint32_t target_bps_; + uint32_t max_bps_; VideoCodecMode mode_; // H.264 specifc parameters bool frame_dropping_on_; diff --git a/webrtc/modules/video_coding/codecs/i420/include/i420.h b/webrtc/modules/video_coding/codecs/i420/include/i420.h index 220aea4b86..38dcafd0c3 100644 --- a/webrtc/modules/video_coding/codecs/i420/include/i420.h +++ b/webrtc/modules/video_coding/codecs/i420/include/i420.h @@ -13,6 +13,7 @@ #include +#include "webrtc/base/checks.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/typedefs.h" @@ -65,10 +66,6 @@ class I420Encoder : public VideoEncoder { // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. int Release() override; - int SetRates(uint32_t /*newBitRate*/, uint32_t /*frameRate*/) override { - return WEBRTC_VIDEO_CODEC_OK; - } - int SetChannelParameters(uint32_t /*packetLoss*/, int64_t /*rtt*/) override { return WEBRTC_VIDEO_CODEC_OK; } diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor.cc b/webrtc/modules/video_coding/codecs/test/videoprocessor.cc index 8436eb0042..0449276b3f 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor.cc +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor.cc @@ -10,14 +10,18 @@ #include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" -#include #include #include #include +#include #include +#include "webrtc/base/checks.h" #include "webrtc/base/timeutils.h" +#include "webrtc/modules/video_coding/include/video_codec_initializer.h" +#include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h" +#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" #include "webrtc/system_wrappers/include/cpu_info.h" namespace webrtc { @@ -35,7 +39,7 @@ TestConfig::TestConfig() frame_length_in_bytes(0), use_single_core(false), keyframe_interval(0), - codec_settings(NULL), + codec_settings(nullptr), verbose(true) {} TestConfig::~TestConfig() {} @@ -54,8 +58,9 @@ VideoProcessorImpl::VideoProcessorImpl(webrtc::VideoEncoder* encoder, packet_manipulator_(packet_manipulator), config_(config), stats_(stats), - encode_callback_(NULL), - decode_callback_(NULL), + encode_callback_(nullptr), + decode_callback_(nullptr), + last_successful_frame_buffer_(nullptr), first_key_frame_has_been_excluded_(false), last_frame_missing_(false), initialized_(false), @@ -65,13 +70,23 @@ VideoProcessorImpl::VideoProcessorImpl(webrtc::VideoEncoder* encoder, num_dropped_frames_(0), num_spatial_resizes_(0), last_encoder_frame_width_(0), - last_encoder_frame_height_(0) { - assert(encoder); - assert(decoder); - assert(frame_reader); - assert(frame_writer); - assert(packet_manipulator); - assert(stats); + last_encoder_frame_height_(0), + bit_rate_factor_(0.0), + encode_start_ns_(0), + decode_start_ns_(0) { + std::unique_ptr tl_factory; + if (config_.codec_settings->codecType == VideoCodecType::kVideoCodecVP8) { + tl_factory.reset(new TemporalLayersFactory()); + config.codec_settings->VP8()->tl_factory = tl_factory.get(); + } + bitrate_allocator_ = VideoCodecInitializer::CreateBitrateAllocator( + *config.codec_settings, std::move(tl_factory)); + RTC_DCHECK(encoder); + RTC_DCHECK(decoder); + RTC_DCHECK(frame_reader); + RTC_DCHECK(frame_writer); + RTC_DCHECK(packet_manipulator); + RTC_DCHECK(stats); } bool VideoProcessorImpl::Init() { @@ -149,8 +164,10 @@ VideoProcessorImpl::~VideoProcessorImpl() { } void VideoProcessorImpl::SetRates(int bit_rate, int frame_rate) { - int set_rates_result = encoder_->SetRates(bit_rate, frame_rate); - assert(set_rates_result >= 0); + int set_rates_result = encoder_->SetRateAllocation( + bitrate_allocator_->GetAllocation(bit_rate * 1000, frame_rate), + frame_rate); + RTC_CHECK_GE(set_rates_result, 0); if (set_rates_result < 0) { fprintf(stderr, "Failed to update encoder with new rate %d, " @@ -178,7 +195,7 @@ int VideoProcessorImpl::NumberSpatialResizes() { } bool VideoProcessorImpl::ProcessFrame(int frame_number) { - assert(frame_number >= 0); + RTC_DCHECK_GE(frame_number, 0); if (!initialized_) { fprintf(stderr, "Attempting to use uninitialized VideoProcessor!\n"); return false; @@ -272,7 +289,7 @@ void VideoProcessorImpl::FrameEncoded( exclude_this_frame = true; break; default: - assert(false); + RTC_NOTREACHED(); } } @@ -341,11 +358,11 @@ void VideoProcessorImpl::FrameDecoded(const VideoFrame& image) { CalcBufferSize(kI420, up_image->width(), up_image->height()); std::unique_ptr image_buffer(new uint8_t[length]); int extracted_length = ExtractBuffer(up_image, length, image_buffer.get()); - assert(extracted_length > 0); + RTC_DCHECK_GT(extracted_length, 0); // Update our copy of the last successful frame: memcpy(last_successful_frame_buffer_, image_buffer.get(), extracted_length); bool write_success = frame_writer_->WriteFrame(image_buffer.get()); - assert(write_success); + RTC_DCHECK(write_success); if (!write_success) { fprintf(stderr, "Failed to write frame %d to disk!", frame_number); } @@ -355,11 +372,11 @@ void VideoProcessorImpl::FrameDecoded(const VideoFrame& image) { size_t length = CalcBufferSize(kI420, image.width(), image.height()); std::unique_ptr image_buffer(new uint8_t[length]); int extracted_length = ExtractBuffer(image, length, image_buffer.get()); - assert(extracted_length > 0); + RTC_DCHECK_GT(extracted_length, 0); memcpy(last_successful_frame_buffer_, image_buffer.get(), extracted_length); bool write_success = frame_writer_->WriteFrame(image_buffer.get()); - assert(write_success); + RTC_DCHECK(write_success); if (!write_success) { fprintf(stderr, "Failed to write frame %d to disk!", frame_number); } @@ -369,8 +386,8 @@ void VideoProcessorImpl::FrameDecoded(const VideoFrame& image) { int VideoProcessorImpl::GetElapsedTimeMicroseconds(int64_t start, int64_t stop) { uint64_t encode_time = (stop - start) / rtc::kNumNanosecsPerMicrosec; - assert(encode_time < - static_cast(std::numeric_limits::max())); + RTC_DCHECK_LT(encode_time, + static_cast(std::numeric_limits::max())); return static_cast(encode_time); } @@ -381,7 +398,7 @@ const char* ExcludeFrameTypesToStr(ExcludeFrameTypes e) { case kExcludeAllKeyFrames: return "ExcludeAllKeyFrames"; default: - assert(false); + RTC_NOTREACHED(); return "Unknown"; } } @@ -399,7 +416,7 @@ const char* VideoCodecTypeToStr(webrtc::VideoCodecType e) { case kVideoCodecUnknown: return "Unknown"; default: - assert(false); + RTC_NOTREACHED(); return "Unknown"; } } diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor.h b/webrtc/modules/video_coding/codecs/test/videoprocessor.h index 0887f4cc68..dba204f169 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor.h +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor.h @@ -11,6 +11,7 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_ #define WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_ +#include #include #include "webrtc/base/checks.h" @@ -23,6 +24,9 @@ #include "webrtc/video_frame.h" namespace webrtc { + +class VideoBitrateAllocator; + namespace test { // Defines which frame types shall be excluded from packet loss and when. @@ -191,6 +195,7 @@ class VideoProcessorImpl : public VideoProcessor { webrtc::VideoEncoder* encoder_; webrtc::VideoDecoder* decoder_; + std::unique_ptr bitrate_allocator_; FrameReader* frame_reader_; FrameWriter* frame_writer_; PacketManipulator* packet_manipulator_; diff --git a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc index 9c99a0a973..2adcdaebff 100644 --- a/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc +++ b/webrtc/modules/video_coding/codecs/test/videoprocessor_integrationtest.cc @@ -15,6 +15,7 @@ #include "webrtc/modules/video_coding/codecs/test/videoprocessor.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" +#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/include/video_coding.h" @@ -113,6 +114,7 @@ class VideoProcessorIntegrationTest : public testing::Test { webrtc::test::TestConfig config_; VideoCodec codec_settings_; webrtc::test::VideoProcessor* processor_; + TemporalLayersFactory tl_factory_; // Quantities defined/updated for every encoder rate update. // Some quantities defined per temporal layer (at most 3 layers in this test). diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc index e2ceb4a282..4f04743fc5 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -13,6 +13,10 @@ #include #include +#include +#include + +#include "webrtc/base/checks.h" #include "webrtc/modules/include/module_common_types.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" @@ -22,16 +26,17 @@ namespace webrtc { -DefaultTemporalLayers::DefaultTemporalLayers(int numberOfTemporalLayers, +DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers, uint8_t initial_tl0_pic_idx) - : number_of_temporal_layers_(numberOfTemporalLayers), + : number_of_temporal_layers_(number_of_temporal_layers), temporal_ids_length_(0), temporal_pattern_length_(0), tl0_pic_idx_(initial_tl0_pic_idx), pattern_idx_(255), timestamp_(0), last_base_layer_sync_(false) { - assert(kMaxTemporalStreams >= numberOfTemporalLayers); + RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers); + RTC_CHECK_GE(number_of_temporal_layers, 0); memset(temporal_ids_, 0, sizeof(temporal_ids_)); memset(temporal_pattern_, 0, sizeof(temporal_pattern_)); } @@ -43,18 +48,50 @@ int DefaultTemporalLayers::CurrentLayerId() const { return temporal_ids_[index]; } -bool DefaultTemporalLayers::ConfigureBitrates(int bitrateKbit, - int max_bitrate_kbit, - int framerate, - vpx_codec_enc_cfg_t* cfg) { +std::vector DefaultTemporalLayers::OnRatesUpdated( + int bitrate_kbps, + int max_bitrate_kbps, + int framerate) { + std::vector bitrates; + const int num_layers = std::max(1, number_of_temporal_layers_); + for (int i = 0; i < num_layers; ++i) { + float layer_bitrate = + bitrate_kbps * kVp8LayerRateAlloction[num_layers - 1][i]; + bitrates.push_back(static_cast(layer_bitrate + 0.5)); + } + new_bitrates_kbps_ = rtc::Optional>(bitrates); + + // Allocation table is of aggregates, transform to individual rates. + uint32_t sum = 0; + for (int i = 0; i < num_layers; ++i) { + uint32_t layer_bitrate = bitrates[i]; + RTC_DCHECK_LE(sum, bitrates[i]); + bitrates[i] -= sum; + sum = layer_bitrate; + + if (sum >= static_cast(bitrate_kbps)) { + // Sum adds up; any subsequent layers will be 0. + bitrates.resize(i + 1); + break; + } + } + + return bitrates; +} + +bool DefaultTemporalLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) { + if (!new_bitrates_kbps_) + return false; + switch (number_of_temporal_layers_) { case 0: + FALLTHROUGH(); case 1: temporal_ids_length_ = 1; temporal_ids_[0] = 0; cfg->ts_number_layers = number_of_temporal_layers_; cfg->ts_periodicity = temporal_ids_length_; - cfg->ts_target_bitrate[0] = bitrateKbit; + cfg->ts_target_bitrate[0] = (*new_bitrates_kbps_)[0]; cfg->ts_rate_decimator[0] = 1; memcpy(cfg->ts_layer_id, temporal_ids_, sizeof(unsigned int) * temporal_ids_length_); @@ -69,8 +106,8 @@ bool DefaultTemporalLayers::ConfigureBitrates(int bitrateKbit, cfg->ts_periodicity = temporal_ids_length_; // Split stream 60% 40%. // Bitrate API for VP8 is the agregated bitrate for all lower layers. - cfg->ts_target_bitrate[0] = bitrateKbit * kVp8LayerRateAlloction[1][0]; - cfg->ts_target_bitrate[1] = bitrateKbit; + cfg->ts_target_bitrate[0] = (*new_bitrates_kbps_)[0]; + cfg->ts_target_bitrate[1] = (*new_bitrates_kbps_)[1]; cfg->ts_rate_decimator[0] = 2; cfg->ts_rate_decimator[1] = 1; memcpy(cfg->ts_layer_id, temporal_ids_, @@ -95,9 +132,9 @@ bool DefaultTemporalLayers::ConfigureBitrates(int bitrateKbit, cfg->ts_periodicity = temporal_ids_length_; // Split stream 40% 20% 40%. // Bitrate API for VP8 is the agregated bitrate for all lower layers. - cfg->ts_target_bitrate[0] = bitrateKbit * kVp8LayerRateAlloction[2][0]; - cfg->ts_target_bitrate[1] = bitrateKbit * kVp8LayerRateAlloction[2][1]; - cfg->ts_target_bitrate[2] = bitrateKbit; + cfg->ts_target_bitrate[0] = (*new_bitrates_kbps_)[0]; + cfg->ts_target_bitrate[1] = (*new_bitrates_kbps_)[1]; + cfg->ts_target_bitrate[2] = (*new_bitrates_kbps_)[2]; cfg->ts_rate_decimator[0] = 4; cfg->ts_rate_decimator[1] = 2; cfg->ts_rate_decimator[2] = 1; @@ -127,10 +164,10 @@ bool DefaultTemporalLayers::ConfigureBitrates(int bitrateKbit, // Bitrate API for VP8 is the agregated bitrate for all lower layers. cfg->ts_number_layers = 4; cfg->ts_periodicity = temporal_ids_length_; - cfg->ts_target_bitrate[0] = bitrateKbit * kVp8LayerRateAlloction[3][0]; - cfg->ts_target_bitrate[1] = bitrateKbit * kVp8LayerRateAlloction[3][1]; - cfg->ts_target_bitrate[2] = bitrateKbit * kVp8LayerRateAlloction[3][2]; - cfg->ts_target_bitrate[3] = bitrateKbit; + cfg->ts_target_bitrate[0] = (*new_bitrates_kbps_)[0]; + cfg->ts_target_bitrate[1] = (*new_bitrates_kbps_)[1]; + cfg->ts_target_bitrate[2] = (*new_bitrates_kbps_)[2]; + cfg->ts_target_bitrate[3] = (*new_bitrates_kbps_)[3]; cfg->ts_rate_decimator[0] = 8; cfg->ts_rate_decimator[1] = 4; cfg->ts_rate_decimator[2] = 2; @@ -156,9 +193,12 @@ bool DefaultTemporalLayers::ConfigureBitrates(int bitrateKbit, temporal_pattern_[15] = kTemporalUpdateNone; break; default: - assert(false); + RTC_NOTREACHED(); return false; } + + new_bitrates_kbps_ = rtc::Optional>(); + return true; } @@ -284,8 +324,18 @@ void DefaultTemporalLayers::PopulateCodecSpecific( } TemporalLayers* TemporalLayersFactory::Create( + int simulcast_id, int temporal_layers, uint8_t initial_tl0_pic_idx) const { - return new DefaultTemporalLayers(temporal_layers, initial_tl0_pic_idx); + TemporalLayers* tl = + new DefaultTemporalLayers(temporal_layers, initial_tl0_pic_idx); + if (listener_) + listener_->OnTemporalLayersCreated(simulcast_id, tl); + return tl; } + +void TemporalLayersFactory::SetListener(TemporalLayersListener* listener) { + listener_ = listener; +} + } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h index 19846ba5ff..0cd0b06451 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -12,8 +12,12 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_ #define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_DEFAULT_TEMPORAL_LAYERS_H_ +#include + #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" +#include "webrtc/base/optional.h" + namespace webrtc { class DefaultTemporalLayers : public TemporalLayers { @@ -26,10 +30,13 @@ class DefaultTemporalLayers : public TemporalLayers { // and/or update the reference buffers. int EncodeFlags(uint32_t timestamp) override; - bool ConfigureBitrates(int bitrate_kbit, - int max_bitrate_kbit, - int framerate, - vpx_codec_enc_cfg_t* cfg) override; + // Update state based on new bitrate target and incoming framerate. + // Returns the bitrate allocation for the active temporal layers. + std::vector OnRatesUpdated(int bitrate_kbps, + int max_bitrate_kbps, + int framerate) override; + + bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override; void PopulateCodecSpecific(bool base_layer_sync, CodecSpecificInfoVP8* vp8_info, @@ -37,8 +44,6 @@ class DefaultTemporalLayers : public TemporalLayers { void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override {} - bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override { return false; } - int CurrentLayerId() const override; private: @@ -77,7 +82,7 @@ class DefaultTemporalLayers : public TemporalLayers { }; enum { kMaxTemporalPattern = 16 }; - int number_of_temporal_layers_; + const int number_of_temporal_layers_; int temporal_ids_length_; int temporal_ids_[kMaxTemporalPattern]; int temporal_pattern_length_; @@ -86,6 +91,7 @@ class DefaultTemporalLayers : public TemporalLayers { uint8_t pattern_idx_; uint32_t timestamp_; bool last_base_layer_sync_; + rtc::Optional> new_bitrates_kbps_; }; } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc index 1e36c44810..e22c8b4675 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc @@ -53,7 +53,8 @@ TEST(TemporalLayersTest, 2Layers) { DefaultTemporalLayers tl(2, 0); vpx_codec_enc_cfg_t cfg; CodecSpecificInfoVP8 vp8_info; - tl.ConfigureBitrates(500, 500, 30, &cfg); + tl.OnRatesUpdated(500, 500, 30); + tl.UpdateConfiguration(&cfg); int expected_flags[16] = { kTemporalUpdateLastAndGoldenRefAltRef, @@ -94,7 +95,8 @@ TEST(TemporalLayersTest, 3Layers) { DefaultTemporalLayers tl(3, 0); vpx_codec_enc_cfg_t cfg; CodecSpecificInfoVP8 vp8_info; - tl.ConfigureBitrates(500, 500, 30, &cfg); + tl.OnRatesUpdated(500, 500, 30); + tl.UpdateConfiguration(&cfg); int expected_flags[16] = { kTemporalUpdateLastAndGoldenRefAltRef, @@ -135,7 +137,8 @@ TEST(TemporalLayersTest, 4Layers) { DefaultTemporalLayers tl(4, 0); vpx_codec_enc_cfg_t cfg; CodecSpecificInfoVP8 vp8_info; - tl.ConfigureBitrates(500, 500, 30, &cfg); + tl.OnRatesUpdated(500, 500, 30); + tl.UpdateConfiguration(&cfg); int expected_flags[16] = { kTemporalUpdateLast, kTemporalUpdateNone, @@ -175,7 +178,8 @@ TEST(TemporalLayersTest, KeyFrame) { DefaultTemporalLayers tl(3, 0); vpx_codec_enc_cfg_t cfg; CodecSpecificInfoVP8 vp8_info; - tl.ConfigureBitrates(500, 500, 30, &cfg); + tl.OnRatesUpdated(500, 500, 30); + tl.UpdateConfiguration(&cfg); int expected_flags[8] = { kTemporalUpdateLastAndGoldenRefAltRef, diff --git a/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc b/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc index b9721cde1b..a3e65e4ce0 100644 --- a/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/realtime_temporal_layers.cc @@ -12,6 +12,8 @@ #include "vpx/vpx_encoder.h" #include "vpx/vp8cx.h" +#include "webrtc/base/checks.h" +#include "webrtc/base/optional.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8_common_types.h" #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" @@ -95,26 +97,20 @@ class RealTimeTemporalLayers : public TemporalLayers { layer_ids_(NULL), encode_flags_length_(0), encode_flags_(NULL) { - assert(max_temporal_layers_ >= 1); - assert(max_temporal_layers_ <= 3); + RTC_CHECK_GE(max_temporal_layers_, 1); + RTC_CHECK_GE(max_temporal_layers_, 3); } virtual ~RealTimeTemporalLayers() {} - bool ConfigureBitrates(int bitrate_kbit, - int max_bitrate_kbit, - int framerate, - vpx_codec_enc_cfg_t* cfg) override { + std::vector OnRatesUpdated(int bitrate_kbps, + int max_bitrate_kbps, + int framerate) override { temporal_layers_ = CalculateNumberOfTemporalLayers(temporal_layers_, framerate); temporal_layers_ = std::min(temporal_layers_, max_temporal_layers_); - assert(temporal_layers_ >= 1 && temporal_layers_ <= 3); - - cfg->ts_number_layers = temporal_layers_; - for (int tl = 0; tl < temporal_layers_; ++tl) { - cfg->ts_target_bitrate[tl] = - bitrate_kbit * kVp8LayerRateAlloction[temporal_layers_ - 1][tl]; - } + RTC_CHECK_GE(temporal_layers_, 1); + RTC_CHECK_LE(temporal_layers_, 3); switch (temporal_layers_) { case 1: { @@ -125,9 +121,6 @@ class RealTimeTemporalLayers : public TemporalLayers { static const int encode_flags[] = {kTemporalUpdateLastRefAll}; encode_flags_length_ = sizeof(encode_flags) / sizeof(*layer_ids); encode_flags_ = encode_flags; - - cfg->ts_rate_decimator[0] = 1; - cfg->ts_periodicity = layer_ids_length_; } break; case 2: { @@ -146,10 +139,6 @@ class RealTimeTemporalLayers : public TemporalLayers { kTemporalUpdateNone}; encode_flags_length_ = sizeof(encode_flags) / sizeof(*layer_ids); encode_flags_ = encode_flags; - - cfg->ts_rate_decimator[0] = 2; - cfg->ts_rate_decimator[1] = 1; - cfg->ts_periodicity = layer_ids_length_; } break; case 3: { @@ -168,19 +157,59 @@ class RealTimeTemporalLayers : public TemporalLayers { kTemporalUpdateNone}; encode_flags_length_ = sizeof(encode_flags) / sizeof(*layer_ids); encode_flags_ = encode_flags; - - cfg->ts_rate_decimator[0] = 4; - cfg->ts_rate_decimator[1] = 2; - cfg->ts_rate_decimator[2] = 1; - cfg->ts_periodicity = layer_ids_length_; } break; default: - assert(false); - return false; + RTC_NOTREACHED(); + return std::vector(); } + + std::vector bitrates; + const int num_layers = std::max(1, temporal_layers_); + for (int i = 0; i < num_layers; ++i) { + float layer_bitrate = + bitrate_kbps * kVp8LayerRateAlloction[num_layers - 1][i]; + bitrates.push_back(static_cast(layer_bitrate + 0.5)); + } + new_bitrates_kbps_ = rtc::Optional>(bitrates); + + // Allocation table is of aggregates, transform to individual rates. + uint32_t sum = 0; + for (int i = 0; i < num_layers; ++i) { + uint32_t layer_bitrate = bitrates[i]; + RTC_DCHECK_LE(sum, bitrates[i]); + bitrates[i] -= sum; + sum += layer_bitrate; + + if (sum == static_cast(bitrate_kbps)) { + // Sum adds up; any subsequent layers will be 0. + bitrates.resize(i); + break; + } + } + + return bitrates; + } + + bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override { + if (!new_bitrates_kbps_) + return false; + + cfg->ts_number_layers = temporal_layers_; + for (int tl = 0; tl < temporal_layers_; ++tl) { + cfg->ts_target_bitrate[tl] = (*new_bitrates_kbps_)[tl]; + } + new_bitrates_kbps_ = rtc::Optional>(); + + cfg->ts_periodicity = layer_ids_length_; + int decimator = 1; + for (int i = temporal_layers_ - 1; i >= 0; --i, decimator *= 2) { + cfg->ts_rate_decimator[i] = decimator; + } + memcpy(cfg->ts_layer_id, layer_ids_, sizeof(unsigned int) * layer_ids_length_); + return true; } @@ -248,8 +277,6 @@ class RealTimeTemporalLayers : public TemporalLayers { void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override {} - bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override { return false; } - private: int temporal_layers_; int max_temporal_layers_; @@ -266,12 +293,19 @@ class RealTimeTemporalLayers : public TemporalLayers { // Pattern of encode flags. int encode_flags_length_; const int* encode_flags_; + + rtc::Optional> new_bitrates_kbps_; }; } // namespace TemporalLayers* RealTimeTemporalLayersFactory::Create( + int simulcast_id, int max_temporal_layers, uint8_t initial_tl0_pic_idx) const { - return new RealTimeTemporalLayers(max_temporal_layers, initial_tl0_pic_idx); + TemporalLayers* tl = + new RealTimeTemporalLayers(max_temporal_layers, initial_tl0_pic_idx); + if (listener_) + listener_->OnTemporalLayersCreated(simulcast_id, tl); + return tl; } } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc index e821497440..8c7fd5466d 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -65,7 +65,8 @@ ScreenshareLayers::ScreenshareLayers(int num_temporal_layers, min_qp_(-1), max_qp_(-1), max_debt_bytes_(0), - frame_rate_(-1) { + framerate_(-1), + bitrate_updated_(false) { RTC_CHECK_GT(num_temporal_layers, 0); RTC_CHECK_LE(num_temporal_layers, 2); } @@ -136,7 +137,7 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { int64_t ts_diff; if (last_timestamp_ == -1) { - ts_diff = kOneSecond90Khz / (frame_rate_ <= 0 ? 5 : frame_rate_); + ts_diff = kOneSecond90Khz / (framerate_ <= 0 ? 5 : framerate_); } else { ts_diff = unwrapped_timestamp - last_timestamp_; } @@ -147,47 +148,19 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { return flags; } -bool ScreenshareLayers::ConfigureBitrates(int bitrate_kbps, - int max_bitrate_kbps, - int framerate, - vpx_codec_enc_cfg_t* cfg) { +std::vector ScreenshareLayers::OnRatesUpdated(int bitrate_kbps, + int max_bitrate_kbps, + int framerate) { layers_[0].target_rate_kbps_ = bitrate_kbps; layers_[1].target_rate_kbps_ = max_bitrate_kbps; + framerate_ = framerate; + bitrate_updated_ = true; - int target_bitrate_kbps = bitrate_kbps; - - if (cfg != nullptr) { - if (number_of_temporal_layers_ > 1) { - // Calculate a codec target bitrate. This may be higher than TL0, gaining - // quality at the expense of frame rate at TL0. Constraints: - // - TL0 frame rate no less than framerate / kMaxTL0FpsReduction. - // - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate. - target_bitrate_kbps = - std::min(bitrate_kbps * kMaxTL0FpsReduction, - max_bitrate_kbps / kAcceptableTargetOvershoot); - - cfg->rc_target_bitrate = std::max(bitrate_kbps, target_bitrate_kbps); - } - - // Don't reconfigure qp limits during quality boost frames. - if (active_layer_ == -1 || - layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) { - min_qp_ = cfg->rc_min_quantizer; - max_qp_ = cfg->rc_max_quantizer; - // After a dropped frame, a frame with max qp will be encoded and the - // quality will then ramp up from there. To boost the speed of recovery, - // encode the next frame with lower max qp. TL0 is the most important to - // improve since the errors in this layer will propagate to TL1. - // Currently, reduce max qp by 20% for TL0 and 15% for TL1. - layers_[0].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 80) / 100); - layers_[1].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 85) / 100); - } - } - - int avg_frame_size = (target_bitrate_kbps * 1000) / (8 * framerate); - max_debt_bytes_ = 4 * avg_frame_size; - - return true; + std::vector allocation; + allocation.push_back(bitrate_kbps); + if (max_bitrate_kbps > bitrate_kbps) + allocation.push_back(max_bitrate_kbps - bitrate_kbps); + return allocation; } void ScreenshareLayers::FrameEncoded(unsigned int size, @@ -279,9 +252,49 @@ bool ScreenshareLayers::TimeToSync(int64_t timestamp) const { } bool ScreenshareLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) { + if (bitrate_updated_) { + uint32_t target_bitrate_kbps = layers_[0].target_rate_kbps_; + + if (number_of_temporal_layers_ > 1) { + // Calculate a codec target bitrate. This may be higher than TL0, gaining + // quality at the expense of frame rate at TL0. Constraints: + // - TL0 frame rate no less than framerate / kMaxTL0FpsReduction. + // - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate. + target_bitrate_kbps = + std::min(layers_[0].target_rate_kbps_ * kMaxTL0FpsReduction, + layers_[1].target_rate_kbps_ / kAcceptableTargetOvershoot); + + cfg->rc_target_bitrate = + std::max(layers_[0].target_rate_kbps_, target_bitrate_kbps); + } + + // Don't reconfigure qp limits during quality boost frames. + if (active_layer_ == -1 || + layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) { + min_qp_ = cfg->rc_min_quantizer; + max_qp_ = cfg->rc_max_quantizer; + // After a dropped frame, a frame with max qp will be encoded and the + // quality will then ramp up from there. To boost the speed of recovery, + // encode the next frame with lower max qp. TL0 is the most important to + // improve since the errors in this layer will propagate to TL1. + // Currently, reduce max qp by 20% for TL0 and 15% for TL1. + layers_[0].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 80) / 100); + layers_[1].enhanced_max_qp = min_qp_ + (((max_qp_ - min_qp_) * 85) / 100); + } + + int avg_frame_size = (target_bitrate_kbps * 1000) / (8 * framerate_); + max_debt_bytes_ = 4 * avg_frame_size; + + bitrate_updated_ = false; + + // Don't try to update boosts state if not active yet. + if (active_layer_ == -1) + return true; + } + + RTC_DCHECK_NE(-1, active_layer_); if (max_qp_ == -1 || number_of_temporal_layers_ <= 1) return false; - RTC_DCHECK_NE(-1, active_layer_); // If layer is in the quality boost state (following a dropped frame), update // the configuration with the adjusted (lower) qp and set the state back to diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h index 8392bc4f35..e4d3c76aab 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -9,9 +9,7 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_ #define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_ -#include - -#include "vpx/vpx_encoder.h" +#include #include "webrtc/base/timeutils.h" #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" @@ -41,10 +39,15 @@ class ScreenshareLayers : public TemporalLayers { // and/or update the reference buffers. int EncodeFlags(uint32_t timestamp) override; - bool ConfigureBitrates(int bitrate_kbps, - int max_bitrate_kbps, - int framerate, - vpx_codec_enc_cfg_t* cfg) override; + // Update state based on new bitrate target and incoming framerate. + // Returns the bitrate allocation for the active temporal layers. + std::vector OnRatesUpdated(int bitrate_kbps, + int max_bitrate_kbps, + int framerate) override; + + // Update the encoder configuration with target bitrates or other parameters. + // Returns true iff the configuration was actually modified. + bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override; void PopulateCodecSpecific(bool base_layer_sync, CodecSpecificInfoVP8* vp8_info, @@ -54,11 +57,6 @@ class ScreenshareLayers : public TemporalLayers { int CurrentLayerId() const override; - // Allows the layers adapter to update the encoder configuration prior to a - // frame being encoded. Return true if the configuration should be updated - // and false if now change is needed. - bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override; - private: bool TimeToSync(int64_t timestamp) const; @@ -75,7 +73,8 @@ class ScreenshareLayers : public TemporalLayers { int min_qp_; int max_qp_; uint32_t max_debt_bytes_; - int frame_rate_; + int framerate_; + bool bitrate_updated_; static const int kMaxNumTemporalLayers = 2; struct TemporalLayer { diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc index e6316d6af9..469cc1524c 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc @@ -22,6 +22,7 @@ #include "webrtc/test/gtest.h" using ::testing::_; +using ::testing::ElementsAre; using ::testing::NiceMock; using ::testing::Return; @@ -61,8 +62,11 @@ class ScreenshareLayerTest : public ::testing::Test { memset(&vpx_cfg, 0, sizeof(vpx_codec_enc_cfg_t)); vpx_cfg.rc_min_quantizer = min_qp_; vpx_cfg.rc_max_quantizer = max_qp_; - EXPECT_TRUE(layers_->ConfigureBitrates( - kDefaultTl0BitrateKbps, kDefaultTl1BitrateKbps, kFrameRate, &vpx_cfg)); + EXPECT_THAT(layers_->OnRatesUpdated(kDefaultTl0BitrateKbps, + kDefaultTl1BitrateKbps, kFrameRate), + ElementsAre(kDefaultTl0BitrateKbps, + kDefaultTl1BitrateKbps - kDefaultTl0BitrateKbps)); + EXPECT_TRUE(layers_->UpdateConfiguration(&vpx_cfg)); frame_size_ = ((vpx_cfg.rc_target_bitrate * 1000) / 8) / kFrameRate; } @@ -373,27 +377,41 @@ TEST_F(ScreenshareLayerTest, TooHighBitrate) { TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL0) { vpx_codec_enc_cfg_t cfg = GetConfig(); - layers_->ConfigureBitrates(100, 1000, 5, &cfg); + const int kTl0_kbps = 100; + const int kTl1_kbps = 1000; + layers_->OnRatesUpdated(kTl0_kbps, kTl1_kbps, 5); + + EXPECT_THAT(layers_->OnRatesUpdated(kTl0_kbps, kTl1_kbps, 5), + ElementsAre(kTl0_kbps, kTl1_kbps - kTl0_kbps)); + EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); EXPECT_EQ(static_cast( - ScreenshareLayers::kMaxTL0FpsReduction * 100 + 0.5), + ScreenshareLayers::kMaxTL0FpsReduction * kTl0_kbps + 0.5), cfg.rc_target_bitrate); } TEST_F(ScreenshareLayerTest, TargetBitrateCappedByTL1) { vpx_codec_enc_cfg_t cfg = GetConfig(); - layers_->ConfigureBitrates(100, 450, 5, &cfg); + const int kTl0_kbps = 100; + const int kTl1_kbps = 450; + EXPECT_THAT(layers_->OnRatesUpdated(kTl0_kbps, kTl1_kbps, 5), + ElementsAre(kTl0_kbps, kTl1_kbps - kTl0_kbps)); + EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); EXPECT_EQ(static_cast( - 450 / ScreenshareLayers::kAcceptableTargetOvershoot), + kTl1_kbps / ScreenshareLayers::kAcceptableTargetOvershoot), cfg.rc_target_bitrate); } TEST_F(ScreenshareLayerTest, TargetBitrateBelowTL0) { vpx_codec_enc_cfg_t cfg = GetConfig(); - layers_->ConfigureBitrates(100, 100, 5, &cfg); + const int kTl0_kbps = 100; + const int kTl1_kbps = 100; + EXPECT_THAT(layers_->OnRatesUpdated(kTl0_kbps, kTl1_kbps, 5), + ElementsAre(kTl0_kbps)); + EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); - EXPECT_EQ(100U, cfg.rc_target_bitrate); + EXPECT_EQ(static_cast(kTl1_kbps), cfg.rc_target_bitrate); } TEST_F(ScreenshareLayerTest, EncoderDrop) { @@ -453,7 +471,8 @@ TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) { const uint32_t kStartTimestamp = 1234; vpx_codec_enc_cfg_t cfg = GetConfig(); - layers_->ConfigureBitrates(kLowBitrateKbps, kLowBitrateKbps, 5, &cfg); + layers_->OnRatesUpdated(kLowBitrateKbps, kLowBitrateKbps, 5); + layers_->UpdateConfiguration(&cfg); EXPECT_EQ(ScreenshareLayers::kTl0Flags, layers_->EncodeFlags(kStartTimestamp)); diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc index 880f45fa7e..a58bdac6cf 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc @@ -83,17 +83,6 @@ int VerifyCodec(const webrtc::VideoCodec* inst) { return WEBRTC_VIDEO_CODEC_OK; } -struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory { - ScreenshareTemporalLayersFactory() {} - virtual ~ScreenshareTemporalLayersFactory() {} - - virtual webrtc::TemporalLayers* Create(int num_temporal_layers, - uint8_t initial_tl0_pic_idx) const { - return new webrtc::ScreenshareLayers(num_temporal_layers, rand(), - webrtc::Clock::GetRealTimeClock()); - } -}; - // An EncodedImageCallback implementation that forwards on calls to a // SimulcastEncoderAdapter, but with the stream index it's registered with as // the first parameter to Encoded. @@ -116,6 +105,25 @@ class AdapterEncodedImageCallback : public webrtc::EncodedImageCallback { const size_t stream_idx_; }; +// Utility class used to adapt the simulcast id as reported by the temporal +// layers factory, since each sub-encoder will report stream 0. +class TemporalLayersFactoryAdapter : public webrtc::TemporalLayersFactory { + public: + TemporalLayersFactoryAdapter(int adapted_simulcast_id, + const TemporalLayersFactory& tl_factory) + : adapted_simulcast_id_(adapted_simulcast_id), tl_factory_(tl_factory) {} + ~TemporalLayersFactoryAdapter() override {} + webrtc::TemporalLayers* Create(int simulcast_id, + int temporal_layers, + uint8_t initial_tl0_pic_idx) const override { + return tl_factory_.Create(adapted_simulcast_id_, temporal_layers, + initial_tl0_pic_idx); + } + + const int adapted_simulcast_id_; + const TemporalLayersFactory& tl_factory_; +}; + } // namespace namespace webrtc { @@ -125,7 +133,6 @@ SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory) encoded_complete_callback_(nullptr), implementation_name_("SimulcastEncoderAdapter") { memset(&codec_, 0, sizeof(webrtc::VideoCodec)); - rate_allocator_.reset(new SimulcastRateAllocator(codec_)); } SimulcastEncoderAdapter::~SimulcastEncoderAdapter() { @@ -173,14 +180,13 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst, } codec_ = *inst; - rate_allocator_.reset(new SimulcastRateAllocator(codec_)); - std::vector start_bitrates = - rate_allocator_->GetAllocation(codec_.startBitrate); - - // Special mode when screensharing on a single stream. - if (number_of_streams == 1 && inst->mode == kScreensharing) { - screensharing_tl_factory_.reset(new ScreenshareTemporalLayersFactory()); - codec_.VP8()->tl_factory = screensharing_tl_factory_.get(); + SimulcastRateAllocator rate_allocator(codec_, nullptr); + BitrateAllocation allocation = rate_allocator.GetAllocation( + codec_.startBitrate * 1000, codec_.maxFramerate); + std::vector start_bitrates; + for (int i = 0; i < kMaxSimulcastStreams; ++i) { + uint32_t stream_bitrate = allocation.GetSpatialLayerSum(i) / 1000; + start_bitrates.push_back(stream_bitrate); } std::string implementation_name; @@ -200,6 +206,9 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst, PopulateStreamCodec(&codec_, i, start_bitrate_kbps, highest_resolution_stream, &stream_codec); } + TemporalLayersFactoryAdapter tl_factory_adapter( + i, *codec_.codecSpecific.VP8.tl_factory); + stream_codec.codecSpecific.VP8.tl_factory = &tl_factory_adapter; // TODO(ronghuawu): Remove once this is handled in VP8EncoderImpl. if (stream_codec.qpMax < kDefaultMinQp) { @@ -340,61 +349,48 @@ int SimulcastEncoderAdapter::SetChannelParameters(uint32_t packet_loss, return WEBRTC_VIDEO_CODEC_OK; } -int SimulcastEncoderAdapter::SetRates(uint32_t new_bitrate_kbit, - uint32_t new_framerate) { - if (!Initialized()) { +int SimulcastEncoderAdapter::SetRateAllocation(const BitrateAllocation& bitrate, + uint32_t new_framerate) { + if (!Initialized()) return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } - if (new_framerate < 1) { - return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; - } - if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate) { - new_bitrate_kbit = codec_.maxBitrate; - } - std::vector stream_bitrates; - if (new_bitrate_kbit > 0) { + if (new_framerate < 1) + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + + if (codec_.maxBitrate > 0 && bitrate.get_sum_kbps() > codec_.maxBitrate) + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + + if (bitrate.get_sum_bps() > 0) { // Make sure the bitrate fits the configured min bitrates. 0 is a special // value that means paused, though, so leave it alone. - if (new_bitrate_kbit < codec_.minBitrate) { - new_bitrate_kbit = codec_.minBitrate; - } + if (bitrate.get_sum_kbps() < codec_.minBitrate) + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + if (codec_.numberOfSimulcastStreams > 0 && - new_bitrate_kbit < codec_.simulcastStream[0].minBitrate) { - new_bitrate_kbit = codec_.simulcastStream[0].minBitrate; + bitrate.get_sum_kbps() < codec_.simulcastStream[0].minBitrate) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } - stream_bitrates = rate_allocator_->GetAllocation(new_bitrate_kbit); } + codec_.maxFramerate = new_framerate; - // Disable any stream not in the current allocation. - stream_bitrates.resize(streaminfos_.size(), 0U); - for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) { - uint32_t stream_bitrate_kbps = stream_bitrates[stream_idx]; + uint32_t stream_bitrate_kbps = + bitrate.GetSpatialLayerSum(stream_idx) / 1000; + // Need a key frame if we have not sent this stream before. if (stream_bitrate_kbps > 0 && !streaminfos_[stream_idx].send_stream) { streaminfos_[stream_idx].key_frame_request = true; } streaminfos_[stream_idx].send_stream = stream_bitrate_kbps > 0; - // TODO(holmer): This is a temporary hack for screensharing, where we - // interpret the startBitrate as the encoder target bitrate. This is - // to allow for a different max bitrate, so if the codec can't meet - // the target we still allow it to overshoot up to the max before dropping - // frames. This hack should be improved. - if (codec_.targetBitrate > 0 && - (codec_.VP8()->numberOfTemporalLayers == 2 || - codec_.simulcastStream[0].numberOfTemporalLayers == 2)) { - stream_bitrate_kbps = std::min(codec_.maxBitrate, stream_bitrate_kbps); - // TODO(ronghuawu): Can't change max bitrate via the VideoEncoder - // interface. And VP8EncoderImpl doesn't take negative framerate. - // max_bitrate = std::min(codec_.maxBitrate, stream_bitrate_kbps); - // new_framerate = -1; - } - - streaminfos_[stream_idx].encoder->SetRates(stream_bitrate_kbps, - new_framerate); + // Slice the temporal layers out of the full allocation and pass it on to + // the encoder handling the current simulcast stream. + BitrateAllocation stream_allocation; + for (int i = 0; i < kMaxTemporalStreams; ++i) + stream_allocation.SetBitrate(0, i, bitrate.GetBitrate(stream_idx, i)); + streaminfos_[stream_idx].encoder->SetRateAllocation(stream_allocation, + new_framerate); } return WEBRTC_VIDEO_CODEC_OK; diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h index 30d5abb6cc..b76e7f4ce9 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h @@ -48,7 +48,8 @@ class SimulcastEncoderAdapter : public VP8Encoder { const std::vector* frame_types) override; int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override; int SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; - int SetRates(uint32_t new_bitrate_kbit, uint32_t new_framerate) override; + int SetRateAllocation(const BitrateAllocation& bitrate, + uint32_t new_framerate) override; // Eventual handler for the contained encoders' EncodedImageCallbacks, but // called from an internal helper that also knows the correct stream @@ -103,12 +104,10 @@ class SimulcastEncoderAdapter : public VP8Encoder { bool Initialized() const; std::unique_ptr factory_; - std::unique_ptr screensharing_tl_factory_; VideoCodec codec_; std::vector streaminfos_; EncodedImageCallback* encoded_complete_callback_; std::string implementation_name_; - std::unique_ptr rate_allocator_; }; } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter_unittest.cc index 9a06c0ac5f..678b150db2 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter_unittest.cc @@ -95,12 +95,6 @@ TEST_F(TestSimulcastEncoderAdapter, TestSpatioTemporalLayers321PatternEncoder) { TestVp8Simulcast::TestSpatioTemporalLayers321PatternEncoder(); } -// TODO(ronghuawu): Enable this test when SkipEncodingUnusedStreams option is -// implemented for SimulcastEncoderAdapter. -TEST_F(TestSimulcastEncoderAdapter, DISABLED_TestSkipEncodingUnusedStreams) { - TestVp8Simulcast::TestSkipEncodingUnusedStreams(); -} - TEST_F(TestSimulcastEncoderAdapter, DISABLED_TestRPSIEncoder) { TestVp8Simulcast::TestRPSIEncoder(); } @@ -132,8 +126,9 @@ class MockVideoEncoder : public VideoEncoder { int32_t Release() /* override */ { return 0; } - int32_t SetRates(uint32_t newBitRate, uint32_t frameRate) /* override */ { - last_set_bitrate_ = static_cast(newBitRate); + int32_t SetRateAllocation(const BitrateAllocation& bitrate_allocation, + uint32_t framerate) { + last_set_bitrate_ = bitrate_allocation; return 0; } @@ -160,13 +155,13 @@ class MockVideoEncoder : public VideoEncoder { void set_supports_native_handle(bool enabled) { supports_native_handle_ = enabled; } - int32_t last_set_bitrate() const { return last_set_bitrate_; } + BitrateAllocation last_set_bitrate() const { return last_set_bitrate_; } MOCK_CONST_METHOD0(ImplementationName, const char*()); private: bool supports_native_handle_ = false; - int32_t last_set_bitrate_ = -1; + BitrateAllocation last_set_bitrate_; VideoCodec codec_; EncodedImageCallback* callback_; @@ -273,6 +268,9 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test, void SetupCodec() { TestVp8Simulcast::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile)); + rate_allocator_.reset(new SimulcastRateAllocator(codec_, nullptr)); + tl_factory_.SetListener(rate_allocator_.get()); + codec_.codecSpecific.VP8.tl_factory = &tl_factory_; EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200)); adapter_->RegisterEncodeCompleteCallback(this); } @@ -301,7 +299,6 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test, EXPECT_EQ(ref.VP8().automaticResizeOn, target.VP8().automaticResizeOn); EXPECT_EQ(ref.VP8().frameDroppingOn, target.VP8().frameDroppingOn); EXPECT_EQ(ref.VP8().keyFrameInterval, target.VP8().keyFrameInterval); - EXPECT_EQ(ref.VP8().tl_factory, target.VP8().tl_factory); EXPECT_EQ(ref.qpMax, target.qpMax); EXPECT_EQ(0, target.numberOfSimulcastStreams); EXPECT_EQ(ref.mode, target.mode); @@ -314,6 +311,7 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test, *ref_codec = codec_; ref_codec->VP8()->numberOfTemporalLayers = kTestTemporalLayerProfile[stream_index]; + ref_codec->codecSpecific.VP8.tl_factory = &tl_factory_; ref_codec->width = codec_.simulcastStream[stream_index].width; ref_codec->height = codec_.simulcastStream[stream_index].height; ref_codec->maxBitrate = codec_.simulcastStream[stream_index].maxBitrate; @@ -357,6 +355,8 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test, int last_encoded_image_width_; int last_encoded_image_height_; int last_encoded_image_simulcast_index_; + TemporalLayersFactory tl_factory_; + std::unique_ptr rate_allocator_; }; TEST_F(TestSimulcastEncoderAdapterFake, InitEncode) { @@ -376,7 +376,7 @@ TEST_F(TestSimulcastEncoderAdapterFake, EncodedCallbackForDifferentEncoders) { SetupCodec(); // Set bitrates so that we send all layers. - adapter_->SetRates(1200, 30); + adapter_->SetRateAllocation(rate_allocator_->GetAllocation(1200, 30), 30); // At this point, the simulcast encoder adapter should have 3 streams: HD, // quarter HD, and quarter quarter HD. We're going to mostly ignore the exact @@ -409,6 +409,7 @@ TEST_F(TestSimulcastEncoderAdapterFake, EncodedCallbackForDifferentEncoders) { TEST_F(TestSimulcastEncoderAdapterFake, SupportsNativeHandleForSingleStreams) { TestVp8Simulcast::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile)); + codec_.codecSpecific.VP8.tl_factory = &tl_factory_; codec_.numberOfSimulcastStreams = 1; EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200)); adapter_->RegisterEncodeCompleteCallback(this); @@ -422,27 +423,37 @@ TEST_F(TestSimulcastEncoderAdapterFake, SupportsNativeHandleForSingleStreams) { TEST_F(TestSimulcastEncoderAdapterFake, SetRatesUnderMinBitrate) { TestVp8Simulcast::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile)); + codec_.codecSpecific.VP8.tl_factory = &tl_factory_; codec_.minBitrate = 50; codec_.numberOfSimulcastStreams = 1; EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200)); + rate_allocator_.reset(new SimulcastRateAllocator(codec_, nullptr)); // Above min should be respected. - adapter_->SetRates(100, 30); - EXPECT_EQ(100, helper_->factory()->encoders()[0]->last_set_bitrate()); + BitrateAllocation target_bitrate = + rate_allocator_->GetAllocation(codec_.minBitrate * 1000, 30); + adapter_->SetRateAllocation(target_bitrate, 30); + EXPECT_EQ(target_bitrate, + helper_->factory()->encoders()[0]->last_set_bitrate()); // Below min but non-zero should be replaced with the min bitrate. - adapter_->SetRates(15, 30); - EXPECT_EQ(50, helper_->factory()->encoders()[0]->last_set_bitrate()); + BitrateAllocation too_low_bitrate = + rate_allocator_->GetAllocation((codec_.minBitrate - 1) * 1000, 30); + adapter_->SetRateAllocation(too_low_bitrate, 30); + EXPECT_EQ(target_bitrate, + helper_->factory()->encoders()[0]->last_set_bitrate()); // Zero should be passed on as is, since it means "pause". - adapter_->SetRates(0, 30); - EXPECT_EQ(0, helper_->factory()->encoders()[0]->last_set_bitrate()); + adapter_->SetRateAllocation(BitrateAllocation(), 30); + EXPECT_EQ(BitrateAllocation(), + helper_->factory()->encoders()[0]->last_set_bitrate()); } TEST_F(TestSimulcastEncoderAdapterFake, SupportsImplementationName) { EXPECT_STREQ("SimulcastEncoderAdapter", adapter_->ImplementationName()); TestVp8Simulcast::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile)); + codec_.codecSpecific.VP8.tl_factory = &tl_factory_; std::vector encoder_names; encoder_names.push_back("codec1"); encoder_names.push_back("codec2"); @@ -465,6 +476,7 @@ TEST_F(TestSimulcastEncoderAdapterFake, SupportsNativeHandleForMultipleStreams) { TestVp8Simulcast::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile)); + codec_.codecSpecific.VP8.tl_factory = &tl_factory_; codec_.numberOfSimulcastStreams = 3; EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200)); adapter_->RegisterEncodeCompleteCallback(this); @@ -493,6 +505,7 @@ TEST_F(TestSimulcastEncoderAdapterFake, NativeHandleForwardingForMultipleStreams) { TestVp8Simulcast::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile)); + codec_.codecSpecific.VP8.tl_factory = &tl_factory_; codec_.numberOfSimulcastStreams = 3; // High start bitrate, so all streams are enabled. codec_.startBitrate = 3000; @@ -517,6 +530,7 @@ TEST_F(TestSimulcastEncoderAdapterFake, TEST_F(TestSimulcastEncoderAdapterFake, TestFailureReturnCodesFromEncodeCalls) { TestVp8Simulcast::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile)); + codec_.codecSpecific.VP8.tl_factory = &tl_factory_; codec_.numberOfSimulcastStreams = 3; EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200)); adapter_->RegisterEncodeCompleteCallback(this); diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.cc index 9d57dc0996..d604c64816 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.cc @@ -86,10 +86,5 @@ TEST_F(TestVp8Impl, TestSpatioTemporalLayers321PatternEncoder) { TEST_F(TestVp8Impl, TestStrideEncodeDecode) { TestVp8Simulcast::TestStrideEncodeDecode(); } - -TEST_F(TestVp8Impl, TestSkipEncodingUnusedStreams) { - TestVp8Simulcast::TestSkipEncodingUnusedStreams(); -} - } // namespace testing } // namespace webrtc diff --git a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h index 56ad3a710d..32481cb152 100644 --- a/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h +++ b/webrtc/modules/video_coding/codecs/vp8/simulcast_unittest.h @@ -12,6 +12,7 @@ #define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_SIMULCAST_UNITTEST_H_ #include +#include #include #include @@ -20,6 +21,7 @@ #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" #include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/modules/video_coding/include/mock/mock_video_codec_interface.h" +#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" #include "webrtc/test/gtest.h" #include "webrtc/video_frame.h" @@ -147,83 +149,6 @@ class Vp8TestDecodedImageCallback : public DecodedImageCallback { int decoded_frames_; }; -class SkipEncodingUnusedStreamsTest { - public: - std::vector RunTest(VP8Encoder* encoder, - VideoCodec* settings, - uint32_t target_bitrate) { - SpyingTemporalLayersFactory spy_factory; - settings->VP8()->tl_factory = &spy_factory; - EXPECT_EQ(0, encoder->InitEncode(settings, 1, 1200)); - - encoder->SetRates(target_bitrate, 30); - - std::vector configured_bitrates; - for (std::vector::const_iterator it = - spy_factory.spying_layers_.begin(); - it != spy_factory.spying_layers_.end(); ++it) { - configured_bitrates.push_back( - static_cast(*it)->configured_bitrate_); - } - return configured_bitrates; - } - - class SpyingTemporalLayers : public TemporalLayers { - public: - explicit SpyingTemporalLayers(TemporalLayers* layers) - : configured_bitrate_(0), layers_(layers) {} - - virtual ~SpyingTemporalLayers() { delete layers_; } - - int EncodeFlags(uint32_t timestamp) override { - return layers_->EncodeFlags(timestamp); - } - - bool ConfigureBitrates(int bitrate_kbit, - int max_bitrate_kbit, - int framerate, - vpx_codec_enc_cfg_t* cfg) override { - configured_bitrate_ = bitrate_kbit; - return layers_->ConfigureBitrates(bitrate_kbit, max_bitrate_kbit, - framerate, cfg); - } - - void PopulateCodecSpecific(bool base_layer_sync, - CodecSpecificInfoVP8* vp8_info, - uint32_t timestamp) override { - layers_->PopulateCodecSpecific(base_layer_sync, vp8_info, timestamp); - } - - void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override { - layers_->FrameEncoded(size, timestamp, qp); - } - - int CurrentLayerId() const override { return layers_->CurrentLayerId(); } - - bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override { - return false; - } - - int configured_bitrate_; - TemporalLayers* layers_; - }; - - class SpyingTemporalLayersFactory : public TemporalLayersFactory { - public: - virtual ~SpyingTemporalLayersFactory() {} - TemporalLayers* Create(int temporal_layers, - uint8_t initial_tl0_pic_idx) const override { - SpyingTemporalLayers* layers = - new SpyingTemporalLayers(TemporalLayersFactory::Create( - temporal_layers, initial_tl0_pic_idx)); - spying_layers_.push_back(layers); - return layers; - } - - mutable std::vector spying_layers_; - }; -}; - class TestVp8Simulcast : public ::testing::Test { public: TestVp8Simulcast(VP8Encoder* encoder, VP8Decoder* decoder) @@ -265,7 +190,7 @@ class TestVp8Simulcast : public ::testing::Test { static void DefaultSettings(VideoCodec* settings, const int* temporal_layer_profile) { - assert(settings); + RTC_CHECK(settings); memset(settings, 0, sizeof(VideoCodec)); strncpy(settings->plName, "VP8", 4); settings->codecType = kVideoCodecVP8; @@ -315,12 +240,18 @@ class TestVp8Simulcast : public ::testing::Test { } protected: - virtual void SetUp() { SetUpCodec(kDefaultTemporalLayerProfile); } + void SetUp() override { SetUpCodec(kDefaultTemporalLayerProfile); } - virtual void SetUpCodec(const int* temporal_layer_profile) { + void TearDown() override { + encoder_->Release(); + decoder_->Release(); + } + + void SetUpCodec(const int* temporal_layer_profile) { encoder_->RegisterEncodeCompleteCallback(&encoder_callback_); decoder_->RegisterDecodeCompleteCallback(&decoder_callback_); DefaultSettings(&settings_, temporal_layer_profile); + SetUpRateAllocator(); EXPECT_EQ(0, encoder_->InitEncode(&settings_, 1, 1200)); EXPECT_EQ(0, decoder_->InitDecode(&settings_, 1)); int half_width = (kDefaultWidth + 1) / 2; @@ -331,9 +262,16 @@ class TestVp8Simulcast : public ::testing::Test { new VideoFrame(input_buffer_, 0, 0, webrtc::kVideoRotation_0)); } - virtual void TearDown() { - encoder_->Release(); - decoder_->Release(); + void SetUpRateAllocator() { + TemporalLayersFactory* tl_factory = new TemporalLayersFactory(); + rate_allocator_.reset(new SimulcastRateAllocator( + settings_, std::unique_ptr(tl_factory))); + settings_.codecSpecific.VP8.tl_factory = tl_factory; + } + + void SetRates(uint32_t bitrate_kbps, uint32_t fps) { + encoder_->SetRateAllocation( + rate_allocator_->GetAllocation(bitrate_kbps * 1000, fps), fps); } void ExpectStreams(FrameType frame_type, int expected_video_streams) { @@ -396,7 +334,7 @@ class TestVp8Simulcast : public ::testing::Test { // We currently expect all active streams to generate a key frame even though // a key frame was only requested for some of them. void TestKeyFrameRequestsOnAllStreams() { - encoder_->SetRates(kMaxBitrates[2], 30); // To get all three streams. + SetRates(kMaxBitrates[2], 30); // To get all three streams. std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); ExpectStreams(kVideoFrameKey, kNumberOfSimulcastStreams); @@ -431,7 +369,7 @@ class TestVp8Simulcast : public ::testing::Test { void TestPaddingAllStreams() { // We should always encode the base layer. - encoder_->SetRates(kMinBitrates[0] - 1, 30); + SetRates(kMinBitrates[0] - 1, 30); std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); ExpectStreams(kVideoFrameKey, 1); @@ -444,7 +382,7 @@ class TestVp8Simulcast : public ::testing::Test { void TestPaddingTwoStreams() { // We have just enough to get only the first stream and padding for two. - encoder_->SetRates(kMinBitrates[0], 30); + SetRates(kMinBitrates[0], 30); std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); ExpectStreams(kVideoFrameKey, 1); @@ -458,7 +396,7 @@ class TestVp8Simulcast : public ::testing::Test { void TestPaddingTwoStreamsOneMaxedOut() { // We are just below limit of sending second stream, so we should get // the first stream maxed out (at |maxBitrate|), and padding for two. - encoder_->SetRates(kTargetBitrates[0] + kMinBitrates[1] - 1, 30); + SetRates(kTargetBitrates[0] + kMinBitrates[1] - 1, 30); std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); ExpectStreams(kVideoFrameKey, 1); @@ -471,7 +409,7 @@ class TestVp8Simulcast : public ::testing::Test { void TestPaddingOneStream() { // We have just enough to send two streams, so padding for one stream. - encoder_->SetRates(kTargetBitrates[0] + kMinBitrates[1], 30); + SetRates(kTargetBitrates[0] + kMinBitrates[1], 30); std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); ExpectStreams(kVideoFrameKey, 2); @@ -485,8 +423,7 @@ class TestVp8Simulcast : public ::testing::Test { void TestPaddingOneStreamTwoMaxedOut() { // We are just below limit of sending third stream, so we should get // first stream's rate maxed out at |targetBitrate|, second at |maxBitrate|. - encoder_->SetRates( - kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2] - 1, 30); + SetRates(kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2] - 1, 30); std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); ExpectStreams(kVideoFrameKey, 2); @@ -499,8 +436,7 @@ class TestVp8Simulcast : public ::testing::Test { void TestSendAllStreams() { // We have just enough to send all streams. - encoder_->SetRates( - kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2], 30); + SetRates(kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2], 30); std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); ExpectStreams(kVideoFrameKey, 3); @@ -513,7 +449,7 @@ class TestVp8Simulcast : public ::testing::Test { void TestDisablingStreams() { // We should get three media streams. - encoder_->SetRates(kMaxBitrates[0] + kMaxBitrates[1] + kMaxBitrates[2], 30); + SetRates(kMaxBitrates[0] + kMaxBitrates[1] + kMaxBitrates[2], 30); std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); ExpectStreams(kVideoFrameKey, 3); @@ -524,36 +460,33 @@ class TestVp8Simulcast : public ::testing::Test { EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL, &frame_types)); // We should only get two streams and padding for one. - encoder_->SetRates( - kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2] / 2, 30); + SetRates(kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2] / 2, 30); ExpectStreams(kVideoFrameDelta, 2); input_frame_->set_timestamp(input_frame_->timestamp() + 3000); EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL, &frame_types)); // We should only get the first stream and padding for two. - encoder_->SetRates(kTargetBitrates[0] + kMinBitrates[1] / 2, 30); + SetRates(kTargetBitrates[0] + kMinBitrates[1] / 2, 30); ExpectStreams(kVideoFrameDelta, 1); input_frame_->set_timestamp(input_frame_->timestamp() + 3000); EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL, &frame_types)); // We don't have enough bitrate for the thumbnail stream, but we should get // it anyway with current configuration. - encoder_->SetRates(kTargetBitrates[0] - 1, 30); + SetRates(kTargetBitrates[0] - 1, 30); ExpectStreams(kVideoFrameDelta, 1); input_frame_->set_timestamp(input_frame_->timestamp() + 3000); EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL, &frame_types)); // We should only get two streams and padding for one. - encoder_->SetRates( - kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2] / 2, 30); + SetRates(kTargetBitrates[0] + kTargetBitrates[1] + kMinBitrates[2] / 2, 30); // We get a key frame because a new stream is being enabled. ExpectStreams(kVideoFrameKey, 2); input_frame_->set_timestamp(input_frame_->timestamp() + 3000); EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL, &frame_types)); // We should get all three streams. - encoder_->SetRates( - kTargetBitrates[0] + kTargetBitrates[1] + kTargetBitrates[2], 30); + SetRates(kTargetBitrates[0] + kTargetBitrates[1] + kTargetBitrates[2], 30); // We get a key frame because a new stream is being enabled. ExpectStreams(kVideoFrameKey, 3); input_frame_->set_timestamp(input_frame_->timestamp() + 3000); @@ -590,10 +523,11 @@ class TestVp8Simulcast : public ::testing::Test { settings_.width; settings_.simulcastStream[settings_.numberOfSimulcastStreams - 1].height = settings_.height; + SetUpRateAllocator(); EXPECT_EQ(0, encoder_->InitEncode(&settings_, 1, 1200)); // Encode one frame and verify. - encoder_->SetRates(kMaxBitrates[0] + kMaxBitrates[1], 30); + SetRates(kMaxBitrates[0] + kMaxBitrates[1], 30); std::vector frame_types(kNumberOfSimulcastStreams, kVideoFrameDelta); EXPECT_CALL( @@ -611,8 +545,9 @@ class TestVp8Simulcast : public ::testing::Test { DefaultSettings(&settings_, kDefaultTemporalLayerProfile); // Start at the lowest bitrate for enabling base stream. settings_.startBitrate = kMinBitrates[0]; + SetUpRateAllocator(); EXPECT_EQ(0, encoder_->InitEncode(&settings_, 1, 1200)); - encoder_->SetRates(settings_.startBitrate, 30); + SetRates(settings_.startBitrate, 30); ExpectStreams(kVideoFrameKey, 1); // Resize |input_frame_| to the new resolution. half_width = (settings_.width + 1) / 2; @@ -634,7 +569,7 @@ class TestVp8Simulcast : public ::testing::Test { Vp8TestEncodedImageCallback encoder_callback; encoder_->RegisterEncodeCompleteCallback(&encoder_callback); - encoder_->SetRates(kMaxBitrates[2], 30); // To get all three streams. + SetRates(kMaxBitrates[2], 30); // To get all three streams. EXPECT_EQ(0, encoder_->Encode(*input_frame_, NULL, NULL)); int picture_id = -1; @@ -703,7 +638,7 @@ class TestVp8Simulcast : public ::testing::Test { encoder_->RegisterEncodeCompleteCallback(&encoder_callback); decoder_->RegisterDecodeCompleteCallback(&decoder_callback); - encoder_->SetRates(kMaxBitrates[2], 30); // To get all three streams. + SetRates(kMaxBitrates[2], 30); // To get all three streams. // Set color. int plane_offset[kNumOfPlanes]; @@ -777,7 +712,7 @@ class TestVp8Simulcast : public ::testing::Test { void TestSaptioTemporalLayers333PatternEncoder() { Vp8TestEncodedImageCallback encoder_callback; encoder_->RegisterEncodeCompleteCallback(&encoder_callback); - encoder_->SetRates(kMaxBitrates[2], 30); // To get all three streams. + SetRates(kMaxBitrates[2], 30); // To get all three streams. int expected_temporal_idx[3] = {-1, -1, -1}; bool expected_layer_sync[3] = {false, false, false}; @@ -846,7 +781,7 @@ class TestVp8Simulcast : public ::testing::Test { SetUpCodec(temporal_layer_profile); Vp8TestEncodedImageCallback encoder_callback; encoder_->RegisterEncodeCompleteCallback(&encoder_callback); - encoder_->SetRates(kMaxBitrates[2], 30); // To get all three streams. + SetRates(kMaxBitrates[2], 30); // To get all three streams. int expected_temporal_idx[3] = {-1, -1, -1}; bool expected_layer_sync[3] = {false, false, false}; @@ -905,7 +840,7 @@ class TestVp8Simulcast : public ::testing::Test { encoder_->RegisterEncodeCompleteCallback(&encoder_callback); decoder_->RegisterDecodeCompleteCallback(&decoder_callback); - encoder_->SetRates(kMaxBitrates[2], 30); // To get all three streams. + SetRates(kMaxBitrates[2], 30); // To get all three streams. // Setting two (possibly) problematic use cases for stride: // 1. stride > width 2. stride_y != stride_uv/2 int stride_y = kDefaultWidth + 20; @@ -941,30 +876,6 @@ class TestVp8Simulcast : public ::testing::Test { EXPECT_EQ(2, decoder_callback.DecodedFrames()); } - void TestSkipEncodingUnusedStreams() { - SkipEncodingUnusedStreamsTest test; - std::vector configured_bitrate = - test.RunTest(encoder_.get(), &settings_, - 1); // Target bit rate 1, to force all streams but the - // base one to be exceeding bandwidth constraints. - EXPECT_EQ(static_cast(kNumberOfSimulcastStreams), - configured_bitrate.size()); - - unsigned int min_bitrate = - std::max(settings_.simulcastStream[0].minBitrate, settings_.minBitrate); - int stream = 0; - for (std::vector::const_iterator it = - configured_bitrate.begin(); - it != configured_bitrate.end(); ++it) { - if (stream == 0) { - EXPECT_EQ(min_bitrate, *it); - } else { - EXPECT_EQ(0u, *it); - } - ++stream; - } - } - std::unique_ptr encoder_; MockEncodedImageCallback encoder_callback_; std::unique_ptr decoder_; @@ -972,6 +883,7 @@ class TestVp8Simulcast : public ::testing::Test { VideoCodec settings_; rtc::scoped_refptr input_buffer_; std::unique_ptr input_frame_; + std::unique_ptr rate_allocator_; }; } // namespace testing diff --git a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h index 0d09839a8c..e1f1e0abae 100644 --- a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h @@ -12,18 +12,21 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_ #define WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_ -#include "vpx/vpx_encoder.h" +#include #include "webrtc/common_video/include/video_image.h" #include "webrtc/typedefs.h" +struct vpx_codec_enc_cfg; +typedef struct vpx_codec_enc_cfg vpx_codec_enc_cfg_t; + namespace webrtc { struct CodecSpecificInfoVP8; class TemporalLayers { public: - // Factory for TemporalLayer strategy. Default behaviour is a fixed pattern + // Factory for TemporalLayer strategy. Default behavior is a fixed pattern // of temporal layers. See default_temporal_layers.cc virtual ~TemporalLayers() {} @@ -31,10 +34,15 @@ class TemporalLayers { // and/or update the reference buffers. virtual int EncodeFlags(uint32_t timestamp) = 0; - virtual bool ConfigureBitrates(int bitrate_kbit, - int max_bitrate_kbit, - int framerate, - vpx_codec_enc_cfg_t* cfg) = 0; + // Update state based on new bitrate target and incoming framerate. + // Returns the bitrate allocation for the active temporal layers. + virtual std::vector OnRatesUpdated(int bitrate_kbps, + int max_bitrate_kbps, + int framerate) = 0; + + // Update the encoder configuration with target bitrates or other parameters. + // Returns true iff the configuration was actually modified. + virtual bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) = 0; virtual void PopulateCodecSpecific(bool base_layer_sync, CodecSpecificInfoVP8* vp8_info, @@ -43,26 +51,42 @@ class TemporalLayers { virtual void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) = 0; virtual int CurrentLayerId() const = 0; - - virtual bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) = 0; }; +class TemporalLayersListener; class TemporalLayersFactory { public: + TemporalLayersFactory() : listener_(nullptr) {} virtual ~TemporalLayersFactory() {} - virtual TemporalLayers* Create(int temporal_layers, + virtual TemporalLayers* Create(int simulcast_id, + int temporal_layers, uint8_t initial_tl0_pic_idx) const; + void SetListener(TemporalLayersListener* listener); + + protected: + TemporalLayersListener* listener_; }; // Factory for a temporal layers strategy that adaptively changes the number of -// layers based on input framerate so that the base layer has an acceptable -// framerate. See realtime_temporal_layers.cc +// layers based on input frame rate so that the base layer has an acceptable +// frame rate. See realtime_temporal_layers.cc class RealTimeTemporalLayersFactory : public TemporalLayersFactory { public: + RealTimeTemporalLayersFactory() {} ~RealTimeTemporalLayersFactory() override {} - TemporalLayers* Create(int num_temporal_layers, + TemporalLayers* Create(int simulcast_id, + int num_temporal_layers, uint8_t initial_tl0_pic_idx) const override; }; +class TemporalLayersListener { + public: + TemporalLayersListener() {} + virtual ~TemporalLayersListener() {} + + virtual void OnTemporalLayersCreated(int simulcast_id, + TemporalLayers* layers) = 0; +}; + } // namespace webrtc #endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_VP8_TEMPORAL_LAYERS_H_ diff --git a/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc index d636b14f50..d95b6d0adc 100644 --- a/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc @@ -16,6 +16,7 @@ #include "webrtc/base/timeutils.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" +#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/test/frame_utils.h" #include "webrtc/test/gtest.h" #include "webrtc/test/testsupport/fileutils.h" @@ -161,6 +162,8 @@ class TestVp8Impl : public ::testing::Test { codec_inst_.maxBitrate = 4000; codec_inst_.qpMax = 56; codec_inst_.VP8()->denoisingOn = true; + codec_inst_.VP8()->tl_factory = &tl_factory_; + codec_inst_.VP8()->numberOfTemporalLayers = 1; EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->InitEncode(&codec_inst_, 1, 1440)); @@ -201,6 +204,7 @@ class TestVp8Impl : public ::testing::Test { EncodedImage encoded_frame_; VideoFrame decoded_frame_; VideoCodec codec_inst_; + TemporalLayersFactory tl_factory_; }; TEST_F(TestVp8Impl, EncoderParameterTest) { @@ -215,11 +219,15 @@ TEST_F(TestVp8Impl, EncoderParameterTest) { codec_inst_.qpMax = 56; codec_inst_.VP8()->complexity = kComplexityNormal; codec_inst_.VP8()->numberOfTemporalLayers = 1; + codec_inst_.VP8()->tl_factory = &tl_factory_; // Calls before InitEncode(). EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release()); int bit_rate = 300; + BitrateAllocation bitrate_allocation; + bitrate_allocation.SetBitrate(0, 0, bit_rate * 1000); EXPECT_EQ(WEBRTC_VIDEO_CODEC_UNINITIALIZED, - encoder_->SetRates(bit_rate, codec_inst_.maxFramerate)); + encoder_->SetRateAllocation(bitrate_allocation, + codec_inst_.maxFramerate)); EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->InitEncode(&codec_inst_, 1, 1440)); diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index 2e510ec897..82f964dec2 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -113,7 +113,6 @@ VP8Decoder* VP8Decoder::Create() { VP8EncoderImpl::VP8EncoderImpl() : encoded_complete_callback_(nullptr), - rate_allocator_(new SimulcastRateAllocator(codec_)), inited_(false), timestamp_(0), feedback_mode_(false), @@ -175,27 +174,32 @@ int VP8EncoderImpl::Release() { return ret_val; } -int VP8EncoderImpl::SetRates(uint32_t new_bitrate_kbit, - uint32_t new_framerate) { - if (!inited_) { +int VP8EncoderImpl::SetRateAllocation(const BitrateAllocation& bitrate, + uint32_t new_framerate) { + if (!inited_) return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } - if (encoders_[0].err) { + + if (encoders_[0].err) return WEBRTC_VIDEO_CODEC_ERROR; - } - if (new_framerate < 1) { + + if (new_framerate < 1) return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + + if (bitrate.get_sum_bps() == 0) { + // Encoder paused, turn off all encoding. + const int num_streams = static_cast(encoders_.size()); + for (int i = 0; i < num_streams; ++i) + SetStreamState(false, i); + return WEBRTC_VIDEO_CODEC_OK; } - if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate) { - new_bitrate_kbit = codec_.maxBitrate; - } - if (new_bitrate_kbit < codec_.minBitrate) { - new_bitrate_kbit = codec_.minBitrate; - } - if (codec_.numberOfSimulcastStreams > 0 && - new_bitrate_kbit < codec_.simulcastStream[0].minBitrate) { - new_bitrate_kbit = codec_.simulcastStream[0].minBitrate; - } + + // At this point, bitrate allocation should already match codec settings. + if (codec_.maxBitrate > 0) + RTC_DCHECK_LE(bitrate.get_sum_kbps(), codec_.maxBitrate); + RTC_DCHECK_GE(bitrate.get_sum_kbps(), codec_.minBitrate); + if (codec_.numberOfSimulcastStreams > 0) + RTC_DCHECK_GE(bitrate.get_sum_kbps(), codec_.simulcastStream[0].minBitrate); + codec_.maxFramerate = new_framerate; if (encoders_.size() == 1) { @@ -207,14 +211,14 @@ int VP8EncoderImpl::SetRates(uint32_t new_bitrate_kbit, // Only trigger keyframes if we are allowed to scale down. if (configurations_[0].rc_resize_allowed) { if (!down_scale_requested_) { - if (k_pixels_per_frame > new_bitrate_kbit) { + if (k_pixels_per_frame > bitrate.get_sum_kbps()) { down_scale_requested_ = true; - down_scale_bitrate_ = new_bitrate_kbit; + down_scale_bitrate_ = bitrate.get_sum_kbps(); key_frame_request_[0] = true; } } else { - if (new_bitrate_kbit > (2 * down_scale_bitrate_) || - new_bitrate_kbit < (down_scale_bitrate_ / 2)) { + if (bitrate.get_sum_kbps() > (2 * down_scale_bitrate_) || + bitrate.get_sum_kbps() < (down_scale_bitrate_ / 2)) { down_scale_requested_ = false; } } @@ -233,31 +237,18 @@ int VP8EncoderImpl::SetRates(uint32_t new_bitrate_kbit, } } - std::vector stream_bitrates = - rate_allocator_->GetAllocation(new_bitrate_kbit); size_t stream_idx = encoders_.size() - 1; for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) { - if (encoders_.size() > 1) - SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx); + unsigned int target_bitrate_kbps = + bitrate.GetSpatialLayerSum(stream_idx) / 1000; + + bool send_stream = target_bitrate_kbps > 0; + if (send_stream || encoders_.size() > 1) + SetStreamState(send_stream, stream_idx); + + configurations_[i].rc_target_bitrate = target_bitrate_kbps; + temporal_layers_[stream_idx]->UpdateConfiguration(&configurations_[i]); - unsigned int target_bitrate = stream_bitrates[stream_idx]; - unsigned int max_bitrate = codec_.maxBitrate; - int framerate = new_framerate; - // TODO(holmer): This is a temporary hack for screensharing, where we - // interpret the startBitrate as the encoder target bitrate. This is - // to allow for a different max bitrate, so if the codec can't meet - // the target we still allow it to overshoot up to the max before dropping - // frames. This hack should be improved. - if (codec_.targetBitrate > 0 && - (codec_.VP8()->numberOfTemporalLayers == 2 || - codec_.simulcastStream[0].numberOfTemporalLayers == 2)) { - int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate); - max_bitrate = std::min(codec_.maxBitrate, target_bitrate); - target_bitrate = tl0_bitrate; - } - configurations_[i].rc_target_bitrate = target_bitrate; - temporal_layers_[stream_idx]->ConfigureBitrates( - target_bitrate, max_bitrate, framerate, &configurations_[i]); if (vpx_codec_enc_config_set(&encoders_[i], &configurations_[i])) { return WEBRTC_VIDEO_CODEC_ERROR; } @@ -287,26 +278,17 @@ void VP8EncoderImpl::SetStreamState(bool send_stream, void VP8EncoderImpl::SetupTemporalLayers(int num_streams, int num_temporal_layers, const VideoCodec& codec) { - TemporalLayersFactory default_factory; - const TemporalLayersFactory* tl_factory = codec.VP8().tl_factory; - if (!tl_factory) - tl_factory = &default_factory; + RTC_DCHECK(codec.codecSpecific.VP8.tl_factory != nullptr); + const TemporalLayersFactory* tl_factory = codec.codecSpecific.VP8.tl_factory; if (num_streams == 1) { - if (codec.mode == kScreensharing) { - // Special mode when screensharing on a single stream. - temporal_layers_.push_back(new ScreenshareLayers( - num_temporal_layers, rand(), webrtc::Clock::GetRealTimeClock())); - } else { - temporal_layers_.push_back( - tl_factory->Create(num_temporal_layers, rand())); - } + temporal_layers_.push_back( + tl_factory->Create(0, num_temporal_layers, rand())); } else { for (int i = 0; i < num_streams; ++i) { - // TODO(andresp): crash if layers is invalid. - int layers = codec.simulcastStream[i].numberOfTemporalLayers; - if (layers < 1) - layers = 1; - temporal_layers_.push_back(tl_factory->Create(layers, rand())); + RTC_CHECK_GT(num_temporal_layers, 0); + int layers = std::max(static_cast(1), + codec.simulcastStream[i].numberOfTemporalLayers); + temporal_layers_.push_back(tl_factory->Create(i, layers, rand())); } } } @@ -351,10 +333,8 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst, int num_temporal_layers = doing_simulcast ? inst->simulcastStream[0].numberOfTemporalLayers : inst->VP8().numberOfTemporalLayers; + RTC_DCHECK_GT(num_temporal_layers, 0); - // TODO(andresp): crash if num temporal layers is bananas. - if (num_temporal_layers < 1) - num_temporal_layers = 1; SetupTemporalLayers(number_of_streams, num_temporal_layers, *inst); feedback_mode_ = inst->VP8().feedbackModeOn; @@ -362,7 +342,6 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst, number_of_cores_ = number_of_cores; timestamp_ = 0; codec_ = *inst; - rate_allocator_.reset(new SimulcastRateAllocator(codec_)); // Code expects simulcastStream resolutions to be correct, make sure they are // filled even when there are no simulcast layers. @@ -514,45 +493,44 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst, vpx_img_wrap(&raw_images_[0], VPX_IMG_FMT_I420, inst->width, inst->height, 1, NULL); - if (encoders_.size() == 1) { - configurations_[0].rc_target_bitrate = inst->startBitrate; - temporal_layers_[0]->ConfigureBitrates(inst->startBitrate, inst->maxBitrate, - inst->maxFramerate, - &configurations_[0]); - } else { - // Note the order we use is different from webm, we have lowest resolution - // at position 0 and they have highest resolution at position 0. - int stream_idx = encoders_.size() - 1; - std::vector stream_bitrates = - rate_allocator_->GetAllocation(inst->startBitrate); + // Note the order we use is different from webm, we have lowest resolution + // at position 0 and they have highest resolution at position 0. + int stream_idx = encoders_.size() - 1; + SimulcastRateAllocator init_allocator(codec_, nullptr); + BitrateAllocation allocation = init_allocator.GetAllocation( + inst->startBitrate * 1000, inst->maxFramerate); + std::vector stream_bitrates; + for (int i = 0; i == 0 || i < inst->numberOfSimulcastStreams; ++i) { + uint32_t bitrate = allocation.GetSpatialLayerSum(i) / 1000; + stream_bitrates.push_back(bitrate); + } + + configurations_[0].rc_target_bitrate = stream_bitrates[stream_idx]; + temporal_layers_[stream_idx]->OnRatesUpdated( + stream_bitrates[stream_idx], inst->maxBitrate, inst->maxFramerate); + temporal_layers_[stream_idx]->UpdateConfiguration(&configurations_[0]); + --stream_idx; + for (size_t i = 1; i < encoders_.size(); ++i, --stream_idx) { + memcpy(&configurations_[i], &configurations_[0], + sizeof(configurations_[0])); + + configurations_[i].g_w = inst->simulcastStream[stream_idx].width; + configurations_[i].g_h = inst->simulcastStream[stream_idx].height; + + // Use 1 thread for lower resolutions. + configurations_[i].g_threads = 1; + + // Setting alignment to 32 - as that ensures at least 16 for all + // planes (32 for Y, 16 for U,V). Libvpx sets the requested stride for + // the y plane, but only half of it to the u and v planes. + vpx_img_alloc(&raw_images_[i], VPX_IMG_FMT_I420, + inst->simulcastStream[stream_idx].width, + inst->simulcastStream[stream_idx].height, kVp832ByteAlign); SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx); - configurations_[0].rc_target_bitrate = stream_bitrates[stream_idx]; - temporal_layers_[stream_idx]->ConfigureBitrates( - stream_bitrates[stream_idx], inst->maxBitrate, inst->maxFramerate, - &configurations_[0]); - --stream_idx; - for (size_t i = 1; i < encoders_.size(); ++i, --stream_idx) { - memcpy(&configurations_[i], &configurations_[0], - sizeof(configurations_[0])); - - configurations_[i].g_w = inst->simulcastStream[stream_idx].width; - configurations_[i].g_h = inst->simulcastStream[stream_idx].height; - - // Use 1 thread for lower resolutions. - configurations_[i].g_threads = 1; - - // Setting alignment to 32 - as that ensures at least 16 for all - // planes (32 for Y, 16 for U,V). Libvpx sets the requested stride for - // the y plane, but only half of it to the u and v planes. - vpx_img_alloc(&raw_images_[i], VPX_IMG_FMT_I420, - inst->simulcastStream[stream_idx].width, - inst->simulcastStream[stream_idx].height, kVp832ByteAlign); - SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx); - configurations_[i].rc_target_bitrate = stream_bitrates[stream_idx]; - temporal_layers_[stream_idx]->ConfigureBitrates( - stream_bitrates[stream_idx], inst->maxBitrate, inst->maxFramerate, - &configurations_[i]); - } + configurations_[i].rc_target_bitrate = stream_bitrates[stream_idx]; + temporal_layers_[stream_idx]->OnRatesUpdated( + stream_bitrates[stream_idx], inst->maxBitrate, inst->maxFramerate); + temporal_layers_[stream_idx]->UpdateConfiguration(&configurations_[i]); } rps_.Init(); diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h index f2f208813c..3b3a78a7e7 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.h @@ -32,7 +32,6 @@ namespace webrtc { -class SimulcastRateAllocator; class TemporalLayers; class VP8EncoderImpl : public VP8Encoder { @@ -55,7 +54,8 @@ class VP8EncoderImpl : public VP8Encoder { int SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; - int SetRates(uint32_t new_bitrate_kbit, uint32_t frame_rate) override; + int SetRateAllocation(const BitrateAllocation& bitrate, + uint32_t new_framerate) override; void OnDroppedFrame() override; @@ -94,7 +94,6 @@ class VP8EncoderImpl : public VP8Encoder { EncodedImageCallback* encoded_complete_callback_; VideoCodec codec_; - std::unique_ptr rate_allocator_; bool inited_; int64_t timestamp_; bool feedback_mode_; diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc index 8c798db665..7ab4a79bd2 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -77,6 +77,7 @@ VP9EncoderImpl::VP9EncoderImpl() frames_since_kf_(0), num_temporal_layers_(0), num_spatial_layers_(0), + is_flexible_mode_(false), frames_encoded_(0), // Use two spatial when screensharing with flexible mode. spatial_layer_(new ScreenshareLayersVP9(2)) { @@ -193,24 +194,28 @@ bool VP9EncoderImpl::SetSvcRates() { return true; } -int VP9EncoderImpl::SetRates(uint32_t new_bitrate_kbit, - uint32_t new_framerate) { +int VP9EncoderImpl::SetRateAllocation( + const BitrateAllocation& bitrate_allocation, + uint32_t frame_rate) { if (!inited_) { return WEBRTC_VIDEO_CODEC_UNINITIALIZED; } if (encoder_->err) { return WEBRTC_VIDEO_CODEC_ERROR; } - if (new_framerate < 1) { + if (frame_rate < 1) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } // Update bit rate - if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate) { - new_bitrate_kbit = codec_.maxBitrate; + if (codec_.maxBitrate > 0 && + bitrate_allocation.get_sum_kbps() > codec_.maxBitrate) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } - config_->rc_target_bitrate = new_bitrate_kbit; - codec_.maxFramerate = new_framerate; - spatial_layer_->ConfigureBitrate(new_bitrate_kbit, 0); + + // TODO(sprang): Actually use BitrateAllocation layer info. + config_->rc_target_bitrate = bitrate_allocation.get_sum_kbps(); + codec_.maxFramerate = frame_rate; + spatial_layer_->ConfigureBitrate(bitrate_allocation.get_sum_kbps(), 0); if (!SetSvcRates()) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; diff --git a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h index 7e65f3b211..4eb31a4256 100644 --- a/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h +++ b/webrtc/modules/video_coding/codecs/vp9/vp9_impl.h @@ -46,7 +46,8 @@ class VP9EncoderImpl : public VP9Encoder { int SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; - int SetRates(uint32_t new_bitrate_kbit, uint32_t frame_rate) override; + int SetRateAllocation(const BitrateAllocation& bitrate_allocation, + uint32_t frame_rate) override; void OnDroppedFrame() override {} diff --git a/webrtc/modules/video_coding/generic_encoder.cc b/webrtc/modules/video_coding/generic_encoder.cc index acef556b24..80e48d1911 100644 --- a/webrtc/modules/video_coding/generic_encoder.cc +++ b/webrtc/modules/video_coding/generic_encoder.cc @@ -28,7 +28,7 @@ VCMGenericEncoder::VCMGenericEncoder( : encoder_(encoder), vcm_encoded_frame_callback_(encoded_frame_callback), internal_source_(internal_source), - encoder_params_({0, 0, 0, 0}), + encoder_params_({BitrateAllocation(), 0, 0, 0}), is_screenshare_(false) {} VCMGenericEncoder::~VCMGenericEncoder() {} @@ -90,11 +90,23 @@ void VCMGenericEncoder::SetEncoderParameters(const EncoderParameters& params) { params.input_frame_rate != encoder_params_.input_frame_rate; encoder_params_ = params; } - if (channel_parameters_have_changed) - encoder_->SetChannelParameters(params.loss_rate, params.rtt); + if (channel_parameters_have_changed) { + int res = encoder_->SetChannelParameters(params.loss_rate, params.rtt); + if (res != 0) { + LOG(LS_WARNING) << "Error set encoder parameters (loss = " + << params.loss_rate << ", rtt = " << params.rtt + << "): " << res; + } + } if (rates_have_changed) { - uint32_t target_bitrate_kbps = (params.target_bitrate + 500) / 1000; - encoder_->SetRates(target_bitrate_kbps, params.input_frame_rate); + int res = encoder_->SetRateAllocation(params.target_bitrate, + params.input_frame_rate); + if (res != 0) { + LOG(LS_WARNING) << "Error set encoder rate (total bitrate bps = " + << params.target_bitrate.get_sum_bps() + << ", framerate = " << params.input_frame_rate + << "): " << res; + } } } diff --git a/webrtc/modules/video_coding/generic_encoder.h b/webrtc/modules/video_coding/generic_encoder.h index bc1afae55f..4d2ea2f364 100644 --- a/webrtc/modules/video_coding/generic_encoder.h +++ b/webrtc/modules/video_coding/generic_encoder.h @@ -28,7 +28,7 @@ class MediaOptimization; } // namespace media_optimization struct EncoderParameters { - uint32_t target_bitrate; + BitrateAllocation target_bitrate; uint8_t loss_rate; int64_t rtt; uint32_t input_frame_rate; diff --git a/webrtc/modules/video_coding/include/mock/mock_video_codec_interface.h b/webrtc/modules/video_coding/include/mock/mock_video_codec_interface.h index ce024973cf..7e2076ccdc 100644 --- a/webrtc/modules/video_coding/include/mock/mock_video_codec_interface.h +++ b/webrtc/modules/video_coding/include/mock/mock_video_codec_interface.h @@ -45,6 +45,9 @@ class MockVideoEncoder : public VideoEncoder { MOCK_METHOD0(Reset, int32_t()); MOCK_METHOD2(SetChannelParameters, int32_t(uint32_t packetLoss, int64_t rtt)); MOCK_METHOD2(SetRates, int32_t(uint32_t newBitRate, uint32_t frameRate)); + MOCK_METHOD2(SetRateAllocation, + int32_t(const BitrateAllocation& newBitRate, + uint32_t frameRate)); MOCK_METHOD1(SetPeriodicKeyFrames, int32_t(bool enable)); }; diff --git a/webrtc/modules/video_coding/include/video_codec_initializer.h b/webrtc/modules/video_coding/include/video_codec_initializer.h new file mode 100644 index 0000000000..8514bbbce0 --- /dev/null +++ b/webrtc/modules/video_coding/include/video_codec_initializer.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_VIDEO_CODING_INCLUDE_VIDEO_CODEC_INITIALIZER_H_ +#define WEBRTC_MODULES_VIDEO_CODING_INCLUDE_VIDEO_CODEC_INITIALIZER_H_ + +#include +#include +#include + +#include "webrtc/video_send_stream.h" + +namespace webrtc { + +class TemporalLayersFactory; +class VideoBitrateAllocator; +class VideoCodec; +class VideoEncoderConfig; + +class VideoCodecInitializer { + public: + // Takes an EncoderSettings, a VideoEncoderConfig and the VideoStream + // configuration and translated them into the old school VideoCodec type. + // It also creates a VideoBitrateAllocator instance, suitable for the codec + // type used. For instance, VP8 will create an allocator than can handle + // simulcast and temporal layering. + // GetBitrateAllocator is called implicitly from here, no need to call again. + static bool SetupCodec( + const VideoEncoderConfig& config, + const VideoSendStream::Config::EncoderSettings settings, + const std::vector& streams, + VideoCodec* codec, + std::unique_ptr* bitrate_allocator); + + // Create a bitrate allocator for the specified codec. |tl_factory| is + // optional, if it is populated, ownership of that instance will be + // transferred to the VideoBitrateAllocator instance. + static std::unique_ptr CreateBitrateAllocator( + const VideoCodec& codec, + std::unique_ptr tl_factory); + + private: + static VideoCodec VideoEncoderConfigToVideoCodec( + const VideoEncoderConfig& config, + const std::vector& streams, + const std::string& payload_name, + int payload_type); +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_VIDEO_CODING_INCLUDE_VIDEO_CODEC_INITIALIZER_H_ diff --git a/webrtc/modules/video_coding/include/video_coding.h b/webrtc/modules/video_coding/include/video_coding.h index eeda8a2e34..7b6bd32416 100644 --- a/webrtc/modules/video_coding/include/video_coding.h +++ b/webrtc/modules/video_coding/include/video_coding.h @@ -35,6 +35,7 @@ class EncodedImageCallback; // removing the VCM and use VideoSender/VideoReceiver as a public interface // directly. class VCMQMSettingsCallback; +class VideoBitrateAllocator; class VideoEncoder; class VideoDecoder; struct CodecSpecificInfo; diff --git a/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.cc b/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.cc new file mode 100644 index 0000000000..cb701f20b6 --- /dev/null +++ b/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.cc @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h" +#include "webrtc/base/basictypes.h" +#include "webrtc/base/constructormagic.h" + +namespace webrtc { + +DefaultVideoBitrateAllocator::DefaultVideoBitrateAllocator( + const VideoCodec& codec) + : codec_(codec) {} + +DefaultVideoBitrateAllocator::~DefaultVideoBitrateAllocator() {} + +BitrateAllocation DefaultVideoBitrateAllocator::GetAllocation( + uint32_t total_bitrate_bps, + uint32_t framerate) { + BitrateAllocation allocation; + if (total_bitrate_bps == 0) + return allocation; + + if (total_bitrate_bps < codec_.minBitrate * 1000) { + allocation.SetBitrate(0, 0, codec_.minBitrate * 1000); + } else if (codec_.maxBitrate > 0 && + total_bitrate_bps > codec_.maxBitrate * 1000) { + allocation.SetBitrate(0, 0, codec_.maxBitrate * 1000); + } else { + allocation.SetBitrate(0, 0, total_bitrate_bps); + } + + return allocation; +} + +uint32_t DefaultVideoBitrateAllocator::GetPreferredBitrateBps( + uint32_t framerate) { + return GetAllocation(codec_.maxBitrate * 1000, framerate).get_sum_bps(); +} + +} // namespace webrtc diff --git a/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h b/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h new file mode 100644 index 0000000000..f254e8d4b1 --- /dev/null +++ b/webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_DEFAULT_VIDEO_BITRATE_ALLOCATOR_H_ +#define WEBRTC_MODULES_VIDEO_CODING_UTILITY_DEFAULT_VIDEO_BITRATE_ALLOCATOR_H_ + +#include "webrtc/common_video/include/video_bitrate_allocator.h" + +namespace webrtc { + +class DefaultVideoBitrateAllocator : public VideoBitrateAllocator { + public: + explicit DefaultVideoBitrateAllocator(const VideoCodec& codec); + ~DefaultVideoBitrateAllocator() override; + + BitrateAllocation GetAllocation(uint32_t total_bitrate, + uint32_t framerate) override; + uint32_t GetPreferredBitrateBps(uint32_t framerate) override; + + private: + const VideoCodec codec_; +}; + +} // namespace webrtc + +#endif // WEBRTC_MODULES_VIDEO_CODING_UTILITY_DEFAULT_VIDEO_BITRATE_ALLOCATOR_H_ diff --git a/webrtc/modules/video_coding/utility/default_video_bitrate_allocator_unittest.cc b/webrtc/modules/video_coding/utility/default_video_bitrate_allocator_unittest.cc new file mode 100644 index 0000000000..df5643353e --- /dev/null +++ b/webrtc/modules/video_coding/utility/default_video_bitrate_allocator_unittest.cc @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h" +#include "webrtc/test/gtest.h" + +namespace webrtc { +namespace { +uint32_t kMaxBitrateBps = 1000000; +uint32_t kMinBitrateBps = 50000; +uint32_t kMaxFramerate = 30; +} // namespace + +class DefaultVideoBitrateAllocatorTest : public ::testing::Test { + public: + DefaultVideoBitrateAllocatorTest() {} + virtual ~DefaultVideoBitrateAllocatorTest() {} + + void SetUp() override { + codec_.codecType = kVideoCodecVP8; + codec_.minBitrate = kMinBitrateBps / 1000; + codec_.maxBitrate = kMaxBitrateBps / 1000; + codec_.targetBitrate = (kMinBitrateBps + kMaxBitrateBps) / 2000; + codec_.maxFramerate = kMaxFramerate; + allocator_.reset(new DefaultVideoBitrateAllocator(codec_)); + } + + protected: + VideoCodec codec_; + std::unique_ptr allocator_; +}; + +TEST_F(DefaultVideoBitrateAllocatorTest, ZeroIsOff) { + BitrateAllocation allocation = allocator_->GetAllocation(0, kMaxFramerate); + EXPECT_EQ(0u, allocation.get_sum_bps()); +} + +TEST_F(DefaultVideoBitrateAllocatorTest, CapsToMin) { + BitrateAllocation allocation = allocator_->GetAllocation(1, kMaxFramerate); + EXPECT_EQ(kMinBitrateBps, allocation.get_sum_bps()); + + allocation = allocator_->GetAllocation(kMinBitrateBps - 1, kMaxFramerate); + EXPECT_EQ(kMinBitrateBps, allocation.get_sum_bps()); + + allocation = allocator_->GetAllocation(kMinBitrateBps, kMaxFramerate); + EXPECT_EQ(kMinBitrateBps, allocation.get_sum_bps()); +} + +TEST_F(DefaultVideoBitrateAllocatorTest, CapsToMax) { + BitrateAllocation allocation = + allocator_->GetAllocation(kMaxBitrateBps, kMaxFramerate); + EXPECT_EQ(kMaxBitrateBps, allocation.get_sum_bps()); + + allocation = allocator_->GetAllocation(kMaxBitrateBps + 1, kMaxFramerate); + EXPECT_EQ(kMaxBitrateBps, allocation.get_sum_bps()); + + allocation = allocator_->GetAllocation(std::numeric_limits::max(), + kMaxFramerate); + EXPECT_EQ(kMaxBitrateBps, allocation.get_sum_bps()); +} + +TEST_F(DefaultVideoBitrateAllocatorTest, GoodInBetween) { + BitrateAllocation allocation = + allocator_->GetAllocation(kMinBitrateBps + 1, kMaxFramerate); + EXPECT_EQ(kMinBitrateBps + 1, allocation.get_sum_bps()); + + allocation = allocator_->GetAllocation(kMaxBitrateBps - 1, kMaxFramerate); + EXPECT_EQ(kMaxBitrateBps - 1, allocation.get_sum_bps()); +} +} // namespace webrtc diff --git a/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc b/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc index f6dccf0dda..83308fc5d3 100644 --- a/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc +++ b/webrtc/modules/video_coding/utility/simulcast_rate_allocator.cc @@ -11,67 +11,133 @@ #include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" #include +#include +#include +#include + +#include "webrtc/base/checks.h" namespace webrtc { -webrtc::SimulcastRateAllocator::SimulcastRateAllocator(const VideoCodec& codec) - : codec_(codec) {} +SimulcastRateAllocator::SimulcastRateAllocator( + const VideoCodec& codec, + std::unique_ptr tl_factory) + : codec_(codec), tl_factory_(std::move(tl_factory)) { + if (tl_factory_.get()) + tl_factory_->SetListener(this); +} -std::vector webrtc::SimulcastRateAllocator::GetAllocation( - uint32_t bitrate_kbps) const { - // Always allocate enough bitrate for the minimum bitrate of the first layer. - // Suspending below min bitrate is controlled outside the codec implementation - // and is not overridden by this. - const uint32_t min_bitrate_bps = codec_.numberOfSimulcastStreams == 0 - ? codec_.minBitrate - : codec_.simulcastStream[0].minBitrate; - uint32_t left_to_allocate = std::max(min_bitrate_bps, bitrate_kbps); - if (codec_.maxBitrate) - left_to_allocate = std::min(left_to_allocate, codec_.maxBitrate); +void SimulcastRateAllocator::OnTemporalLayersCreated(int simulcast_id, + TemporalLayers* layers) { + RTC_DCHECK(temporal_layers_.find(simulcast_id) == temporal_layers_.end()); + temporal_layers_[simulcast_id] = layers; +} - if (codec_.numberOfSimulcastStreams < 2) { +BitrateAllocation SimulcastRateAllocator::GetAllocation( + uint32_t total_bitrate_bps, + uint32_t framerate) { + uint32_t left_to_allocate = total_bitrate_bps; + if (codec_.maxBitrate && codec_.maxBitrate * 1000 < left_to_allocate) + left_to_allocate = codec_.maxBitrate * 1000; + + BitrateAllocation allocated_bitrates_bps; + if (codec_.numberOfSimulcastStreams == 0) { // No simulcast, just set the target as this has been capped already. - return std::vector(1, left_to_allocate); + allocated_bitrates_bps.SetBitrate( + 0, 0, std::max(codec_.minBitrate * 1000, left_to_allocate)); + } else { + // Always allocate enough bitrate for the minimum bitrate of the first + // layer. Suspending below min bitrate is controlled outside the codec + // implementation and is not overridden by this. + left_to_allocate = + std::max(codec_.simulcastStream[0].minBitrate * 1000, left_to_allocate); + + // Begin by allocating bitrate to simulcast streams, putting all bitrate in + // temporal layer 0. We'll then distribute this bitrate, across potential + // temporal layers, when stream allocation is done. + + // Allocate up to the target bitrate for each simulcast layer. + size_t layer = 0; + for (; layer < codec_.numberOfSimulcastStreams; ++layer) { + const SimulcastStream& stream = codec_.simulcastStream[layer]; + if (left_to_allocate < stream.minBitrate * 1000) + break; + uint32_t allocation = + std::min(left_to_allocate, stream.targetBitrate * 1000); + allocated_bitrates_bps.SetBitrate(layer, 0, allocation); + RTC_DCHECK_LE(allocation, left_to_allocate); + left_to_allocate -= allocation; + } + + // Next, try allocate remaining bitrate, up to max bitrate, in top stream. + // TODO(sprang): Allocate up to max bitrate for all layers once we have a + // better idea of possible performance implications. + if (left_to_allocate > 0) { + size_t active_layer = layer - 1; + const SimulcastStream& stream = codec_.simulcastStream[active_layer]; + uint32_t bitrate_bps = + allocated_bitrates_bps.GetSpatialLayerSum(active_layer); + uint32_t allocation = + std::min(left_to_allocate, stream.maxBitrate * 1000 - bitrate_bps); + bitrate_bps += allocation; + RTC_DCHECK_LE(allocation, left_to_allocate); + left_to_allocate -= allocation; + allocated_bitrates_bps.SetBitrate(active_layer, 0, bitrate_bps); + } } - // Initialize bitrates with zeroes. - std::vector allocated_bitrates_bps(codec_.numberOfSimulcastStreams, - 0); + const int num_spatial_streams = + std::max(1, static_cast(codec_.numberOfSimulcastStreams)); - // First try to allocate up to the target bitrate for each substream. - size_t layer = 0; - for (; layer < codec_.numberOfSimulcastStreams; ++layer) { - const SimulcastStream& stream = codec_.simulcastStream[layer]; - if (left_to_allocate < stream.minBitrate) - break; - uint32_t allocation = std::min(left_to_allocate, stream.targetBitrate); - allocated_bitrates_bps[layer] = allocation; - left_to_allocate -= allocation; - } + // Finally, distribute the bitrate for the simulcast streams across the + // available temporal layers. + for (int simulcast_id = 0; simulcast_id < num_spatial_streams; + ++simulcast_id) { + auto tl_it = temporal_layers_.find(simulcast_id); + if (tl_it == temporal_layers_.end()) + continue; // TODO(sprang): If > 1 SS, assume default TL alloc? - // Next, try allocate remaining bitrate, up to max bitrate, in top layer. - // TODO(sprang): Allocate up to max bitrate for all layers once we have a - // better idea of possible performance implications. - if (left_to_allocate > 0) { - size_t active_layer = layer - 1; - const SimulcastStream& stream = codec_.simulcastStream[active_layer]; - uint32_t allocation = - std::min(left_to_allocate, - stream.maxBitrate - allocated_bitrates_bps[active_layer]); - left_to_allocate -= allocation; - allocated_bitrates_bps[active_layer] += allocation; + uint32_t target_bitrate_kbps = + allocated_bitrates_bps.GetBitrate(simulcast_id, 0) / 1000; + RTC_DCHECK_EQ( + target_bitrate_kbps, + allocated_bitrates_bps.GetSpatialLayerSum(simulcast_id) / 1000); + uint32_t max_bitrate_kbps; + if (codec_.numberOfSimulcastStreams == 0) { + max_bitrate_kbps = codec_.maxBitrate; + + // TODO(holmer): This is a temporary hack for screensharing, where we + // interpret the startBitrate as the encoder target bitrate. This is + // to allow for a different max bitrate, so if the codec can't meet + // the target we still allow it to overshoot up to the max before dropping + // frames. This hack should be improved. + if (codec_.mode == kScreensharing && codec_.targetBitrate > 0 && + (codec_.codecSpecific.VP8.numberOfTemporalLayers == 2 || + codec_.simulcastStream[0].numberOfTemporalLayers == 2)) { + int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate_kbps); + max_bitrate_kbps = std::min(codec_.maxBitrate, target_bitrate_kbps); + target_bitrate_kbps = tl0_bitrate; + } + } else { + max_bitrate_kbps = codec_.simulcastStream[simulcast_id].maxBitrate; + } + + std::vector tl_allocation = tl_it->second->OnRatesUpdated( + target_bitrate_kbps, max_bitrate_kbps, framerate); + + for (size_t tl_index = 0; tl_index < tl_allocation.size(); ++tl_index) { + allocated_bitrates_bps.SetBitrate(simulcast_id, tl_index, + tl_allocation[tl_index] * 1000); + } } return allocated_bitrates_bps; } -uint32_t SimulcastRateAllocator::GetPreferedBitrate() const { - std::vector rates = GetAllocation(codec_.maxBitrate); - uint32_t preferred_bitrate = 0; - for (const uint32_t& rate : rates) { - preferred_bitrate += rate; - } - return preferred_bitrate; +uint32_t SimulcastRateAllocator::GetPreferredBitrateBps(uint32_t framerate) { + BitrateAllocation allocation = + GetAllocation(codec_.maxBitrate * 1000, framerate); + return allocation.get_sum_bps(); } const VideoCodec& webrtc::SimulcastRateAllocator::GetCodec() const { diff --git a/webrtc/modules/video_coding/utility/simulcast_rate_allocator.h b/webrtc/modules/video_coding/utility/simulcast_rate_allocator.h index e7ee772b3c..149e866588 100644 --- a/webrtc/modules/video_coding/utility/simulcast_rate_allocator.h +++ b/webrtc/modules/video_coding/utility/simulcast_rate_allocator.h @@ -11,24 +11,36 @@ #ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_SIMULCAST_RATE_ALLOCATOR_H_ #define WEBRTC_MODULES_VIDEO_CODING_UTILITY_SIMULCAST_RATE_ALLOCATOR_H_ -#include +#include +#include #include "webrtc/base/basictypes.h" #include "webrtc/base/constructormagic.h" +#include "webrtc/common_video/include/video_bitrate_allocator.h" +#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/video_encoder.h" namespace webrtc { -class SimulcastRateAllocator { +class SimulcastRateAllocator : public VideoBitrateAllocator, + public TemporalLayersListener { public: - explicit SimulcastRateAllocator(const VideoCodec& codec); + explicit SimulcastRateAllocator( + const VideoCodec& codec, + std::unique_ptr tl_factory); - std::vector GetAllocation(uint32_t bitrate_kbps) const; - uint32_t GetPreferedBitrate() const; + void OnTemporalLayersCreated(int simulcast_id, + TemporalLayers* layers) override; + + BitrateAllocation GetAllocation(uint32_t total_bitrate_bps, + uint32_t framerate) override; + uint32_t GetPreferredBitrateBps(uint32_t framerate) override; const VideoCodec& GetCodec() const; private: const VideoCodec codec_; + std::map temporal_layers_; + std::unique_ptr tl_factory_; RTC_DISALLOW_COPY_AND_ASSIGN(SimulcastRateAllocator); }; diff --git a/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc b/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc index 57fd5d22bd..ef95cb113f 100644 --- a/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc +++ b/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc @@ -12,6 +12,7 @@ #include #include +#include #include "webrtc/test/gtest.h" @@ -41,43 +42,59 @@ class SimulcastRateAllocatorTest : public ::testing::Test { EXPECT_EQ(expected[i], actual[i]) << "Mismatch at index " << i; } + template + void ExpectEqual(uint32_t (&expected)[S], const BitrateAllocation& actual) { + // EXPECT_EQ(S, actual.size()); + uint32_t sum = 0; + for (size_t i = 0; i < S; ++i) { + uint32_t layer_bitrate = actual.GetSpatialLayerSum(i); + EXPECT_EQ(expected[i] * 1000, layer_bitrate) << "Mismatch at index " << i; + sum += layer_bitrate; + } + EXPECT_EQ(sum, actual.get_sum_bps()); + } + void CreateAllocator() { - allocator_.reset(new SimulcastRateAllocator(codec_)); + allocator_.reset(new SimulcastRateAllocator(codec_, nullptr)); + } + + BitrateAllocation GetAllocation(uint32_t target_bitrate) { + return allocator_->GetAllocation(target_bitrate * 1000, kDefaultFrameRate); } protected: + static const int kDefaultFrameRate = 30; VideoCodec codec_; std::unique_ptr allocator_; }; TEST_F(SimulcastRateAllocatorTest, NoSimulcastBelowMin) { uint32_t expected[] = {codec_.minBitrate}; - ExpectEqual(expected, allocator_->GetAllocation(codec_.minBitrate - 1)); - ExpectEqual(expected, allocator_->GetAllocation(1)); - ExpectEqual(expected, allocator_->GetAllocation(0)); + ExpectEqual(expected, GetAllocation(codec_.minBitrate - 1)); + ExpectEqual(expected, GetAllocation(1)); + ExpectEqual(expected, GetAllocation(0)); } TEST_F(SimulcastRateAllocatorTest, NoSimulcastAboveMax) { uint32_t expected[] = {codec_.maxBitrate}; - ExpectEqual(expected, allocator_->GetAllocation(codec_.maxBitrate + 1)); - ExpectEqual(expected, - allocator_->GetAllocation(std::numeric_limits::max())); + ExpectEqual(expected, GetAllocation(codec_.maxBitrate + 1)); + ExpectEqual(expected, GetAllocation(std::numeric_limits::max())); } TEST_F(SimulcastRateAllocatorTest, NoSimulcastNoMax) { - constexpr uint32_t kMax = std::numeric_limits::max(); + const uint32_t kMax = BitrateAllocation::kMaxBitrateBps / 1000; codec_.maxBitrate = 0; CreateAllocator(); uint32_t expected[] = {kMax}; - ExpectEqual(expected, allocator_->GetAllocation(kMax)); + ExpectEqual(expected, GetAllocation(kMax)); } TEST_F(SimulcastRateAllocatorTest, NoSimulcastWithinLimits) { for (uint32_t bitrate = codec_.minBitrate; bitrate <= codec_.maxBitrate; ++bitrate) { uint32_t expected[] = {bitrate}; - ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + ExpectEqual(expected, GetAllocation(bitrate)); } } @@ -90,9 +107,9 @@ TEST_F(SimulcastRateAllocatorTest, SingleSimulcastBelowMin) { CreateAllocator(); uint32_t expected[] = {kMin}; - ExpectEqual(expected, allocator_->GetAllocation(kMin - 1)); - ExpectEqual(expected, allocator_->GetAllocation(1)); - ExpectEqual(expected, allocator_->GetAllocation(0)); + ExpectEqual(expected, GetAllocation(kMin - 1)); + ExpectEqual(expected, GetAllocation(1)); + ExpectEqual(expected, GetAllocation(0)); } TEST_F(SimulcastRateAllocatorTest, SingleSimulcastAboveMax) { @@ -103,9 +120,9 @@ TEST_F(SimulcastRateAllocatorTest, SingleSimulcastAboveMax) { CreateAllocator(); uint32_t expected[] = {kMax}; - ExpectEqual(expected, allocator_->GetAllocation(kMax + 1)); - ExpectEqual(expected, - allocator_->GetAllocation(std::numeric_limits::max())); + ExpectEqual(expected, GetAllocation(kMax)); + ExpectEqual(expected, GetAllocation(kMax + 1)); + ExpectEqual(expected, GetAllocation(std::numeric_limits::max())); } TEST_F(SimulcastRateAllocatorTest, SingleSimulcastWithinLimits) { @@ -117,7 +134,7 @@ TEST_F(SimulcastRateAllocatorTest, SingleSimulcastWithinLimits) { for (uint32_t bitrate = kMinBitrate; bitrate <= kMaxBitrate; ++bitrate) { uint32_t expected[] = {bitrate}; - ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + ExpectEqual(expected, GetAllocation(bitrate)); } } @@ -139,14 +156,14 @@ TEST_F(SimulcastRateAllocatorTest, OneToThreeStreams) { // Single stream, min bitrate. const uint32_t bitrate = codec_.simulcastStream[0].minBitrate; uint32_t expected[] = {bitrate, 0, 0}; - ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + ExpectEqual(expected, GetAllocation(bitrate)); } { // Single stream at target bitrate. const uint32_t bitrate = codec_.simulcastStream[0].targetBitrate; uint32_t expected[] = {bitrate, 0, 0}; - ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + ExpectEqual(expected, GetAllocation(bitrate)); } { @@ -154,7 +171,7 @@ TEST_F(SimulcastRateAllocatorTest, OneToThreeStreams) { const uint32_t bitrate = codec_.simulcastStream[0].targetBitrate + codec_.simulcastStream[1].minBitrate - 1; uint32_t expected[] = {bitrate, 0, 0}; - ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + ExpectEqual(expected, GetAllocation(bitrate)); } { @@ -163,7 +180,7 @@ TEST_F(SimulcastRateAllocatorTest, OneToThreeStreams) { codec_.simulcastStream[1].minBitrate; uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, codec_.simulcastStream[1].minBitrate, 0}; - ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + ExpectEqual(expected, GetAllocation(bitrate)); } { @@ -172,7 +189,7 @@ TEST_F(SimulcastRateAllocatorTest, OneToThreeStreams) { codec_.simulcastStream[1].maxBitrate; uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, codec_.simulcastStream[1].maxBitrate, 0}; - ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + ExpectEqual(expected, GetAllocation(bitrate)); } { @@ -182,7 +199,7 @@ TEST_F(SimulcastRateAllocatorTest, OneToThreeStreams) { codec_.simulcastStream[1].maxBitrate + 499; uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, codec_.simulcastStream[1].maxBitrate, 0}; - ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + ExpectEqual(expected, GetAllocation(bitrate)); } { @@ -193,7 +210,7 @@ TEST_F(SimulcastRateAllocatorTest, OneToThreeStreams) { uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, codec_.simulcastStream[1].targetBitrate, codec_.simulcastStream[2].minBitrate}; - ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + ExpectEqual(expected, GetAllocation(bitrate)); } { @@ -204,12 +221,13 @@ TEST_F(SimulcastRateAllocatorTest, OneToThreeStreams) { uint32_t expected[] = {codec_.simulcastStream[0].targetBitrate, codec_.simulcastStream[1].targetBitrate, codec_.simulcastStream[2].maxBitrate}; - ExpectEqual(expected, allocator_->GetAllocation(bitrate)); + ExpectEqual(expected, GetAllocation(bitrate)); } } -TEST_F(SimulcastRateAllocatorTest, GetPreferredBitrate) { - EXPECT_EQ(codec_.maxBitrate, allocator_->GetPreferedBitrate()); +TEST_F(SimulcastRateAllocatorTest, GetPreferredBitrateBps) { + EXPECT_EQ(codec_.maxBitrate * 1000, + allocator_->GetPreferredBitrateBps(codec_.maxFramerate)); } TEST_F(SimulcastRateAllocatorTest, GetPreferredBitrateSimulcast) { @@ -228,12 +246,13 @@ TEST_F(SimulcastRateAllocatorTest, GetPreferredBitrateSimulcast) { codec_.simulcastStream[2].maxBitrate = 4000; CreateAllocator(); - uint32_t preferred_bitrate; - preferred_bitrate = codec_.simulcastStream[0].targetBitrate; - preferred_bitrate += codec_.simulcastStream[1].targetBitrate; - preferred_bitrate += codec_.simulcastStream[2].maxBitrate; + uint32_t preferred_bitrate_kbps; + preferred_bitrate_kbps = codec_.simulcastStream[0].targetBitrate; + preferred_bitrate_kbps += codec_.simulcastStream[1].targetBitrate; + preferred_bitrate_kbps += codec_.simulcastStream[2].maxBitrate; - EXPECT_EQ(preferred_bitrate, allocator_->GetPreferedBitrate()); + EXPECT_EQ(preferred_bitrate_kbps * 1000, + allocator_->GetPreferredBitrateBps(codec_.maxFramerate)); } } // namespace webrtc diff --git a/webrtc/modules/video_coding/utility/video_coding_utility.gyp b/webrtc/modules/video_coding/utility/video_coding_utility.gyp index b1158a2298..91d1ac18a3 100644 --- a/webrtc/modules/video_coding/utility/video_coding_utility.gyp +++ b/webrtc/modules/video_coding/utility/video_coding_utility.gyp @@ -19,6 +19,8 @@ '<(webrtc_root)/system_wrappers/system_wrappers.gyp:system_wrappers', ], 'sources': [ + 'default_video_bitrate_allocator.cc', + 'default_video_bitrate_allocator.h', 'frame_dropper.cc', 'frame_dropper.h', 'ivf_file_writer.cc', diff --git a/webrtc/modules/video_coding/video_codec_initializer.cc b/webrtc/modules/video_coding/video_codec_initializer.cc new file mode 100644 index 0000000000..ef1b61b022 --- /dev/null +++ b/webrtc/modules/video_coding/video_codec_initializer.cc @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "webrtc/modules/video_coding/include/video_codec_initializer.h" + +#include "webrtc/base/basictypes.h" +#include "webrtc/common_video/include/video_bitrate_allocator.h" +#include "webrtc/common_types.h" +#include "webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h" +#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" +#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" +#include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h" +#include "webrtc/system_wrappers/include/clock.h" + +namespace webrtc { + +struct ScreenshareTemporalLayersFactory : webrtc::TemporalLayersFactory { + ScreenshareTemporalLayersFactory() {} + virtual ~ScreenshareTemporalLayersFactory() {} + + virtual webrtc::TemporalLayers* Create(int simulcast_id, + int num_temporal_layers, + uint8_t initial_tl0_pic_idx) const { + webrtc::TemporalLayers* tl = new webrtc::ScreenshareLayers( + num_temporal_layers, rand(), webrtc::Clock::GetRealTimeClock()); + if (listener_) + listener_->OnTemporalLayersCreated(simulcast_id, tl); + return tl; + } +}; + +bool VideoCodecInitializer::SetupCodec( + const VideoEncoderConfig& config, + const VideoSendStream::Config::EncoderSettings settings, + const std::vector& streams, + VideoCodec* codec, + std::unique_ptr* bitrate_allocator) { + *codec = VideoEncoderConfigToVideoCodec( + config, streams, settings.payload_name, settings.payload_type); + + std::unique_ptr tl_factory; + switch (codec->codecType) { + case kVideoCodecVP8: { + if (!codec->VP8()->tl_factory) { + if (codec->mode == kScreensharing && + codec->numberOfSimulcastStreams == 1 && + codec->VP8()->numberOfTemporalLayers == 2) { + // Conference mode temporal layering for screen content. + tl_factory.reset(new ScreenshareTemporalLayersFactory()); + } else { + // Standard video temporal layers. + tl_factory.reset(new TemporalLayersFactory()); + } + codec->VP8()->tl_factory = tl_factory.get(); + } + break; + } + default: { + // TODO(sprang): Warn, once we have specific allocators for all supported + // codec types. + break; + } + } + *bitrate_allocator = CreateBitrateAllocator(*codec, std::move(tl_factory)); + + return true; +} + +std::unique_ptr +VideoCodecInitializer::CreateBitrateAllocator( + const VideoCodec& codec, + std::unique_ptr tl_factory) { + std::unique_ptr rate_allocator; + + switch (codec.codecType) { + case kVideoCodecVP8: { + // Set up default VP8 temporal layer factory, if not provided. + rate_allocator.reset( + new SimulcastRateAllocator(codec, std::move(tl_factory))); + } break; + default: + rate_allocator.reset(new DefaultVideoBitrateAllocator(codec)); + } + + return rate_allocator; +} + +// TODO(sprang): Split this up and separate the codec specific parts. +VideoCodec VideoCodecInitializer::VideoEncoderConfigToVideoCodec( + const VideoEncoderConfig& config, + const std::vector& streams, + const std::string& payload_name, + int payload_type) { + static const int kEncoderMinBitrateKbps = 30; + RTC_DCHECK(!streams.empty()); + RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0); + + VideoCodec video_codec; + memset(&video_codec, 0, sizeof(video_codec)); + video_codec.codecType = PayloadNameToCodecType(payload_name) + .value_or(VideoCodecType::kVideoCodecGeneric); + + switch (config.content_type) { + case VideoEncoderConfig::ContentType::kRealtimeVideo: + video_codec.mode = kRealtimeVideo; + break; + case VideoEncoderConfig::ContentType::kScreen: + video_codec.mode = kScreensharing; + if (streams.size() == 1 && + streams[0].temporal_layer_thresholds_bps.size() == 1) { + video_codec.targetBitrate = + streams[0].temporal_layer_thresholds_bps[0] / 1000; + } + break; + } + + if (config.encoder_specific_settings) + config.encoder_specific_settings->FillEncoderSpecificSettings(&video_codec); + + switch (video_codec.codecType) { + case kVideoCodecVP8: { + if (!config.encoder_specific_settings) + video_codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings(); + video_codec.codecSpecific.VP8.numberOfTemporalLayers = + static_cast( + streams.back().temporal_layer_thresholds_bps.size() + 1); + break; + } + case kVideoCodecVP9: { + if (!config.encoder_specific_settings) + video_codec.codecSpecific.VP9 = VideoEncoder::GetDefaultVp9Settings(); + if (video_codec.mode == kScreensharing && + config.encoder_specific_settings) { + video_codec.codecSpecific.VP9.flexibleMode = true; + // For now VP9 screensharing use 1 temporal and 2 spatial layers. + RTC_DCHECK_EQ(1, video_codec.codecSpecific.VP9.numberOfTemporalLayers); + RTC_DCHECK_EQ(2, video_codec.codecSpecific.VP9.numberOfSpatialLayers); + } + video_codec.codecSpecific.VP9.numberOfTemporalLayers = + static_cast( + streams.back().temporal_layer_thresholds_bps.size() + 1); + break; + } + case kVideoCodecH264: { + if (!config.encoder_specific_settings) + video_codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings(); + break; + } + default: + // TODO(pbos): Support encoder_settings codec-agnostically. + RTC_DCHECK(!config.encoder_specific_settings) + << "Encoder-specific settings for codec type not wired up."; + break; + } + + strncpy(video_codec.plName, payload_name.c_str(), kPayloadNameSize - 1); + video_codec.plName[kPayloadNameSize - 1] = '\0'; + video_codec.plType = payload_type; + video_codec.numberOfSimulcastStreams = + static_cast(streams.size()); + video_codec.minBitrate = streams[0].min_bitrate_bps / 1000; + if (video_codec.minBitrate < kEncoderMinBitrateKbps) + video_codec.minBitrate = kEncoderMinBitrateKbps; + RTC_DCHECK_LE(streams.size(), static_cast(kMaxSimulcastStreams)); + if (video_codec.codecType == kVideoCodecVP9) { + // If the vector is empty, bitrates will be configured automatically. + RTC_DCHECK(config.spatial_layers.empty() || + config.spatial_layers.size() == + video_codec.codecSpecific.VP9.numberOfSpatialLayers); + RTC_DCHECK_LE(video_codec.codecSpecific.VP9.numberOfSpatialLayers, + kMaxSimulcastStreams); + for (size_t i = 0; i < config.spatial_layers.size(); ++i) + video_codec.spatialLayers[i] = config.spatial_layers[i]; + } + for (size_t i = 0; i < streams.size(); ++i) { + SimulcastStream* sim_stream = &video_codec.simulcastStream[i]; + RTC_DCHECK_GT(streams[i].width, 0u); + RTC_DCHECK_GT(streams[i].height, 0u); + RTC_DCHECK_GT(streams[i].max_framerate, 0); + // Different framerates not supported per stream at the moment. + RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate); + RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0); + RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps); + RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps); + RTC_DCHECK_GE(streams[i].max_qp, 0); + + sim_stream->width = static_cast(streams[i].width); + sim_stream->height = static_cast(streams[i].height); + sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000; + sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000; + sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000; + sim_stream->qpMax = streams[i].max_qp; + sim_stream->numberOfTemporalLayers = static_cast( + streams[i].temporal_layer_thresholds_bps.size() + 1); + + video_codec.width = + std::max(video_codec.width, static_cast(streams[i].width)); + video_codec.height = + std::max(video_codec.height, static_cast(streams[i].height)); + video_codec.minBitrate = + std::min(static_cast(video_codec.minBitrate), + static_cast(streams[i].min_bitrate_bps / 1000)); + video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000; + video_codec.qpMax = std::max(video_codec.qpMax, + static_cast(streams[i].max_qp)); + } + + if (video_codec.maxBitrate == 0) { + // Unset max bitrate -> cap to one bit per pixel. + video_codec.maxBitrate = + (video_codec.width * video_codec.height * video_codec.maxFramerate) / + 1000; + } + if (video_codec.maxBitrate < kEncoderMinBitrateKbps) + video_codec.maxBitrate = kEncoderMinBitrateKbps; + + RTC_DCHECK_GT(streams[0].max_framerate, 0); + video_codec.maxFramerate = streams[0].max_framerate; + return video_codec; +} + +} // namespace webrtc diff --git a/webrtc/modules/video_coding/video_coding.gypi b/webrtc/modules/video_coding/video_coding.gypi index 0b682c7d99..f9b1cf512b 100644 --- a/webrtc/modules/video_coding/video_coding.gypi +++ b/webrtc/modules/video_coding/video_coding.gypi @@ -23,6 +23,7 @@ ], 'sources': [ # interfaces + 'include/video_codec_initializer.h', 'include/video_coding.h', 'include/video_coding_defines.h', @@ -86,6 +87,7 @@ 'session_info.cc', 'timestamp_map.cc', 'timing.cc', + 'video_codec_initializer.cc', 'video_coding_impl.cc', 'video_sender.cc', 'video_receiver.cc', diff --git a/webrtc/modules/video_coding/video_coding_impl.cc b/webrtc/modules/video_coding/video_coding_impl.cc index 2f709b610c..021281bffa 100644 --- a/webrtc/modules/video_coding/video_coding_impl.cc +++ b/webrtc/modules/video_coding/video_coding_impl.cc @@ -11,10 +11,14 @@ #include "webrtc/modules/video_coding/video_coding_impl.h" #include +#include #include "webrtc/common_types.h" +#include "webrtc/common_video/include/video_bitrate_allocator.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/base/criticalsection.h" +#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" +#include "webrtc/modules/video_coding/include/video_codec_initializer.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/encoded_frame.h" #include "webrtc/modules/video_coding/jitter_buffer.h" @@ -104,6 +108,21 @@ class VideoCodingModuleImpl : public VideoCodingModule { int32_t RegisterSendCodec(const VideoCodec* sendCodec, uint32_t numberOfCores, uint32_t maxPayloadSize) override { + if (sendCodec != nullptr && sendCodec->codecType == kVideoCodecVP8) { + // Set up a rate allocator and temporal layers factory for this vp8 + // instance. The codec impl will have a raw pointer to the TL factory, + // and will call it when initializing. Since this can happen + // asynchronously keep the instance alive until destruction or until a + // new send codec is registered. + VideoCodec vp8_codec = *sendCodec; + std::unique_ptr tl_factory( + new TemporalLayersFactory()); + vp8_codec.VP8()->tl_factory = tl_factory.get(); + rate_allocator_ = VideoCodecInitializer::CreateBitrateAllocator( + vp8_codec, std::move(tl_factory)); + return sender_.RegisterSendCodec(&vp8_codec, numberOfCores, + maxPayloadSize); + } return sender_.RegisterSendCodec(sendCodec, numberOfCores, maxPayloadSize); } @@ -126,7 +145,8 @@ class VideoCodingModuleImpl : public VideoCodingModule { int32_t SetChannelParameters(uint32_t target_bitrate, // bits/s. uint8_t lossRate, int64_t rtt) override { - return sender_.SetChannelParameters(target_bitrate, lossRate, rtt); + return sender_.SetChannelParameters(target_bitrate, lossRate, rtt, + rate_allocator_.get()); } int32_t RegisterProtectionCallback( @@ -256,6 +276,7 @@ class VideoCodingModuleImpl : public VideoCodingModule { private: EncodedImageCallbackWrapper post_encode_callback_; vcm::VideoSender sender_; + std::unique_ptr rate_allocator_; vcm::VideoReceiver receiver_; }; } // namespace diff --git a/webrtc/modules/video_coding/video_coding_impl.h b/webrtc/modules/video_coding/video_coding_impl.h index cb24654041..0533634546 100644 --- a/webrtc/modules/video_coding/video_coding_impl.h +++ b/webrtc/modules/video_coding/video_coding_impl.h @@ -34,6 +34,8 @@ namespace webrtc { +class VideoBitrateAllocator; + namespace vcm { class VCMProcessTimer { @@ -75,9 +77,15 @@ class VideoSender : public Module { int Bitrate(unsigned int* bitrate) const; int FrameRate(unsigned int* framerate) const; - int32_t SetChannelParameters(uint32_t target_bitrate, // bits/s. + int32_t SetChannelParameters(uint32_t target_bitrate_bps, uint8_t lossRate, - int64_t rtt); + int64_t rtt, + VideoBitrateAllocator* bitrate_allocator); + // Calls SetChannelParameters(), with the previous target bitrate, loss rate + // and RTT, but reallocates the bitrate allocation based on a presumably + // updated codec configuration. + int32_t UpdateChannelParemeters(VideoBitrateAllocator* bitrate_allocator); + // Deprecated: // TODO(perkj): Remove once no projects use it. int32_t RegisterProtectionCallback(VCMProtectionCallback* protection); diff --git a/webrtc/modules/video_coding/video_sender.cc b/webrtc/modules/video_coding/video_sender.cc index b7664a719b..35fd39cd05 100644 --- a/webrtc/modules/video_coding/video_sender.cc +++ b/webrtc/modules/video_coding/video_sender.cc @@ -14,9 +14,12 @@ #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/common_types.h" +#include "webrtc/common_video/include/video_bitrate_allocator.h" #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" +#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/encoded_frame.h" +#include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h" #include "webrtc/modules/video_coding/utility/quality_scaler.h" #include "webrtc/modules/video_coding/video_coding_impl.h" #include "webrtc/system_wrappers/include/clock.h" @@ -36,7 +39,7 @@ VideoSender::VideoSender(Clock* clock, frame_dropper_enabled_(true), _sendStatsTimer(1000, clock_), current_codec_(), - encoder_params_({0, 0, 0, 0}), + encoder_params_({BitrateAllocation(), 0, 0, 0}), encoder_has_internal_source_(false), next_frame_types_(1, kVideoFrameDelta) { _mediaOpt.Reset(); @@ -172,7 +175,7 @@ int VideoSender::Bitrate(unsigned int* bitrate) const { if (!_encoder) return VCM_UNINITIALIZED; - *bitrate = _encoder->GetEncoderParameters().target_bitrate; + *bitrate = _encoder->GetEncoderParameters().target_bitrate.get_sum_bps(); return 0; } @@ -189,15 +192,38 @@ int VideoSender::FrameRate(unsigned int* framerate) const { return 0; } -int32_t VideoSender::SetChannelParameters(uint32_t target_bitrate, - uint8_t lossRate, - int64_t rtt) { - uint32_t target_rate = - _mediaOpt.SetTargetRates(target_bitrate, lossRate, rtt); +int32_t VideoSender::UpdateChannelParemeters( + VideoBitrateAllocator* bitrate_allocator) { + EncoderParameters encoder_params; + { + rtc::CritScope cs(¶ms_crit_); + encoder_params = encoder_params_; + } + return SetChannelParameters(encoder_params.target_bitrate.get_sum_bps(), + encoder_params.loss_rate, encoder_params.rtt, + bitrate_allocator); +} + +int32_t VideoSender::SetChannelParameters( + uint32_t target_bitrate_bps, + uint8_t lossRate, + int64_t rtt, + VideoBitrateAllocator* bitrate_allocator) { + uint32_t video_target_rate_bps = + _mediaOpt.SetTargetRates(target_bitrate_bps, lossRate, rtt); uint32_t input_frame_rate = _mediaOpt.InputFrameRate(); + BitrateAllocation bitrate_allocation; + if (bitrate_allocator) { + bitrate_allocation = bitrate_allocator->GetAllocation(video_target_rate_bps, + input_frame_rate); + } else { + DefaultVideoBitrateAllocator default_allocator(current_codec_); + bitrate_allocation = default_allocator.GetAllocation(video_target_rate_bps, + input_frame_rate); + } - EncoderParameters encoder_params = {target_rate, lossRate, rtt, + EncoderParameters encoder_params = {bitrate_allocation, lossRate, rtt, input_frame_rate}; bool encoder_has_internal_source; { @@ -228,7 +254,7 @@ void VideoSender::SetEncoderParameters(EncoderParameters params, // encoder implementations behave when given a zero target bitrate. // TODO(perkj): Make sure all known encoder implementations handle zero // target bitrate and remove this check. - if (!has_internal_source && params.target_bitrate == 0) + if (!has_internal_source && params.target_bitrate.get_sum_bps() == 0) return; if (params.input_frame_rate == 0) { @@ -268,7 +294,8 @@ int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame, SetEncoderParameters(encoder_params, encoder_has_internal_source); if (_mediaOpt.DropFrame()) { LOG(LS_VERBOSE) << "Drop Frame " - << "target bitrate " << encoder_params.target_bitrate + << "target bitrate " + << encoder_params.target_bitrate.get_sum_bps() << " loss rate " << encoder_params.loss_rate << " rtt " << encoder_params.rtt << " input frame rate " << encoder_params.input_frame_rate; diff --git a/webrtc/modules/video_coding/video_sender_unittest.cc b/webrtc/modules/video_coding/video_sender_unittest.cc index 71602f0ea1..dd59b02b4d 100644 --- a/webrtc/modules/video_coding/video_sender_unittest.cc +++ b/webrtc/modules/video_coding/video_sender_unittest.cc @@ -19,6 +19,8 @@ #include "webrtc/modules/video_coding/include/video_coding.h" #include "webrtc/modules/video_coding/test/test_util.h" #include "webrtc/modules/video_coding/video_coding_impl.h" +#include "webrtc/modules/video_coding/utility/default_video_bitrate_allocator.h" +#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" #include "webrtc/system_wrappers/include/clock.h" #include "webrtc/test/frame_generator.h" #include "webrtc/test/gtest.h" @@ -196,6 +198,10 @@ class TestVideoSender : public ::testing::Test { }; class TestVideoSenderWithMockEncoder : public TestVideoSender { + public: + TestVideoSenderWithMockEncoder() {} + ~TestVideoSenderWithMockEncoder() override {} + protected: void SetUp() override { TestVideoSender::SetUp(); @@ -212,6 +218,7 @@ class TestVideoSenderWithMockEncoder : public TestVideoSender { generator_.reset( new EmptyFrameGenerator(settings_.width, settings_.height)); EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200)); + rate_allocator_.reset(new DefaultVideoBitrateAllocator(settings_)); } void TearDown() override { sender_.reset(); } @@ -261,6 +268,7 @@ class TestVideoSenderWithMockEncoder : public TestVideoSender { VideoCodec settings_; NiceMock encoder_; + std::unique_ptr rate_allocator_; }; TEST_F(TestVideoSenderWithMockEncoder, TestIntraRequests) { @@ -291,14 +299,19 @@ TEST_F(TestVideoSenderWithMockEncoder, TestIntraRequests) { } TEST_F(TestVideoSenderWithMockEncoder, TestSetRate) { - const uint32_t new_bitrate = settings_.startBitrate + 300; - EXPECT_CALL(encoder_, SetRates(new_bitrate, _)).Times(1).WillOnce(Return(0)); - sender_->SetChannelParameters(new_bitrate * 1000, 0, 200); + const uint32_t new_bitrate_kbps = settings_.startBitrate + 300; + BitrateAllocation new_rate_allocation = rate_allocator_->GetAllocation( + new_bitrate_kbps * 1000, settings_.maxFramerate); + EXPECT_CALL(encoder_, SetRateAllocation(new_rate_allocation, _)) + .Times(1) + .WillOnce(Return(0)); + sender_->SetChannelParameters(new_bitrate_kbps * 1000, 0, 200, + rate_allocator_.get()); AddFrame(); // Expect no call to encoder_.SetRates if the new bitrate is zero. - EXPECT_CALL(encoder_, SetRates(new_bitrate, _)).Times(0); - sender_->SetChannelParameters(0, 0, 200); + EXPECT_CALL(encoder_, SetRateAllocation(_, _)).Times(0); + sender_->SetChannelParameters(0, 0, 200, rate_allocator_.get()); AddFrame(); } @@ -329,13 +342,19 @@ TEST_F(TestVideoSenderWithMockEncoder, TestEncoderParametersForInternalSource) { EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200)); // Update encoder bitrate parameters. We expect that to immediately call // SetRates on the encoder without waiting for AddFrame processing. - const uint32_t new_bitrate = settings_.startBitrate + 300; - EXPECT_CALL(encoder_, SetRates(new_bitrate, _)).Times(1).WillOnce(Return(0)); - sender_->SetChannelParameters(new_bitrate * 1000, 0, 200); + const uint32_t new_bitrate_kbps = settings_.startBitrate + 300; + BitrateAllocation new_rate_allocation = rate_allocator_->GetAllocation( + new_bitrate_kbps * 1000, settings_.maxFramerate); + EXPECT_CALL(encoder_, SetRateAllocation(new_rate_allocation, _)) + .Times(1) + .WillOnce(Return(0)); + sender_->SetChannelParameters(new_bitrate_kbps * 1000, 0, 200, + rate_allocator_.get()); } TEST_F(TestVideoSenderWithMockEncoder, EncoderFramerateUpdatedViaProcess) { - sender_->SetChannelParameters(settings_.startBitrate * 1000, 0, 200); + sender_->SetChannelParameters(settings_.startBitrate * 1000, 0, 200, + rate_allocator_.get()); const int64_t kRateStatsWindowMs = 2000; const uint32_t kInputFps = 20; int64_t start_time = clock_.TimeInMilliseconds(); @@ -343,7 +362,9 @@ TEST_F(TestVideoSenderWithMockEncoder, EncoderFramerateUpdatedViaProcess) { AddFrame(); clock_.AdvanceTimeMilliseconds(1000 / kInputFps); } - EXPECT_CALL(encoder_, SetRates(_, kInputFps)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(encoder_, SetRateAllocation(_, kInputFps)) + .Times(1) + .WillOnce(Return(0)); sender_->Process(); AddFrame(); } @@ -361,23 +382,29 @@ TEST_F(TestVideoSenderWithMockEncoder, EXPECT_CALL(encoder_, SetChannelParameters(kLossRate, kRtt)) .Times(1) .WillOnce(Return(0)); - sender_->SetChannelParameters(settings_.startBitrate * 1000, kLossRate, kRtt); + sender_->SetChannelParameters(settings_.startBitrate * 1000, kLossRate, kRtt, + rate_allocator_.get()); while (clock_.TimeInMilliseconds() < start_time + kRateStatsWindowMs) { AddFrame(); clock_.AdvanceTimeMilliseconds(1000 / kInputFps); } // After process, input framerate should be updated but not ChannelParameters // as they are the same as before. - EXPECT_CALL(encoder_, SetRates(_, kInputFps)).Times(1).WillOnce(Return(0)); + EXPECT_CALL(encoder_, SetRateAllocation(_, kInputFps)) + .Times(1) + .WillOnce(Return(0)); sender_->Process(); AddFrame(); // Call to SetChannelParameters with changed bitrate should call encoder // SetRates but not encoder SetChannelParameters (that are unchanged). - EXPECT_CALL(encoder_, SetRates(2 * settings_.startBitrate, kInputFps)) + uint32_t new_bitrate_bps = 2 * settings_.startBitrate * 1000; + BitrateAllocation new_rate_allocation = + rate_allocator_->GetAllocation(new_bitrate_bps, kInputFps); + EXPECT_CALL(encoder_, SetRateAllocation(new_rate_allocation, kInputFps)) .Times(1) .WillOnce(Return(0)); - sender_->SetChannelParameters(2 * settings_.startBitrate * 1000, kLossRate, - kRtt); + sender_->SetChannelParameters(new_bitrate_bps, kLossRate, kRtt, + rate_allocator_.get()); AddFrame(); } @@ -400,6 +427,12 @@ class TestVideoSenderWithVp8 : public TestVideoSender { codec_.minBitrate = 10; codec_.startBitrate = codec_bitrate_kbps_; codec_.maxBitrate = codec_bitrate_kbps_; + + TemporalLayersFactory* tl_factory = new TemporalLayersFactory(); + rate_allocator_.reset(new SimulcastRateAllocator( + codec_, std::unique_ptr(tl_factory))); + codec_.codecSpecific.VP8.tl_factory = tl_factory; + encoder_.reset(VP8Encoder::Create()); sender_->RegisterExternalEncoder(encoder_.get(), codec_.plType, false); EXPECT_EQ(0, sender_->RegisterSendCodec(&codec_, 1, 1200)); @@ -425,8 +458,9 @@ class TestVideoSenderWithVp8 : public TestVideoSender { // Note: SetChannelParameters fails if less than 2 frames are in the // buffer since it will fail to calculate the framerate. if (i != 0) { - EXPECT_EQ(VCM_OK, sender_->SetChannelParameters( - available_bitrate_kbps_ * 1000, 0, 200)); + EXPECT_EQ(VCM_OK, + sender_->SetChannelParameters(available_bitrate_kbps_ * 1000, + 0, 200, rate_allocator_.get())); } } } @@ -447,6 +481,7 @@ class TestVideoSenderWithVp8 : public TestVideoSender { VideoCodec codec_; int codec_bitrate_kbps_; int available_bitrate_kbps_; + std::unique_ptr rate_allocator_; }; #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) @@ -476,11 +511,15 @@ TEST_F(TestVideoSenderWithVp8, MAYBE_FixedTemporalLayersStrategy) { #endif TEST_F(TestVideoSenderWithVp8, MAYBE_RealTimeTemporalLayersStrategy) { VideoCodec codec = MakeVp8VideoCodec(352, 288, 3); - RealTimeTemporalLayersFactory realtime_tl_factory; - codec.VP8()->tl_factory = &realtime_tl_factory; codec.minBitrate = 10; codec.startBitrate = codec_bitrate_kbps_; codec.maxBitrate = codec_bitrate_kbps_; + + TemporalLayersFactory* tl_factory = new RealTimeTemporalLayersFactory(); + rate_allocator_.reset(new SimulcastRateAllocator( + codec, std::unique_ptr(tl_factory))); + codec.VP8()->tl_factory = tl_factory; + EXPECT_EQ(0, sender_->RegisterSendCodec(&codec, 1, 1200)); const int low_b = codec_bitrate_kbps_ * 0.4; diff --git a/webrtc/test/configurable_frame_size_encoder.cc b/webrtc/test/configurable_frame_size_encoder.cc index 905b69aae8..bef62405dd 100644 --- a/webrtc/test/configurable_frame_size_encoder.cc +++ b/webrtc/test/configurable_frame_size_encoder.cc @@ -12,6 +12,7 @@ #include +#include "webrtc/base/checks.h" #include "webrtc/common_video/include/video_image.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/test/gtest.h" @@ -72,8 +73,9 @@ int32_t ConfigurableFrameSizeEncoder::SetChannelParameters(uint32_t packet_loss, return WEBRTC_VIDEO_CODEC_OK; } -int32_t ConfigurableFrameSizeEncoder::SetRates(uint32_t new_bit_rate, - uint32_t frame_rate) { +int32_t ConfigurableFrameSizeEncoder::SetRateAllocation( + const BitrateAllocation& allocation, + uint32_t framerate) { return WEBRTC_VIDEO_CODEC_OK; } @@ -82,7 +84,7 @@ int32_t ConfigurableFrameSizeEncoder::SetPeriodicKeyFrames(bool enable) { } int32_t ConfigurableFrameSizeEncoder::SetFrameSize(size_t size) { - assert(size <= max_frame_size_); + RTC_DCHECK_LE(size, max_frame_size_); current_frame_size_ = size; return WEBRTC_VIDEO_CODEC_OK; } diff --git a/webrtc/test/configurable_frame_size_encoder.h b/webrtc/test/configurable_frame_size_encoder.h index d269441667..a3820baf7e 100644 --- a/webrtc/test/configurable_frame_size_encoder.h +++ b/webrtc/test/configurable_frame_size_encoder.h @@ -39,7 +39,8 @@ class ConfigurableFrameSizeEncoder : public VideoEncoder { int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; - int32_t SetRates(uint32_t new_bit_rate, uint32_t frame_rate) override; + int32_t SetRateAllocation(const BitrateAllocation& allocation, + uint32_t framerate) override; int32_t SetPeriodicKeyFrames(bool enable) override; diff --git a/webrtc/test/fake_encoder.cc b/webrtc/test/fake_encoder.cc index f518ce3919..26ce0c6608 100644 --- a/webrtc/test/fake_encoder.cc +++ b/webrtc/test/fake_encoder.cc @@ -23,7 +23,6 @@ namespace test { FakeEncoder::FakeEncoder(Clock* clock) : clock_(clock), callback_(NULL), - target_bitrate_kbps_(0), max_target_bitrate_kbps_(-1), last_encode_time_ms_(0) { // Generate some arbitrary not-all-zero data @@ -43,7 +42,7 @@ int32_t FakeEncoder::InitEncode(const VideoCodec* config, int32_t number_of_cores, size_t max_payload_size) { config_ = *config; - target_bitrate_kbps_ = config_.startBitrate; + target_bitrate_.SetBitrate(0, 0, config_.startBitrate * 1000); return 0; } @@ -65,8 +64,8 @@ int32_t FakeEncoder::Encode(const VideoFrame& input_image, time_since_last_encode_ms = 3 * 1000 / config_.maxFramerate; } - size_t bits_available = - static_cast(target_bitrate_kbps_ * time_since_last_encode_ms); + size_t bits_available = static_cast(target_bitrate_.get_sum_kbps() * + time_since_last_encode_ms); size_t min_bits = static_cast( config_.simulcastStream[0].minBitrate * time_since_last_encode_ms); if (bits_available < min_bits) @@ -133,8 +132,9 @@ int32_t FakeEncoder::SetChannelParameters(uint32_t packet_loss, int64_t rtt) { return 0; } -int32_t FakeEncoder::SetRates(uint32_t new_target_bitrate, uint32_t framerate) { - target_bitrate_kbps_ = new_target_bitrate; +int32_t FakeEncoder::SetRateAllocation(const BitrateAllocation& rate_allocation, + uint32_t framerate) { + target_bitrate_ = rate_allocation; return 0; } diff --git a/webrtc/test/fake_encoder.h b/webrtc/test/fake_encoder.h index d5e7b480df..57b39a4404 100644 --- a/webrtc/test/fake_encoder.h +++ b/webrtc/test/fake_encoder.h @@ -39,7 +39,8 @@ class FakeEncoder : public VideoEncoder { EncodedImageCallback* callback) override; int32_t Release() override; int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; - int32_t SetRates(uint32_t new_target_bitrate, uint32_t framerate) override; + int32_t SetRateAllocation(const BitrateAllocation& rate_allocation, + uint32_t framerate) override; const char* ImplementationName() const override; static const char* kImplementationName; @@ -48,7 +49,7 @@ class FakeEncoder : public VideoEncoder { Clock* const clock_; VideoCodec config_; EncodedImageCallback* callback_; - int target_bitrate_kbps_; + BitrateAllocation target_bitrate_; int max_target_bitrate_kbps_; int64_t last_encode_time_ms_; uint8_t encoded_buffer_[100000]; diff --git a/webrtc/video/end_to_end_tests.cc b/webrtc/video/end_to_end_tests.cc index 492f55d6ec..fde2ce29aa 100644 --- a/webrtc/video/end_to_end_tests.cc +++ b/webrtc/video/end_to_end_tests.cc @@ -2550,12 +2550,13 @@ TEST_F(EndToEndTest, ReportsSetEncoderRates) { RTC_DCHECK_EQ(1u, encoder_config->number_of_streams); } - int32_t SetRates(uint32_t new_target_bitrate, uint32_t framerate) override { + int32_t SetRateAllocation(const BitrateAllocation& rate_allocation, + uint32_t framerate) override { // Make sure not to trigger on any default zero bitrates. - if (new_target_bitrate == 0) + if (rate_allocation.get_sum_bps() == 0) return 0; rtc::CritScope lock(&crit_); - bitrate_kbps_ = new_target_bitrate; + bitrate_kbps_ = rate_allocation.get_sum_kbps(); observation_complete_.Set(); return 0; } diff --git a/webrtc/video/video_encoder.cc b/webrtc/video/video_encoder.cc index bb6e9e2308..dbfd0eb8db 100644 --- a/webrtc/video/video_encoder.cc +++ b/webrtc/video/video_encoder.cc @@ -67,8 +67,13 @@ VideoEncoder::EncoderType VideoEncoder::CodecToEncoderType( VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper( VideoCodecType codec_type, webrtc::VideoEncoder* encoder) - : rates_set_(false), + : number_of_cores_(0), + max_payload_size_(0), + rates_set_(false), + framerate_(0), channel_parameters_set_(false), + packet_loss_(0), + rtt_(0), encoder_type_(CodecToEncoderType(codec_type)), encoder_(encoder), callback_(nullptr) {} @@ -92,7 +97,7 @@ bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder() { if (callback_) fallback_encoder_->RegisterEncodeCompleteCallback(callback_); if (rates_set_) - fallback_encoder_->SetRates(bitrate_, framerate_); + fallback_encoder_->SetRateAllocation(bitrate_allocation_, framerate_); if (channel_parameters_set_) fallback_encoder_->SetChannelParameters(packet_loss_, rtt_); @@ -189,14 +194,15 @@ int32_t VideoEncoderSoftwareFallbackWrapper::SetChannelParameters( return ret; } -int32_t VideoEncoderSoftwareFallbackWrapper::SetRates(uint32_t bitrate, - uint32_t framerate) { +int32_t VideoEncoderSoftwareFallbackWrapper::SetRateAllocation( + const BitrateAllocation& bitrate_allocation, + uint32_t framerate) { rates_set_ = true; - bitrate_ = bitrate; + bitrate_allocation_ = bitrate_allocation; framerate_ = framerate; - int32_t ret = encoder_->SetRates(bitrate, framerate); + int32_t ret = encoder_->SetRateAllocation(bitrate_allocation_, framerate); if (fallback_encoder_) - return fallback_encoder_->SetRates(bitrate, framerate); + return fallback_encoder_->SetRateAllocation(bitrate_allocation_, framerate); return ret; } diff --git a/webrtc/video/video_encoder_unittest.cc b/webrtc/video/video_encoder_unittest.cc index 5f6f17e7b4..d14215aa99 100644 --- a/webrtc/video/video_encoder_unittest.cc +++ b/webrtc/video/video_encoder_unittest.cc @@ -10,8 +10,11 @@ #include "webrtc/video_encoder.h" +#include "webrtc/base/checks.h" +#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/modules/video_coding/include/video_codec_interface.h" #include "webrtc/modules/video_coding/include/video_error_codes.h" +#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" #include "webrtc/test/gtest.h" namespace webrtc { @@ -63,7 +66,8 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { return WEBRTC_VIDEO_CODEC_OK; } - int32_t SetRates(uint32_t bitrate, uint32_t framerate) override { + int32_t SetRateAllocation(const BitrateAllocation& bitrate_allocation, + uint32_t framerate) override { ++set_rates_count_; return WEBRTC_VIDEO_CODEC_OK; } @@ -117,6 +121,7 @@ class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test { VideoEncoderSoftwareFallbackWrapper fallback_wrapper_; VideoCodec codec_ = {}; std::unique_ptr frame_; + std::unique_ptr rate_allocator_; }; void VideoEncoderSoftwareFallbackWrapperTest::EncodeFrame() { @@ -139,10 +144,19 @@ void VideoEncoderSoftwareFallbackWrapperTest::UtilizeFallbackEncoder() { codec_.maxFramerate = 30; codec_.width = kWidth; codec_.height = kHeight; + codec_.codecSpecific.VP8.numberOfTemporalLayers = 1; + std::unique_ptr tl_factory( + new TemporalLayersFactory()); + codec_.codecSpecific.VP8.tl_factory = tl_factory.get(); + rate_allocator_.reset( + new SimulcastRateAllocator(codec_, std::move(tl_factory))); + fake_encoder_.init_encode_return_code_ = WEBRTC_VIDEO_CODEC_ERROR; EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.InitEncode(&codec_, 2, kMaxPayloadSize)); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.SetRates(300, 30)); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_.SetRateAllocation( + rate_allocator_->GetAllocation(300000, 30), 30)); int callback_count = callback_.callback_count_; int encode_count = fake_encoder_.encode_count_; @@ -157,8 +171,16 @@ void VideoEncoderSoftwareFallbackWrapperTest::FallbackFromEncodeRequest() { codec_.maxFramerate = 30; codec_.width = kWidth; codec_.height = kHeight; + codec_.codecSpecific.VP8.numberOfTemporalLayers = 1; + std::unique_ptr tl_factory( + new TemporalLayersFactory()); + codec_.codecSpecific.VP8.tl_factory = tl_factory.get(); + rate_allocator_.reset( + new SimulcastRateAllocator(codec_, std::move(tl_factory))); fallback_wrapper_.InitEncode(&codec_, 2, kMaxPayloadSize); - EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.SetRates(300, 30)); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + fallback_wrapper_.SetRateAllocation( + rate_allocator_->GetAllocation(300000, 30), 30)); EXPECT_EQ(1, fake_encoder_.init_encode_count_); // Have the non-fallback encoder request a software fallback. @@ -241,7 +263,7 @@ TEST_F(VideoEncoderSoftwareFallbackWrapperTest, SetRatesForwardedDuringFallback) { UtilizeFallbackEncoder(); EXPECT_EQ(1, fake_encoder_.set_rates_count_); - fallback_wrapper_.SetRates(1, 1); + fallback_wrapper_.SetRateAllocation(BitrateAllocation(), 1); EXPECT_EQ(2, fake_encoder_.set_rates_count_); EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, fallback_wrapper_.Release()); } diff --git a/webrtc/video/video_send_stream_tests.cc b/webrtc/video/video_send_stream_tests.cc index 2b5c6e6909..267487a1e5 100644 --- a/webrtc/video/video_send_stream_tests.cc +++ b/webrtc/video/video_send_stream_tests.cc @@ -1478,7 +1478,8 @@ TEST_F(VideoSendStreamTest, return; } } - init_encode_called_.Wait(VideoSendStreamTest::kDefaultTimeoutMs); + EXPECT_TRUE( + init_encode_called_.Wait(VideoSendStreamTest::kDefaultTimeoutMs)); { rtc::CritScope lock(&crit_); EXPECT_EQ(width, last_initialized_frame_width_); @@ -1494,9 +1495,7 @@ TEST_F(VideoSendStreamTest, last_initialized_frame_width_ = config->width; last_initialized_frame_height_ = config->height; ++number_of_initializations_; - // First time InitEncode is called, the frame size is unknown. - if (number_of_initializations_ > 1) - init_encode_called_.Set(); + init_encode_called_.Set(); return FakeEncoder::InitEncode(config, number_of_cores, max_payload_size); } @@ -1626,11 +1625,12 @@ TEST_F(VideoSendStreamTest, VideoSendStreamStopSetEncoderRateToZero) { return FakeEncoder::InitEncode(config, number_of_cores, max_payload_size); } - int32_t SetRates(uint32_t new_target_bitrate, uint32_t framerate) override { + int32_t SetRateAllocation(const BitrateAllocation& bitrate, + uint32_t framerate) override { rtc::CritScope lock(&crit_); - bitrate_kbps_ = new_target_bitrate; + bitrate_kbps_ = bitrate.get_sum_kbps(); bitrate_changed_.Set(); - return FakeEncoder::SetRates(new_target_bitrate, framerate); + return FakeEncoder::SetRateAllocation(bitrate, framerate); } int GetBitrateKbps() const { @@ -1664,13 +1664,17 @@ TEST_F(VideoSendStreamTest, VideoSendStreamStopSetEncoderRateToZero) { CreateVideoStreams(); EXPECT_TRUE(encoder.WaitForEncoderInit()); - EXPECT_GT(encoder.GetBitrateKbps(), 0); + EXPECT_TRUE(encoder.WaitBitrateChanged()); + EXPECT_EQ(0, encoder.GetBitrateKbps()); + video_send_stream_->Start(); EXPECT_TRUE(encoder.WaitBitrateChanged()); EXPECT_GT(encoder.GetBitrateKbps(), 0); + video_send_stream_->Stop(); EXPECT_TRUE(encoder.WaitBitrateChanged()); EXPECT_EQ(0, encoder.GetBitrateKbps()); + video_send_stream_->Start(); EXPECT_TRUE(encoder.WaitBitrateChanged()); EXPECT_GT(encoder.GetBitrateKbps(), 0); @@ -2085,10 +2089,12 @@ void VideoCodecConfigObserver::VerifyCodecSpecifics( } // Set expected temporal layers as they should have been set when - // reconfiguring the encoder and not match the set config. + // reconfiguring the encoder and not match the set config. Also copy the + // TemporalLayersFactory pointer that has been injected by ViEEncoder. VideoCodecVP8 encoder_settings = encoder_settings_; encoder_settings.numberOfTemporalLayers = kVideoCodecConfigObserverNumberOfTemporalLayers; + encoder_settings.tl_factory = config.VP8().tl_factory; EXPECT_EQ( 0, memcmp(&config.VP8(), &encoder_settings, sizeof(encoder_settings_))); } @@ -2310,16 +2316,17 @@ TEST_F(VideoSendStreamTest, ReconfigureBitratesSetsEncoderBitratesCorrectly) { maxPayloadSize); } - int32_t SetRates(uint32_t newBitRate, uint32_t frameRate) override { + int32_t SetRateAllocation(const BitrateAllocation& bitrate, + uint32_t frameRate) override { { rtc::CritScope lock(&crit_); - if (target_bitrate_ == newBitRate) { - return FakeEncoder::SetRates(newBitRate, frameRate); + if (target_bitrate_ == bitrate.get_sum_kbps()) { + return FakeEncoder::SetRateAllocation(bitrate, frameRate); } - target_bitrate_ = newBitRate; + target_bitrate_ = bitrate.get_sum_kbps(); } bitrate_changed_event_.Set(); - return FakeEncoder::SetRates(newBitRate, frameRate); + return FakeEncoder::SetRateAllocation(bitrate, frameRate); } void WaitForSetRates(uint32_t expected_bitrate) { diff --git a/webrtc/video/vie_encoder.cc b/webrtc/video/vie_encoder.cc index 7cc799a644..75493e9e7c 100644 --- a/webrtc/video/vie_encoder.cc +++ b/webrtc/video/vie_encoder.cc @@ -14,11 +14,13 @@ #include #include +#include "webrtc/modules/video_coding/include/video_codec_initializer.h" #include "webrtc/base/checks.h" #include "webrtc/base/logging.h" #include "webrtc/base/trace_event.h" #include "webrtc/base/timeutils.h" #include "webrtc/modules/pacing/paced_sender.h" +#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h" #include "webrtc/modules/video_coding/include/video_coding.h" #include "webrtc/modules/video_coding/include/video_coding_defines.h" #include "webrtc/video/overuse_frame_detector.h" @@ -31,148 +33,6 @@ namespace { // Time interval for logging frame counts. const int64_t kFrameLogIntervalMs = 60000; -VideoCodecType PayloadNameToCodecType(const std::string& payload_name) { - if (payload_name == "VP8") - return kVideoCodecVP8; - if (payload_name == "VP9") - return kVideoCodecVP9; - if (payload_name == "H264") - return kVideoCodecH264; - return kVideoCodecGeneric; -} - -VideoCodec VideoEncoderConfigToVideoCodec( - const VideoEncoderConfig& config, - const std::vector& streams, - const std::string& payload_name, - int payload_type) { - static const int kEncoderMinBitrateKbps = 30; - RTC_DCHECK(!streams.empty()); - RTC_DCHECK_GE(config.min_transmit_bitrate_bps, 0); - - VideoCodec video_codec; - memset(&video_codec, 0, sizeof(video_codec)); - video_codec.codecType = PayloadNameToCodecType(payload_name); - - switch (config.content_type) { - case VideoEncoderConfig::ContentType::kRealtimeVideo: - video_codec.mode = kRealtimeVideo; - break; - case VideoEncoderConfig::ContentType::kScreen: - video_codec.mode = kScreensharing; - if (streams.size() == 1 && - streams[0].temporal_layer_thresholds_bps.size() == 1) { - video_codec.targetBitrate = - streams[0].temporal_layer_thresholds_bps[0] / 1000; - } - break; - } - - if (config.encoder_specific_settings) - config.encoder_specific_settings->FillEncoderSpecificSettings(&video_codec); - - switch (video_codec.codecType) { - case kVideoCodecVP8: { - if (!config.encoder_specific_settings) - video_codec.codecSpecific.VP8 = VideoEncoder::GetDefaultVp8Settings(); - video_codec.codecSpecific.VP8.numberOfTemporalLayers = - static_cast( - streams.back().temporal_layer_thresholds_bps.size() + 1); - break; - } - case kVideoCodecVP9: { - if (!config.encoder_specific_settings) - video_codec.codecSpecific.VP9 = VideoEncoder::GetDefaultVp9Settings(); - if (video_codec.mode == kScreensharing && - config.encoder_specific_settings) { - video_codec.codecSpecific.VP9.flexibleMode = true; - // For now VP9 screensharing use 1 temporal and 2 spatial layers. - RTC_DCHECK_EQ(1, video_codec.codecSpecific.VP9.numberOfTemporalLayers); - RTC_DCHECK_EQ(2, video_codec.codecSpecific.VP9.numberOfSpatialLayers); - } - video_codec.codecSpecific.VP9.numberOfTemporalLayers = - static_cast( - streams.back().temporal_layer_thresholds_bps.size() + 1); - break; - } - case kVideoCodecH264: { - if (!config.encoder_specific_settings) - video_codec.codecSpecific.H264 = VideoEncoder::GetDefaultH264Settings(); - break; - } - default: - // TODO(pbos): Support encoder_settings codec-agnostically. - RTC_DCHECK(!config.encoder_specific_settings) - << "Encoder-specific settings for codec type not wired up."; - break; - } - - strncpy(video_codec.plName, payload_name.c_str(), kPayloadNameSize - 1); - video_codec.plName[kPayloadNameSize - 1] = '\0'; - video_codec.plType = payload_type; - video_codec.numberOfSimulcastStreams = - static_cast(streams.size()); - video_codec.minBitrate = streams[0].min_bitrate_bps / 1000; - if (video_codec.minBitrate < kEncoderMinBitrateKbps) - video_codec.minBitrate = kEncoderMinBitrateKbps; - RTC_DCHECK_LE(streams.size(), static_cast(kMaxSimulcastStreams)); - if (video_codec.codecType == kVideoCodecVP9) { - // If the vector is empty, bitrates will be configured automatically. - RTC_DCHECK(config.spatial_layers.empty() || - config.spatial_layers.size() == - video_codec.codecSpecific.VP9.numberOfSpatialLayers); - RTC_DCHECK_LE(video_codec.codecSpecific.VP9.numberOfSpatialLayers, - kMaxSimulcastStreams); - for (size_t i = 0; i < config.spatial_layers.size(); ++i) - video_codec.spatialLayers[i] = config.spatial_layers[i]; - } - for (size_t i = 0; i < streams.size(); ++i) { - SimulcastStream* sim_stream = &video_codec.simulcastStream[i]; - RTC_DCHECK_GT(streams[i].width, 0u); - RTC_DCHECK_GT(streams[i].height, 0u); - RTC_DCHECK_GT(streams[i].max_framerate, 0); - // Different framerates not supported per stream at the moment. - RTC_DCHECK_EQ(streams[i].max_framerate, streams[0].max_framerate); - RTC_DCHECK_GE(streams[i].min_bitrate_bps, 0); - RTC_DCHECK_GE(streams[i].target_bitrate_bps, streams[i].min_bitrate_bps); - RTC_DCHECK_GE(streams[i].max_bitrate_bps, streams[i].target_bitrate_bps); - RTC_DCHECK_GE(streams[i].max_qp, 0); - - sim_stream->width = static_cast(streams[i].width); - sim_stream->height = static_cast(streams[i].height); - sim_stream->minBitrate = streams[i].min_bitrate_bps / 1000; - sim_stream->targetBitrate = streams[i].target_bitrate_bps / 1000; - sim_stream->maxBitrate = streams[i].max_bitrate_bps / 1000; - sim_stream->qpMax = streams[i].max_qp; - sim_stream->numberOfTemporalLayers = static_cast( - streams[i].temporal_layer_thresholds_bps.size() + 1); - - video_codec.width = std::max(video_codec.width, - static_cast(streams[i].width)); - video_codec.height = std::max( - video_codec.height, static_cast(streams[i].height)); - video_codec.minBitrate = - std::min(static_cast(video_codec.minBitrate), - static_cast(streams[i].min_bitrate_bps / 1000)); - video_codec.maxBitrate += streams[i].max_bitrate_bps / 1000; - video_codec.qpMax = std::max(video_codec.qpMax, - static_cast(streams[i].max_qp)); - } - - if (video_codec.maxBitrate == 0) { - // Unset max bitrate -> cap to one bit per pixel. - video_codec.maxBitrate = - (video_codec.width * video_codec.height * video_codec.maxFramerate) / - 1000; - } - if (video_codec.maxBitrate < kEncoderMinBitrateKbps) - video_codec.maxBitrate = kEncoderMinBitrateKbps; - - RTC_DCHECK_GT(streams[0].max_framerate, 0); - video_codec.maxFramerate = streams[0].max_framerate; - return video_codec; -} - // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle // pipelining encoders better (multiple input frames before something comes // out). This should effectively turn off CPU adaptations for systems that @@ -372,7 +232,8 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores, source_proxy_(new VideoSourceProxy(this)), sink_(nullptr), settings_(settings), - codec_type_(PayloadNameToCodecType(settings.payload_name)), + codec_type_(PayloadNameToCodecType(settings.payload_name) + .value_or(VideoCodecType::kVideoCodecUnknown)), video_sender_(Clock::GetRealTimeClock(), this, this), overuse_detector_(Clock::GetRealTimeClock(), GetCpuOveruseOptions(settings.full_overuse_time), @@ -424,6 +285,7 @@ void ViEEncoder::Stop() { encoder_queue_.PostTask([this] { RTC_DCHECK_RUN_ON(&encoder_queue_); overuse_detector_.StopCheckForOveruse(); + rate_allocator_.reset(); video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type, false); shutdown_event_.Set(); @@ -518,8 +380,11 @@ void ViEEncoder::ReconfigureEncoder() { encoder_config_.video_stream_factory->CreateEncoderStreams( last_frame_info_->width, last_frame_info_->height, encoder_config_); - VideoCodec codec = VideoEncoderConfigToVideoCodec( - encoder_config_, streams, settings_.payload_name, settings_.payload_type); + VideoCodec codec; + if (!VideoCodecInitializer::SetupCodec(encoder_config_, settings_, streams, + &codec, &rate_allocator_)) { + LOG(LS_ERROR) << "Failed to create encoder configuration."; + } codec.startBitrate = std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate); @@ -534,17 +399,18 @@ void ViEEncoder::ReconfigureEncoder() { RTC_DCHECK(success); } - rate_allocator_.reset(new SimulcastRateAllocator(codec)); + video_sender_.UpdateChannelParemeters(rate_allocator_.get()); + if (stats_proxy_) { - stats_proxy_->OnEncoderReconfigured(encoder_config_, - rate_allocator_->GetPreferedBitrate()); + int framerate = stats_proxy_->GetSendFrameRate(); + if (framerate == 0) + framerate = codec.maxFramerate; + stats_proxy_->OnEncoderReconfigured( + encoder_config_, rate_allocator_->GetPreferredBitrateBps(framerate)); } pending_encoder_reconfiguration_ = false; - if (stats_proxy_) { - stats_proxy_->OnEncoderReconfigured(encoder_config_, - rate_allocator_->GetPreferedBitrate()); - } + sink_->OnEncoderConfigurationChanged( std::move(streams), encoder_config_.min_transmit_bitrate_bps); } @@ -770,13 +636,12 @@ void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps, << " rtt " << round_trip_time_ms; video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, - round_trip_time_ms); + round_trip_time_ms, rate_allocator_.get()); encoder_start_bitrate_bps_ = bitrate_bps != 0 ? bitrate_bps : encoder_start_bitrate_bps_; bool video_is_suspended = bitrate_bps == 0; - bool video_suspension_changed = - video_is_suspended != (last_observed_bitrate_bps_ == 0); + bool video_suspension_changed = video_is_suspended != EncoderPaused(); last_observed_bitrate_bps_ = bitrate_bps; if (stats_proxy_ && video_suspension_changed) { diff --git a/webrtc/video/vie_encoder.h b/webrtc/video/vie_encoder.h index b89f66e9d9..433e9ceb5e 100644 --- a/webrtc/video/vie_encoder.h +++ b/webrtc/video/vie_encoder.h @@ -21,10 +21,10 @@ #include "webrtc/base/task_queue.h" #include "webrtc/call.h" #include "webrtc/common_types.h" +#include "webrtc/common_video/include/video_bitrate_allocator.h" #include "webrtc/common_video/rotation.h" #include "webrtc/media/base/videosinkinterface.h" #include "webrtc/modules/video_coding/include/video_coding_defines.h" -#include "webrtc/modules/video_coding/utility/simulcast_rate_allocator.h" #include "webrtc/modules/video_coding/video_coding_impl.h" #include "webrtc/modules/video_processing/include/video_processing.h" #include "webrtc/system_wrappers/include/atomic32.h" @@ -184,10 +184,7 @@ class ViEEncoder : public rtc::VideoSinkInterface, rtc::ThreadChecker thread_checker_; VideoEncoderConfig encoder_config_ ACCESS_ON(&encoder_queue_); - // TODO(sprang): Change |rate_allocator_| to be a codec type - // agnostic interface. It is currently VP8 simulcast specific if more than - // one layer is specified. - std::unique_ptr rate_allocator_ + std::unique_ptr rate_allocator_ ACCESS_ON(&encoder_queue_); // Set when ConfigureEncoder has been called in order to lazy reconfigure the diff --git a/webrtc/video/vie_encoder_unittest.cc b/webrtc/video/vie_encoder_unittest.cc index 9f4fdc47e3..f6046be652 100644 --- a/webrtc/video/vie_encoder_unittest.cc +++ b/webrtc/video/vie_encoder_unittest.cc @@ -93,6 +93,7 @@ class ViEEncoderTest : public ::testing::Test { VideoEncoderConfig video_encoder_config; test::FillEncoderConfiguration(1, &video_encoder_config); + video_encoder_config_ = video_encoder_config.Copy(); vie_encoder_.reset(new ViEEncoderUnderTest( stats_proxy_.get(), video_send_config_.encoder_settings)); vie_encoder_->SetSink(&sink_, false /* rotation_applied */); @@ -236,6 +237,7 @@ class ViEEncoderTest : public ::testing::Test { }; VideoSendStream::Config video_send_config_; + VideoEncoderConfig video_encoder_config_; int codec_width_; int codec_height_; TestEncoder fake_encoder_; @@ -595,6 +597,20 @@ TEST_F(ViEEncoderTest, StatsTracksAdaptationStatsWhenSwitchingSource) { vie_encoder_->Stop(); } +TEST_F(ViEEncoderTest, StatsTracksPreferredBitrate) { + const int kTargetBitrateBps = 100000; + vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + + video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + sink_.WaitForEncodedFrame(1); + + VideoSendStream::Stats stats = stats_proxy_->GetStats(); + EXPECT_EQ(video_encoder_config_.max_bitrate_bps, + stats.preferred_media_bitrate_bps); + + vie_encoder_->Stop(); +} + TEST_F(ViEEncoderTest, UMACpuLimitedResolutionInPercent) { const int kTargetBitrateBps = 100000; vie_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); diff --git a/webrtc/video_encoder.h b/webrtc/video_encoder.h index 8bfa72fe1e..2dc8334d62 100644 --- a/webrtc/video_encoder.h +++ b/webrtc/video_encoder.h @@ -15,6 +15,7 @@ #include #include +#include "webrtc/base/checks.h" #include "webrtc/common_types.h" #include "webrtc/typedefs.h" #include "webrtc/video_frame.h" @@ -148,7 +149,17 @@ class VideoEncoder { // - framerate : The target frame rate // // Return value : WEBRTC_VIDEO_CODEC_OK if OK, < 0 otherwise. - virtual int32_t SetRates(uint32_t bitrate, uint32_t framerate) = 0; + virtual int32_t SetRates(uint32_t bitrate, uint32_t framerate) { + RTC_NOTREACHED() << "SetRate(uint32_t, uint32_t) is deprecated."; + return -1; + } + + // Default fallback: Just use the sum of bitrates as the single target rate. + // TODO(sprang): Remove this default implementation when we remove SetRates(). + virtual int32_t SetRateAllocation(const BitrateAllocation& allocation, + uint32_t framerate) { + return SetRates(allocation.get_sum_kbps(), framerate); + } virtual int32_t SetPeriodicKeyFrames(bool enable) { return -1; } virtual void OnDroppedFrame() {} @@ -176,8 +187,8 @@ class VideoEncoderSoftwareFallbackWrapper : public VideoEncoder { const CodecSpecificInfo* codec_specific_info, const std::vector* frame_types) override; int32_t SetChannelParameters(uint32_t packet_loss, int64_t rtt) override; - - int32_t SetRates(uint32_t bitrate, uint32_t framerate) override; + int32_t SetRateAllocation(const BitrateAllocation& bitrate_allocation, + uint32_t framerate) override; void OnDroppedFrame() override; bool SupportsNativeHandle() const override; @@ -192,7 +203,7 @@ class VideoEncoderSoftwareFallbackWrapper : public VideoEncoder { // The last bitrate/framerate set, and a flag for noting they are set. bool rates_set_; - uint32_t bitrate_; + BitrateAllocation bitrate_allocation_; uint32_t framerate_; // The last channel parameters set, and a flag for noting they are set.