diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index f4490cb9a4..a321f32b4e 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -153,7 +153,6 @@ rtc_static_library("video_coding") { "video_coding_impl.cc", "video_coding_impl.h", "video_receiver.cc", - "video_sender.cc", ] if (!build_with_chromium && is_clang) { @@ -873,7 +872,6 @@ if (rtc_include_tests) { "video_codec_initializer_unittest.cc", "video_packet_buffer_unittest.cc", "video_receiver_unittest.cc", - "video_sender_unittest.cc", ] if (rtc_use_h264) { sources += [ diff --git a/modules/video_coding/video_sender.cc b/modules/video_coding/video_sender.cc deleted file mode 100644 index 4493dd7c67..0000000000 --- a/modules/video_coding/video_sender.cc +++ /dev/null @@ -1,265 +0,0 @@ -/* - * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include -#include -#include - -#include "api/scoped_refptr.h" -#include "api/video/video_bitrate_allocation.h" -#include "api/video/video_bitrate_allocator.h" -#include "api/video/video_frame.h" -#include "api/video/video_frame_buffer.h" -#include "api/video_codecs/video_codec.h" -#include "api/video_codecs/video_encoder.h" -#include "common_types.h" // NOLINT(build/include) -#include "modules/video_coding/encoder_database.h" -#include "modules/video_coding/generic_encoder.h" -#include "modules/video_coding/include/video_codec_interface.h" -#include "modules/video_coding/include/video_coding_defines.h" -#include "modules/video_coding/include/video_error_codes.h" -#include "modules/video_coding/internal_defines.h" -#include "modules/video_coding/utility/default_video_bitrate_allocator.h" -#include "modules/video_coding/video_coding_impl.h" -#include "rtc_base/checks.h" -#include "rtc_base/critical_section.h" -#include "rtc_base/logging.h" -#include "rtc_base/sequenced_task_checker.h" -#include "system_wrappers/include/clock.h" -#include "system_wrappers/include/field_trial.h" - -namespace webrtc { -namespace vcm { - -VideoSender::VideoSender(Clock* clock, - EncodedImageCallback* post_encode_callback) - : _encoder(nullptr), - _encodedFrameCallback(post_encode_callback), - _codecDataBase(&_encodedFrameCallback), - current_codec_(), - encoder_has_internal_source_(false), - next_frame_types_(1, kVideoFrameDelta) { - // Allow VideoSender to be created on one thread but used on another, post - // construction. This is currently how this class is being used by at least - // one external project (diffractor). - sequenced_checker_.Detach(); -} - -VideoSender::~VideoSender() {} - -// Register the send codec to be used. -int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec, - uint32_t numberOfCores, - uint32_t maxPayloadSize) { - RTC_DCHECK(sequenced_checker_.CalledSequentially()); - rtc::CritScope lock(&encoder_crit_); - if (sendCodec == nullptr) { - return VCM_PARAMETER_ERROR; - } - - bool ret = - _codecDataBase.SetSendCodec(sendCodec, numberOfCores, maxPayloadSize); - - // Update encoder regardless of result to make sure that we're not holding on - // to a deleted instance. - _encoder = _codecDataBase.GetEncoder(); - // Cache the current codec here so they can be fetched from this thread - // without requiring the _sendCritSect lock. - current_codec_ = *sendCodec; - - if (!ret) { - RTC_LOG(LS_ERROR) << "Failed to initialize set encoder with codec type '" - << sendCodec->codecType << "'."; - return VCM_CODEC_ERROR; - } - - // SetSendCodec succeeded, _encoder should be set. - RTC_DCHECK(_encoder); - - { - rtc::CritScope cs(¶ms_crit_); - next_frame_types_.clear(); - next_frame_types_.resize(VCM_MAX(sendCodec->numberOfSimulcastStreams, 1), - kVideoFrameKey); - // Cache InternalSource() to have this available from IntraFrameRequest() - // without having to acquire encoder_crit_ (avoid blocking on encoder use). - encoder_has_internal_source_ = _encoder->InternalSource(); - } - - RTC_LOG(LS_VERBOSE) << " max bitrate " << sendCodec->maxBitrate - << " start bitrate " << sendCodec->startBitrate - << " max frame rate " << sendCodec->maxFramerate - << " max payload size " << maxPayloadSize; - return VCM_OK; -} - -// Register an external decoder object. -// This can not be used together with external decoder callbacks. -void VideoSender::RegisterExternalEncoder(VideoEncoder* externalEncoder, - bool internalSource /*= false*/) { - RTC_DCHECK(sequenced_checker_.CalledSequentially()); - - rtc::CritScope lock(&encoder_crit_); - - if (externalEncoder == nullptr) { - _codecDataBase.DeregisterExternalEncoder(); - { - // Make sure the VCM doesn't use the de-registered codec - rtc::CritScope params_lock(¶ms_crit_); - _encoder = nullptr; - encoder_has_internal_source_ = false; - } - return; - } - _codecDataBase.RegisterExternalEncoder(externalEncoder, - internalSource); -} - -int32_t VideoSender::SetChannelParameters( - const VideoBitrateAllocation& bitrate_allocation, - uint32_t framerate_fps) { - bool encoder_has_internal_source; - { - rtc::CritScope cs(¶ms_crit_); - encoder_has_internal_source = encoder_has_internal_source_; - } - - { - rtc::CritScope cs(&encoder_crit_); - if (_encoder) { - // |target_bitrate == 0 | means that the network is down or the send pacer - // is full. We currently only report this if the encoder has an internal - // source. If the encoder does not have an internal source, higher levels - // are expected to not call AddVideoFrame. We do this since its unclear - // how current encoder implementations behave when given a zero target - // bitrate. - // TODO(perkj): Make sure all known encoder implementations handle zero - // target bitrate and remove this check. - if (!encoder_has_internal_source && - bitrate_allocation.get_sum_bps() == 0) { - return VCM_OK; - } - - if (framerate_fps == 0) { - // No frame rate estimate available, use default. - framerate_fps = current_codec_.maxFramerate; - } - if (_encoder != nullptr) - _encoder->SetEncoderParameters(bitrate_allocation, framerate_fps); - } - } - - return VCM_OK; -} - -// Add one raw video frame to the encoder, blocking. -int32_t VideoSender::AddVideoFrame( - const VideoFrame& videoFrame, - const CodecSpecificInfo* codecSpecificInfo, - absl::optional encoder_info) { - std::vector next_frame_types; - bool encoder_has_internal_source = false; - { - rtc::CritScope lock(¶ms_crit_); - next_frame_types = next_frame_types_; - encoder_has_internal_source = encoder_has_internal_source_; - } - rtc::CritScope lock(&encoder_crit_); - if (_encoder == nullptr) - return VCM_UNINITIALIZED; - // TODO(pbos): Make sure setting send codec is synchronized with video - // processing so frame size always matches. - if (!_codecDataBase.MatchesCurrentResolution(videoFrame.width(), - videoFrame.height())) { - RTC_LOG(LS_ERROR) - << "Incoming frame doesn't match set resolution. Dropping."; - return VCM_PARAMETER_ERROR; - } - VideoFrame converted_frame = videoFrame; - const VideoFrameBuffer::Type buffer_type = - converted_frame.video_frame_buffer()->type(); - const bool is_buffer_type_supported = - buffer_type == VideoFrameBuffer::Type::kI420 || - (buffer_type == VideoFrameBuffer::Type::kNative && - encoder_info->supports_native_handle); - if (!is_buffer_type_supported) { - // This module only supports software encoding. - // TODO(pbos): Offload conversion from the encoder thread. - rtc::scoped_refptr converted_buffer( - converted_frame.video_frame_buffer()->ToI420()); - - if (!converted_buffer) { - RTC_LOG(LS_ERROR) << "Frame conversion failed, dropping frame."; - return VCM_PARAMETER_ERROR; - } - - // UpdatedRect is not propagated because buffer was converted, - // therefore we can't guarantee that pixels outside of UpdateRect didn't - // change comparing to the previous frame. - converted_frame = VideoFrame::Builder() - .set_video_frame_buffer(converted_buffer) - .set_timestamp_rtp(converted_frame.timestamp()) - .set_timestamp_ms(converted_frame.render_time_ms()) - .set_rotation(converted_frame.rotation()) - .set_id(converted_frame.id()) - .build(); - } - int32_t ret = - _encoder->Encode(converted_frame, codecSpecificInfo, next_frame_types); - if (ret < 0) { - RTC_LOG(LS_ERROR) << "Failed to encode frame. Error code: " << ret; - return ret; - } - - { - rtc::CritScope lock(¶ms_crit_); - // Change all keyframe requests to encode delta frames the next time. - for (size_t i = 0; i < next_frame_types_.size(); ++i) { - // Check for equality (same requested as before encoding) to not - // accidentally drop a keyframe request while encoding. - if (next_frame_types[i] == next_frame_types_[i]) - next_frame_types_[i] = kVideoFrameDelta; - } - } - return VCM_OK; -} - -int32_t VideoSender::IntraFrameRequest(size_t stream_index) { - { - rtc::CritScope lock(¶ms_crit_); - if (stream_index >= next_frame_types_.size()) { - return -1; - } - next_frame_types_[stream_index] = kVideoFrameKey; - if (!encoder_has_internal_source_) - return VCM_OK; - } - // TODO(pbos): Remove when InternalSource() is gone. Both locks have to be - // held here for internal consistency, since _encoder could be removed while - // not holding encoder_crit_. Checks have to be performed again since - // params_crit_ was dropped to not cause lock-order inversions with - // encoder_crit_. - rtc::CritScope lock(&encoder_crit_); - rtc::CritScope params_lock(¶ms_crit_); - if (stream_index >= next_frame_types_.size()) - return -1; - if (_encoder != nullptr && _encoder->InternalSource()) { - // Try to request the frame if we have an external encoder with - // internal source since AddVideoFrame never will be called. - if (_encoder->RequestFrame(next_frame_types_) == WEBRTC_VIDEO_CODEC_OK) { - // Try to remove just-performed keyframe request, if stream still exists. - next_frame_types_[stream_index] = kVideoFrameDelta; - } - } - return VCM_OK; -} - -} // namespace vcm -} // namespace webrtc diff --git a/modules/video_coding/video_sender_unittest.cc b/modules/video_coding/video_sender_unittest.cc deleted file mode 100644 index 11a69be137..0000000000 --- a/modules/video_coding/video_sender_unittest.cc +++ /dev/null @@ -1,498 +0,0 @@ -/* - * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#include -#include - -#include "absl/memory/memory.h" -#include "api/test/mock_video_encoder.h" -#include "api/video/i420_buffer.h" -#include "api/video_codecs/vp8_temporal_layers.h" -#include "modules/video_coding/codecs/vp8/include/vp8.h" -#include "modules/video_coding/include/mock/mock_vcm_callbacks.h" -#include "modules/video_coding/include/video_coding.h" -#include "modules/video_coding/utility/default_video_bitrate_allocator.h" -#include "modules/video_coding/utility/simulcast_rate_allocator.h" -#include "modules/video_coding/video_coding_impl.h" -#include "system_wrappers/include/clock.h" -#include "test/frame_generator.h" -#include "test/gtest.h" -#include "test/testsupport/file_utils.h" -#include "test/video_codec_settings.h" - -using ::testing::_; -using ::testing::AllOf; -using ::testing::ElementsAre; -using ::testing::ElementsAreArray; -using ::testing::Field; -using ::testing::NiceMock; -using ::testing::Pointee; -using ::testing::Return; -using ::testing::FloatEq; -using std::vector; -using webrtc::test::FrameGenerator; - -namespace webrtc { -namespace vcm { -namespace { -static const int kDefaultHeight = 720; -static const int kDefaultWidth = 1280; -static const int kMaxNumberOfTemporalLayers = 3; -static const int kNumberOfLayers = 3; -static const int kNumberOfStreams = 3; -static const int kUnusedPayloadType = 10; - -struct Vp8StreamInfo { - float framerate_fps[kMaxNumberOfTemporalLayers]; - int bitrate_kbps[kMaxNumberOfTemporalLayers]; -}; - -MATCHER_P(MatchesVp8StreamInfo, expected, "") { - bool res = true; - for (int tl = 0; tl < kMaxNumberOfTemporalLayers; ++tl) { - if (fabs(expected.framerate_fps[tl] - arg.framerate_fps[tl]) > 0.5) { - *result_listener << " framerate_fps[" << tl - << "] = " << arg.framerate_fps[tl] << " (expected " - << expected.framerate_fps[tl] << ") "; - res = false; - } - if (abs(expected.bitrate_kbps[tl] - arg.bitrate_kbps[tl]) > 10) { - *result_listener << " bitrate_kbps[" << tl - << "] = " << arg.bitrate_kbps[tl] << " (expected " - << expected.bitrate_kbps[tl] << ") "; - res = false; - } - } - return res; -} - -class EmptyFrameGenerator : public FrameGenerator { - public: - EmptyFrameGenerator(int width, int height) : width_(width), height_(height) {} - VideoFrame* NextFrame() override { - frame_ = absl::make_unique( - VideoFrame::Builder() - .set_video_frame_buffer(I420Buffer::Create(width_, height_)) - .set_rotation(webrtc::kVideoRotation_0) - .set_timestamp_us(0) - .build()); - return frame_.get(); - } - - private: - const int width_; - const int height_; - std::unique_ptr frame_; -}; - -class EncodedImageCallbackImpl : public EncodedImageCallback { - public: - explicit EncodedImageCallbackImpl(Clock* clock) - : clock_(clock), start_time_ms_(clock_->TimeInMilliseconds()) {} - - virtual ~EncodedImageCallbackImpl() {} - - Result OnEncodedImage(const EncodedImage& encoded_image, - const CodecSpecificInfo* codec_specific_info, - const RTPFragmentationHeader* fragmentation) override { - assert(codec_specific_info); - frame_data_.push_back( - FrameData(encoded_image.size(), *codec_specific_info)); - return Result(Result::OK, encoded_image.Timestamp()); - } - - void Reset() { - frame_data_.clear(); - start_time_ms_ = clock_->TimeInMilliseconds(); - } - - float FramerateFpsWithinTemporalLayer(int temporal_layer) { - return CountFramesWithinTemporalLayer(temporal_layer) * - (1000.0 / interval_ms()); - } - - float BitrateKbpsWithinTemporalLayer(int temporal_layer) { - return SumPayloadBytesWithinTemporalLayer(temporal_layer) * 8.0 / - interval_ms(); - } - - Vp8StreamInfo CalculateVp8StreamInfo() { - Vp8StreamInfo info; - for (int tl = 0; tl < 3; ++tl) { - info.framerate_fps[tl] = FramerateFpsWithinTemporalLayer(tl); - info.bitrate_kbps[tl] = BitrateKbpsWithinTemporalLayer(tl); - } - return info; - } - - private: - struct FrameData { - FrameData() : payload_size(0) {} - - FrameData(size_t payload_size, const CodecSpecificInfo& codec_specific_info) - : payload_size(payload_size), - codec_specific_info(codec_specific_info) {} - - size_t payload_size; - CodecSpecificInfo codec_specific_info; - }; - - int64_t interval_ms() { - int64_t diff = (clock_->TimeInMilliseconds() - start_time_ms_); - EXPECT_GT(diff, 0); - return diff; - } - - int CountFramesWithinTemporalLayer(int temporal_layer) { - int frames = 0; - for (size_t i = 0; i < frame_data_.size(); ++i) { - EXPECT_EQ(kVideoCodecVP8, frame_data_[i].codec_specific_info.codecType); - const uint8_t temporal_idx = - frame_data_[i].codec_specific_info.codecSpecific.VP8.temporalIdx; - if (temporal_idx <= temporal_layer || temporal_idx == kNoTemporalIdx) - frames++; - } - return frames; - } - - size_t SumPayloadBytesWithinTemporalLayer(int temporal_layer) { - size_t payload_size = 0; - for (size_t i = 0; i < frame_data_.size(); ++i) { - EXPECT_EQ(kVideoCodecVP8, frame_data_[i].codec_specific_info.codecType); - const uint8_t temporal_idx = - frame_data_[i].codec_specific_info.codecSpecific.VP8.temporalIdx; - if (temporal_idx <= temporal_layer || temporal_idx == kNoTemporalIdx) - payload_size += frame_data_[i].payload_size; - } - return payload_size; - } - - Clock* clock_; - int64_t start_time_ms_; - vector frame_data_; -}; - -class TestVideoSender : public ::testing::Test { - protected: - // Note: simulated clock starts at 1 seconds, since parts of webrtc use 0 as - // a special case (e.g. frame rate in media optimization). - TestVideoSender() : clock_(1000), encoded_frame_callback_(&clock_) {} - - void SetUp() override { - sender_.reset(new VideoSender(&clock_, &encoded_frame_callback_)); - } - - void AddFrame() { - assert(generator_.get()); - sender_->AddVideoFrame(*generator_->NextFrame(), nullptr, - encoder_ ? absl::optional( - encoder_->GetEncoderInfo()) - : absl::nullopt); - } - - SimulatedClock clock_; - EncodedImageCallbackImpl encoded_frame_callback_; - // Used by subclassing tests, need to outlive sender_. - std::unique_ptr encoder_; - std::unique_ptr sender_; - std::unique_ptr generator_; -}; - -class TestVideoSenderWithMockEncoder : public TestVideoSender { - public: - TestVideoSenderWithMockEncoder() {} - ~TestVideoSenderWithMockEncoder() override {} - - protected: - void SetUp() override { - TestVideoSender::SetUp(); - sender_->RegisterExternalEncoder(&encoder_, false); - webrtc::test::CodecSettings(kVideoCodecVP8, &settings_); - settings_.numberOfSimulcastStreams = kNumberOfStreams; - ConfigureStream(kDefaultWidth / 4, kDefaultHeight / 4, 100, - &settings_.simulcastStream[0]); - ConfigureStream(kDefaultWidth / 2, kDefaultHeight / 2, 500, - &settings_.simulcastStream[1]); - ConfigureStream(kDefaultWidth, kDefaultHeight, 1200, - &settings_.simulcastStream[2]); - settings_.plType = kUnusedPayloadType; // Use the mocked encoder. - generator_.reset( - new EmptyFrameGenerator(settings_.width, settings_.height)); - EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200)); - rate_allocator_.reset(new DefaultVideoBitrateAllocator(settings_)); - } - - void TearDown() override { sender_.reset(); } - - void ExpectIntraRequest(int stream) { - ExpectEncodeWithFrameTypes(stream, false); - } - - void ExpectInitialKeyFrames() { ExpectEncodeWithFrameTypes(-1, true); } - - void ExpectEncodeWithFrameTypes(int intra_request_stream, bool first_frame) { - if (intra_request_stream == -1) { - // No intra request expected, keyframes on first frame. - FrameType frame_type = first_frame ? kVideoFrameKey : kVideoFrameDelta; - EXPECT_CALL( - encoder_, - Encode(_, _, - Pointee(ElementsAre(frame_type, frame_type, frame_type)))) - .Times(1) - .WillRepeatedly(Return(0)); - return; - } - ASSERT_FALSE(first_frame); - ASSERT_GE(intra_request_stream, 0); - ASSERT_LT(intra_request_stream, kNumberOfStreams); - std::vector frame_types(kNumberOfStreams, kVideoFrameDelta); - frame_types[intra_request_stream] = kVideoFrameKey; - EXPECT_CALL( - encoder_, - Encode(_, _, - Pointee(ElementsAreArray(&frame_types[0], frame_types.size())))) - .Times(1) - .WillRepeatedly(Return(0)); - } - - static void ConfigureStream(int width, - int height, - int max_bitrate, - SimulcastStream* stream) { - assert(stream); - stream->width = width; - stream->height = height; - stream->maxBitrate = max_bitrate; - stream->numberOfTemporalLayers = kNumberOfLayers; - stream->qpMax = 45; - } - - VideoCodec settings_; - NiceMock encoder_; - std::unique_ptr rate_allocator_; -}; - -TEST_F(TestVideoSenderWithMockEncoder, TestIntraRequests) { - // Initial request should be all keyframes. - ExpectInitialKeyFrames(); - AddFrame(); - EXPECT_EQ(0, sender_->IntraFrameRequest(0)); - ExpectIntraRequest(0); - AddFrame(); - ExpectIntraRequest(-1); - AddFrame(); - - EXPECT_EQ(0, sender_->IntraFrameRequest(1)); - ExpectIntraRequest(1); - AddFrame(); - ExpectIntraRequest(-1); - AddFrame(); - - EXPECT_EQ(0, sender_->IntraFrameRequest(2)); - ExpectIntraRequest(2); - AddFrame(); - ExpectIntraRequest(-1); - AddFrame(); - - EXPECT_EQ(-1, sender_->IntraFrameRequest(3)); - ExpectIntraRequest(-1); - AddFrame(); -} - -TEST_F(TestVideoSenderWithMockEncoder, TestSetRate) { - // Let actual fps be half of max, so it can be distinguished from default. - const uint32_t kActualFrameRate = settings_.maxFramerate / 2; - const int64_t kFrameIntervalMs = 1000 / kActualFrameRate; - const uint32_t new_bitrate_kbps = settings_.startBitrate + 300; - - // Initial frame rate is taken from config, as we have no data yet. - VideoBitrateAllocation new_rate_allocation = rate_allocator_->GetAllocation( - new_bitrate_kbps * 1000, settings_.maxFramerate); - EXPECT_CALL(encoder_, - SetRateAllocation(new_rate_allocation, settings_.maxFramerate)) - .Times(1) - .WillOnce(Return(0)); - sender_->SetChannelParameters(new_rate_allocation, settings_.maxFramerate); - AddFrame(); - clock_.AdvanceTimeMilliseconds(kFrameIntervalMs); - - // Expect no call to encoder_.SetRates if the new bitrate is zero. - EXPECT_CALL(encoder_, SetRateAllocation(_, _)).Times(0); - sender_->SetChannelParameters(VideoBitrateAllocation(), - settings_.maxFramerate); - AddFrame(); -} - -TEST_F(TestVideoSenderWithMockEncoder, TestIntraRequestsInternalCapture) { - // De-register current external encoder. - sender_->RegisterExternalEncoder(nullptr, false); - // Register encoder with internal capture. - sender_->RegisterExternalEncoder(&encoder_, true); - EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200)); - // Initial request should be all keyframes. - ExpectInitialKeyFrames(); - AddFrame(); - ExpectIntraRequest(0); - EXPECT_EQ(0, sender_->IntraFrameRequest(0)); - ExpectIntraRequest(1); - EXPECT_EQ(0, sender_->IntraFrameRequest(1)); - ExpectIntraRequest(2); - EXPECT_EQ(0, sender_->IntraFrameRequest(2)); - // No requests expected since these indices are out of bounds. - EXPECT_EQ(-1, sender_->IntraFrameRequest(3)); -} - -TEST_F(TestVideoSenderWithMockEncoder, TestEncoderParametersForInternalSource) { - // De-register current external encoder. - sender_->RegisterExternalEncoder(nullptr, false); - // Register encoder with internal capture. - sender_->RegisterExternalEncoder(&encoder_, true); - EXPECT_EQ(0, sender_->RegisterSendCodec(&settings_, 1, 1200)); - // Update encoder bitrate parameters. We expect that to immediately call - // SetRates on the encoder without waiting for AddFrame processing. - const uint32_t new_bitrate_kbps = settings_.startBitrate + 300; - VideoBitrateAllocation new_rate_allocation = rate_allocator_->GetAllocation( - new_bitrate_kbps * 1000, settings_.maxFramerate); - EXPECT_CALL(encoder_, SetRateAllocation(new_rate_allocation, _)) - .Times(1) - .WillOnce(Return(0)); - sender_->SetChannelParameters(new_rate_allocation, settings_.maxFramerate); -} - -TEST_F(TestVideoSenderWithMockEncoder, - NoRedundantSetChannelParameterOrSetRatesCalls) { - const int64_t kRateStatsWindowMs = 2000; - const uint32_t kInputFps = 20; - int64_t start_time = clock_.TimeInMilliseconds(); - // Expect initial call to SetChannelParameters. Rates are initialized through - // InitEncode and expects no additional call before the framerate (or bitrate) - // updates. - sender_->SetChannelParameters( - rate_allocator_->GetAllocation(settings_.startBitrate * 1000, kInputFps), - kInputFps); - while (clock_.TimeInMilliseconds() < start_time + kRateStatsWindowMs) { - AddFrame(); - clock_.AdvanceTimeMilliseconds(1000 / kInputFps); - } - - // Call to SetChannelParameters with changed bitrate should call encoder - // SetRates but not encoder SetChannelParameters (that are unchanged). - uint32_t new_bitrate_bps = 2 * settings_.startBitrate * 1000; - VideoBitrateAllocation new_rate_allocation = - rate_allocator_->GetAllocation(new_bitrate_bps, kInputFps); - EXPECT_CALL(encoder_, SetRateAllocation(new_rate_allocation, kInputFps)) - .Times(1) - .WillOnce(Return(0)); - sender_->SetChannelParameters(new_rate_allocation, kInputFps); - AddFrame(); -} - -class TestVideoSenderWithVp8 : public TestVideoSender { - public: - TestVideoSenderWithVp8() - : codec_bitrate_kbps_(300), available_bitrate_kbps_(1000) {} - - void SetUp() override { - TestVideoSender::SetUp(); - - const char* input_video = "foreman_cif"; - const int width = 352; - const int height = 288; - generator_ = FrameGenerator::CreateFromYuvFile( - std::vector(1, test::ResourcePath(input_video, "yuv")), - width, height, 1); - - codec_ = MakeVp8VideoCodec(width, height, 3); - codec_.minBitrate = 10; - codec_.startBitrate = codec_bitrate_kbps_; - codec_.maxBitrate = codec_bitrate_kbps_; - - rate_allocator_.reset(new SimulcastRateAllocator(codec_)); - - encoder_ = VP8Encoder::Create(); - sender_->RegisterExternalEncoder(encoder_.get(), false); - EXPECT_EQ(0, sender_->RegisterSendCodec(&codec_, 1, 1200)); - } - - static VideoCodec MakeVp8VideoCodec(int width, - int height, - int temporal_layers) { - VideoCodec codec; - webrtc::test::CodecSettings(kVideoCodecVP8, &codec); - codec.width = width; - codec.height = height; - codec.VP8()->numberOfTemporalLayers = temporal_layers; - return codec; - } - - void InsertFrames(float framerate, float seconds) { - for (int i = 0; i < seconds * framerate; ++i) { - clock_.AdvanceTimeMilliseconds(1000.0f / framerate); - AddFrame(); - // SetChannelParameters needs to be called frequently to propagate - // framerate from the media optimization into the encoder. - const VideoBitrateAllocation bitrate_allocation = - rate_allocator_->GetAllocation(available_bitrate_kbps_ * 1000, - static_cast(framerate)); - if (i != 0) { - EXPECT_EQ(VCM_OK, - sender_->SetChannelParameters( - bitrate_allocation, static_cast(framerate))); - } - } - } - - Vp8StreamInfo SimulateWithFramerate(float framerate) { - const float short_simulation_interval = 5.0; - const float long_simulation_interval = 10.0; - // It appears that this 5 seconds simulation is needed to allow - // bitrate and framerate to stabilize. - InsertFrames(framerate, short_simulation_interval); - encoded_frame_callback_.Reset(); - - InsertFrames(framerate, long_simulation_interval); - return encoded_frame_callback_.CalculateVp8StreamInfo(); - } - - protected: - VideoCodec codec_; - int codec_bitrate_kbps_; - int available_bitrate_kbps_; - std::unique_ptr rate_allocator_; -}; - -#if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) -#define MAYBE_FixedTemporalLayersStrategy DISABLED_FixedTemporalLayersStrategy -#else -#define MAYBE_FixedTemporalLayersStrategy FixedTemporalLayersStrategy -#endif -TEST_F(TestVideoSenderWithVp8, MAYBE_FixedTemporalLayersStrategy) { - const int low_b = - codec_bitrate_kbps_ * - webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(3, 0); - const int mid_b = - codec_bitrate_kbps_ * - webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(3, 1); - const int high_b = - codec_bitrate_kbps_ * - webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(3, 2); - { - Vp8StreamInfo expected = {{7.5, 15.0, 30.0}, {low_b, mid_b, high_b}}; - EXPECT_THAT(SimulateWithFramerate(30.0), MatchesVp8StreamInfo(expected)); - } - { - Vp8StreamInfo expected = {{3.75, 7.5, 15.0}, {low_b, mid_b, high_b}}; - EXPECT_THAT(SimulateWithFramerate(15.0), MatchesVp8StreamInfo(expected)); - } -} - -} // namespace -} // namespace vcm -} // namespace webrtc diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 6066ac5d75..8dcb8d92cd 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -366,7 +366,6 @@ VideoStreamEncoder::VideoStreamEncoder( sink_(nullptr), settings_(settings), rate_control_settings_(RateControlSettings::ParseFromFieldTrials()), - video_sender_(Clock::GetRealTimeClock(), this), overuse_detector_(std::move(overuse_detector)), encoder_stats_observer_(encoder_stats_observer), max_framerate_(-1), @@ -393,9 +392,14 @@ VideoStreamEncoder::VideoStreamEncoder( force_disable_frame_dropper_(false), input_framerate_(kFrameRateAvergingWindowSizeMs, 1000), pending_frame_drops_(0), + generic_encoder_(nullptr), + generic_encoder_callback_(this), + codec_database_(&generic_encoder_callback_), + next_frame_types_(1, kVideoFrameDelta), encoder_queue_("EncoderQueue") { RTC_DCHECK(encoder_stats_observer); RTC_DCHECK(overuse_detector_); + RTC_DCHECK_GE(number_of_cores, 1); } VideoStreamEncoder::~VideoStreamEncoder() { @@ -412,7 +416,8 @@ void VideoStreamEncoder::Stop() { overuse_detector_->StopCheckForOveruse(); rate_allocator_.reset(); bitrate_observer_ = nullptr; - video_sender_.RegisterExternalEncoder(nullptr, false); + codec_database_.DeregisterExternalEncoder(); + generic_encoder_ = nullptr; quality_scaler_ = nullptr; shutdown_event_.Set(); }); @@ -502,10 +507,11 @@ void VideoStreamEncoder::ConfigureEncoderOnTaskQueue( RTC_DCHECK(sink_); RTC_LOG(LS_INFO) << "ConfigureEncoder requested."; - max_data_payload_length_ = max_data_payload_length; pending_encoder_creation_ = - (!encoder_ || encoder_config_.video_format != config.video_format); + (!encoder_ || encoder_config_.video_format != config.video_format || + max_data_payload_length_ != max_data_payload_length); encoder_config_ = std::move(config); + max_data_payload_length_ = max_data_payload_length; pending_encoder_reconfiguration_ = true; // Reconfigure the encoder now if the encoder has an internal source or @@ -514,11 +520,13 @@ void VideoStreamEncoder::ConfigureEncoderOnTaskQueue( // The codec configuration depends on incoming video frame size. if (last_frame_info_) { ReconfigureEncoder(); - } else if (settings_.encoder_factory - ->QueryVideoEncoder(encoder_config_.video_format) - .has_internal_source) { - last_frame_info_ = VideoFrameInfo(176, 144, false); - ReconfigureEncoder(); + } else { + codec_info_ = settings_.encoder_factory->QueryVideoEncoder( + encoder_config_.video_format); + if (HasInternalSource()) { + last_frame_info_ = VideoFrameInfo(176, 144, false); + ReconfigureEncoder(); + } } } @@ -574,6 +582,8 @@ void VideoStreamEncoder::ReconfigureEncoder() { std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate); codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate); codec.expect_encode_from_texture = last_frame_info_->is_texture; + // Make sure the start bit rate is sane... + RTC_DCHECK_LE(codec.startBitrate, 1000000); max_framerate_ = codec.maxFramerate; // Inform source about max configured framerate. @@ -589,7 +599,8 @@ void VideoStreamEncoder::ReconfigureEncoder() { // encoder_->InitEncode(). if (pending_encoder_creation_) { if (encoder_) { - video_sender_.RegisterExternalEncoder(nullptr, false); + codec_database_.DeregisterExternalEncoder(); + generic_encoder_ = nullptr; } encoder_ = settings_.encoder_factory->CreateVideoEncoder( @@ -598,19 +609,29 @@ void VideoStreamEncoder::ReconfigureEncoder() { // or just discard incoming frames? RTC_CHECK(encoder_); - const webrtc::VideoEncoderFactory::CodecInfo info = - settings_.encoder_factory->QueryVideoEncoder( - encoder_config_.video_format); + codec_info_ = settings_.encoder_factory->QueryVideoEncoder( + encoder_config_.video_format); - video_sender_.RegisterExternalEncoder(encoder_.get(), - info.has_internal_source); + codec_database_.RegisterExternalEncoder(encoder_.get(), + HasInternalSource()); } - // RegisterSendCodec implies an unconditional call to - // encoder_->InitEncode(). - bool success = video_sender_.RegisterSendCodec( - &codec, number_of_cores_, - static_cast(max_data_payload_length_)) == VCM_OK; - if (!success) { + + // SetSendCodec implies an unconditional call to encoder_->InitEncode(). + bool success = codec_database_.SetSendCodec(&codec, number_of_cores_, + max_data_payload_length_); + generic_encoder_ = codec_database_.GetEncoder(); + + if (success) { + RTC_DCHECK(generic_encoder_); + next_frame_types_.clear(); + next_frame_types_.resize( + std::max(static_cast(codec.numberOfSimulcastStreams), 1), + kVideoFrameKey); + RTC_LOG(LS_VERBOSE) << " max bitrate " << codec.maxBitrate + << " start bitrate " << codec.startBitrate + << " max frame rate " << codec.maxFramerate + << " max payload size " << max_data_payload_length_; + } else { RTC_LOG(LS_ERROR) << "Failed to configure encoder."; rate_allocator_.reset(); } @@ -642,7 +663,6 @@ void VideoStreamEncoder::ReconfigureEncoder() { frame_dropper_.Reset(); frame_dropper_.SetRates(codec.startBitrate, max_framerate_); - uint32_t framerate_fps = GetInputFramerateFps(); // Force-disable frame dropper if either: // * We have screensharing with layers. // * "WebRTC-FrameDropper" field trial is "Disabled". @@ -659,11 +679,10 @@ void VideoStreamEncoder::ReconfigureEncoder() { if (rate_allocator_ && last_observed_bitrate_bps_ > 0) { // We have a new rate allocator instance and already configured target // bitrate. Update the rate allocation and notify observsers. - VideoBitrateAllocation bitrate_allocation = - GetBitrateAllocationAndNotifyObserver(last_observed_bitrate_bps_, - framerate_fps); - - video_sender_.SetChannelParameters(bitrate_allocation, framerate_fps); + const uint32_t framerate_fps = GetInputFramerateFps(); + SetEncoderRates(GetBitrateAllocationAndNotifyObserver( + last_observed_bitrate_bps_, framerate_fps), + framerate_fps); } encoder_stats_observer_->OnEncoderReconfigured(encoder_config_, streams); @@ -871,8 +890,35 @@ VideoStreamEncoder::GetBitrateAllocationAndNotifyObserver( uint32_t VideoStreamEncoder::GetInputFramerateFps() { const uint32_t default_fps = max_framerate_ != -1 ? max_framerate_ : 30; - return input_framerate_.Rate(clock_->TimeInMilliseconds()) - .value_or(default_fps); + absl::optional input_fps = + input_framerate_.Rate(clock_->TimeInMilliseconds()); + if (!input_fps || *input_fps == 0) { + return default_fps; + } + return *input_fps; +} + +void VideoStreamEncoder::SetEncoderRates( + const VideoBitrateAllocation& bitrate_allocation, + uint32_t framerate_fps) { + if (!generic_encoder_) { + return; + } + + // |bitrate_allocation| is 0 it means that the network is down or the send + // pacer is full. We currently only report this if the encoder has an internal + // source. If the encoder does not have an internal source, higher levels + // are expected to not call AddVideoFrame. We do this since its unclear + // how current encoder implementations behave when given a zero target + // bitrate. + // TODO(perkj): Make sure all known encoder implementations handle zero + // target bitrate and remove this check. + if (!HasInternalSource() && bitrate_allocation.get_sum_bps() == 0) { + return; + } + + RTC_DCHECK_GT(framerate_fps, 0); + generic_encoder_->SetEncoderParameters(bitrate_allocation, framerate_fps); } void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, @@ -901,9 +947,11 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, // from GetScalingSettings should enable or disable the frame drop. // Update input frame rate before we start using it. If we update it after - // any potential frame drop we are going to artifically increase frame sizes. - input_framerate_.Update(1u, clock_->TimeInMilliseconds()); + // any potential frame drop we are going to artificially increase frame sizes. + // Poll the rate before updating, otherwise we risk the rate being estimated + // a little too high at the start of the call when then window is small. uint32_t framerate_fps = GetInputFramerateFps(); + input_framerate_.Update(1u, clock_->TimeInMilliseconds()); int64_t now_ms = clock_->TimeInMilliseconds(); if (pending_encoder_reconfiguration_) { @@ -912,10 +960,9 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, } else if (!last_parameters_update_ms_ || now_ms - *last_parameters_update_ms_ >= vcm::VCMProcessTimer::kDefaultProcessIntervalMs) { - video_sender_.SetChannelParameters( - GetBitrateAllocationAndNotifyObserver(last_observed_bitrate_bps_, - framerate_fps), - framerate_fps); + SetEncoderRates(GetBitrateAllocationAndNotifyObserver( + last_observed_bitrate_bps_, framerate_fps), + framerate_fps); last_parameters_update_ms_.emplace(now_ms); } @@ -1069,7 +1116,48 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, } encoder_info_ = info; - video_sender_.AddVideoFrame(out_frame, nullptr, encoder_info_); + RTC_DCHECK(generic_encoder_); + RTC_DCHECK(codec_database_.MatchesCurrentResolution(out_frame.width(), + out_frame.height())); + const VideoFrameBuffer::Type buffer_type = + out_frame.video_frame_buffer()->type(); + const bool is_buffer_type_supported = + buffer_type == VideoFrameBuffer::Type::kI420 || + (buffer_type == VideoFrameBuffer::Type::kNative && + encoder_info_.supports_native_handle); + + if (!is_buffer_type_supported) { + // This module only supports software encoding. + rtc::scoped_refptr converted_buffer( + out_frame.video_frame_buffer()->ToI420()); + + if (!converted_buffer) { + RTC_LOG(LS_ERROR) << "Frame conversion failed, dropping frame."; + return; + } + + // UpdatedRect is not propagated because buffer was converted, + // therefore we can't guarantee that pixels outside of UpdateRect didn't + // change comparing to the previous frame. + out_frame = VideoFrame::Builder() + .set_video_frame_buffer(converted_buffer) + .set_timestamp_rtp(out_frame.timestamp()) + .set_timestamp_ms(out_frame.render_time_ms()) + .set_rotation(out_frame.rotation()) + .set_id(out_frame.id()) + .build(); + } + const int32_t encode_status = + generic_encoder_->Encode(out_frame, nullptr, next_frame_types_); + if (encode_status < 0) { + RTC_LOG(LS_ERROR) << "Failed to encode frame. Error code: " + << encode_status; + return; + } + + for (auto& it : next_frame_types_) { + it = kVideoFrameDelta; + } } void VideoStreamEncoder::SendKeyFrame() { @@ -1079,7 +1167,18 @@ void VideoStreamEncoder::SendKeyFrame() { } RTC_DCHECK_RUN_ON(&encoder_queue_); TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); - video_sender_.IntraFrameRequest(0); + RTC_DCHECK(!next_frame_types_.empty()); + next_frame_types_[0] = kVideoFrameKey; + if (HasInternalSource()) { + // Try to request the frame if we have an external encoder with + // internal source since AddVideoFrame never will be called. + RTC_DCHECK(generic_encoder_); + if (generic_encoder_->RequestFrame(next_frame_types_) == + WEBRTC_VIDEO_CODEC_OK) { + // Try to remove just-performed keyframe request, if stream still exists. + next_frame_types_[0] = kVideoFrameDelta; + } + } } EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage( @@ -1187,10 +1286,9 @@ void VideoStreamEncoder::OnBitrateUpdated(uint32_t bitrate_bps, uint32_t framerate_fps = GetInputFramerateFps(); frame_dropper_.SetRates((bitrate_bps + 500) / 1000, framerate_fps); - - VideoBitrateAllocation bitrate_allocation = - GetBitrateAllocationAndNotifyObserver(bitrate_bps, framerate_fps); - video_sender_.SetChannelParameters(bitrate_allocation, framerate_fps); + SetEncoderRates( + GetBitrateAllocationAndNotifyObserver(bitrate_bps, framerate_fps), + framerate_fps); encoder_start_bitrate_bps_ = bitrate_bps != 0 ? bitrate_bps : encoder_start_bitrate_bps_; @@ -1480,7 +1578,7 @@ void VideoStreamEncoder::RunPostEncode(EncodedImage encoded_image, frame_dropper_.Fill(frame_size, !keyframe); } - if (encoder_info_.has_internal_source) { + if (HasInternalSource()) { // Update frame dropper after the fact for internal sources. input_framerate_.Update(1u, clock_->TimeInMilliseconds()); frame_dropper_.Leak(GetInputFramerateFps()); @@ -1501,6 +1599,12 @@ void VideoStreamEncoder::RunPostEncode(EncodedImage encoded_image, } } +bool VideoStreamEncoder::HasInternalSource() const { + // TODO(sprang): Checking both info from encoder and from encoder factory + // until we have deprecated and removed the encoder factory info. + return codec_info_.has_internal_source || encoder_info_.has_internal_source; +} + // Class holding adaptation information. VideoStreamEncoder::AdaptCounter::AdaptCounter() { fps_counters_.resize(kScaleReasonSize); diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 861028f104..11aa57029e 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -139,6 +139,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, const uint32_t target_bitrate_bps, uint32_t framerate_fps) RTC_RUN_ON(&encoder_queue_); uint32_t GetInputFramerateFps() RTC_RUN_ON(&encoder_queue_); + void SetEncoderRates(const VideoBitrateAllocation& bitrate_allocation, + uint32_t framerate_fps) RTC_RUN_ON(&encoder_queue_); // Class holding adaptation information. class AdaptCounter final { @@ -185,6 +187,7 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, void RunPostEncode(EncodedImage encoded_image, int64_t time_sent_us, int temporal_index); + bool HasInternalSource() const RTC_RUN_ON(&encoder_queue_); rtc::Event shutdown_event_; @@ -201,7 +204,6 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, const VideoStreamEncoderSettings settings_; const RateControlSettings rate_control_settings_; - vcm::VideoSender video_sender_ RTC_GUARDED_BY(&encoder_queue_); const std::unique_ptr overuse_detector_ RTC_PT_GUARDED_BY(&encoder_queue_); std::unique_ptr quality_scaler_ RTC_GUARDED_BY(&encoder_queue_) @@ -283,6 +285,7 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, RTC_GUARDED_BY(&encoder_queue_); VideoEncoder::EncoderInfo encoder_info_ RTC_GUARDED_BY(&encoder_queue_); + VideoEncoderFactory::CodecInfo codec_info_ RTC_GUARDED_BY(&encoder_queue_); FrameDropper frame_dropper_ RTC_GUARDED_BY(&encoder_queue_); // If frame dropper is not force disabled, frame dropping might still be @@ -300,6 +303,16 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, std::unique_ptr bitrate_adjuster_ RTC_GUARDED_BY(&encoder_queue_); + // TODO(webrtc:10164): Refactor/remove these VCM classes. + // |generic_encoder_| instance is owned by |codec_database_|. + VCMGenericEncoder* generic_encoder_ RTC_GUARDED_BY(&encoder_queue_); + VCMEncodedFrameCallback generic_encoder_callback_ + RTC_GUARDED_BY(&encoder_queue_); + VCMEncoderDataBase codec_database_ RTC_GUARDED_BY(&encoder_queue_); + // TODO(sprang): Change actually support keyframe per simulcast stream, or + // turn this into a simple bool |pending_keyframe_request_|. + std::vector next_frame_types_ RTC_GUARDED_BY(&encoder_queue_); + // All public methods are proxied to |encoder_queue_|. It must must be // destroyed first to make sure no tasks are run that use other members. rtc::TaskQueue encoder_queue_; diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index b419e60c2d..9bf9cfd697 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -49,8 +49,9 @@ const int kMinFramerateFps = 2; const int kMinBalancedFramerateFps = 7; const int64_t kFrameTimeoutMs = 100; const size_t kMaxPayloadLength = 1440; -const int kTargetBitrateBps = 1000000; -const int kLowTargetBitrateBps = kTargetBitrateBps / 10; +const uint32_t kTargetBitrateBps = 1000000; +const uint32_t kSimulcastTargetBitrateBps = 3150000; +const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10; const int kMaxInitialFramedrop = 4; const int kDefaultFramerate = 30; const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate; @@ -342,7 +343,8 @@ class VideoStreamEncoderTest : public ::testing::Test { VideoEncoderConfig video_encoder_config; video_encoder_config.codec_type = PayloadStringToCodecType(payload_name); video_encoder_config.number_of_streams = num_streams; - video_encoder_config.max_bitrate_bps = kTargetBitrateBps; + video_encoder_config.max_bitrate_bps = + num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps; video_encoder_config.video_stream_factory = new rtc::RefCountedObject(num_temporal_layers, kDefaultFramerate); @@ -584,16 +586,42 @@ class VideoStreamEncoderTest : public ::testing::Test { rate_factor_ = rate_factor; } - uint32_t GetLastFramerate() { + uint32_t GetLastFramerate() const { rtc::CritScope lock(&local_crit_sect_); return last_framerate_; } - VideoFrame::UpdateRect GetLastUpdateRect() { + VideoFrame::UpdateRect GetLastUpdateRect() const { rtc::CritScope lock(&local_crit_sect_); return last_update_rect_; } + const std::vector& LastFrameTypes() const { + rtc::CritScope lock(&local_crit_sect_); + return last_frame_types_; + } + + void InjectFrame(const VideoFrame& input_image, bool keyframe) { + const std::vector frame_type = {keyframe ? kVideoFrameKey + : kVideoFrameDelta}; + { + rtc::CritScope lock(&local_crit_sect_); + last_frame_types_ = frame_type; + } + FakeEncoder::Encode(input_image, nullptr, &frame_type); + } + + void ExpectNullFrame() { + rtc::CritScope lock(&local_crit_sect_); + expect_null_frame_ = true; + } + + absl::optional GetAndResetLastBitrateAllocation() { + auto allocation = last_bitrate_allocation_; + last_bitrate_allocation_.reset(); + return allocation; + } + private: int32_t Encode(const VideoFrame& input_image, const CodecSpecificInfo* codec_specific_info, @@ -601,9 +629,16 @@ class VideoStreamEncoderTest : public ::testing::Test { bool block_encode; { rtc::CritScope lock(&local_crit_sect_); - EXPECT_GT(input_image.timestamp(), timestamp_); - EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_); - EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90); + if (expect_null_frame_) { + EXPECT_EQ(input_image.timestamp(), 0u); + EXPECT_EQ(input_image.width(), 1); + last_frame_types_ = *frame_types; + expect_null_frame_ = false; + } else { + EXPECT_GT(input_image.timestamp(), timestamp_); + EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_); + EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90); + } timestamp_ = input_image.timestamp(); ntp_time_ms_ = input_image.ntp_time_ms(); @@ -612,6 +647,7 @@ class VideoStreamEncoderTest : public ::testing::Test { block_encode = block_next_encode_; block_next_encode_ = false; last_update_rect_ = input_image.update_rect(); + last_frame_types_ = *frame_types; } int32_t result = FakeEncoder::Encode(input_image, codec_specific_info, frame_types); @@ -658,6 +694,7 @@ class VideoStreamEncoderTest : public ::testing::Test { } } last_framerate_ = framerate; + last_bitrate_allocation_ = rate_allocation; return FakeEncoder::SetRateAllocation(adjusted_rate_allocation, framerate); } @@ -677,8 +714,11 @@ class VideoStreamEncoderTest : public ::testing::Test { bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false; double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0; uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0; + absl::optional last_bitrate_allocation_; VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0}; + std::vector last_frame_types_; + bool expect_null_frame_ = false; }; class TestSink : public VideoStreamEncoder::EncoderSink { @@ -744,6 +784,11 @@ class VideoStreamEncoderTest : public ::testing::Test { return min_transmit_bitrate_bps_; } + void SetNumExpectedLayers(size_t num_layers) { + rtc::CritScope lock(&crit_); + num_expected_layers_ = num_layers; + } + private: Result OnEncodedImage( const EncodedImage& encoded_image, @@ -751,10 +796,18 @@ class VideoStreamEncoderTest : public ::testing::Test { const RTPFragmentationHeader* fragmentation) override { rtc::CritScope lock(&crit_); EXPECT_TRUE(expect_frames_); - last_timestamp_ = encoded_image.Timestamp(); + uint32_t timestamp = encoded_image.Timestamp(); + if (last_timestamp_ != timestamp) { + num_received_layers_ = 1; + } else { + ++num_received_layers_; + } + last_timestamp_ = timestamp; last_width_ = encoded_image._encodedWidth; last_height_ = encoded_image._encodedHeight; - encoded_frame_event_.Set(); + if (num_received_layers_ == num_expected_layers_) { + encoded_frame_event_.Set(); + } return Result(Result::OK, last_timestamp_); } @@ -773,6 +826,8 @@ class VideoStreamEncoderTest : public ::testing::Test { uint32_t last_timestamp_ = 0; uint32_t last_height_ = 0; uint32_t last_width_ = 0; + size_t num_expected_layers_ = 1; + size_t num_received_layers_ = 0; bool expect_frames_ = true; int number_of_reconfigurations_ = 0; int min_transmit_bitrate_bps_ = 0; @@ -2168,28 +2223,38 @@ TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) { .Times(1); video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0); - const int64_t kStartTimeMs = 1; video_source_.IncomingCapturedFrame( - CreateFrame(kStartTimeMs, codec_width_, codec_height_)); - WaitForEncodedFrame(kStartTimeMs); + CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_)); + WaitForEncodedFrame(rtc::TimeMillis()); + absl::optional bitrate_allocation = + fake_encoder_.GetAndResetLastBitrateAllocation(); + // Check that encoder has been updated too, not just allocation observer. + EXPECT_EQ(bitrate_allocation->get_sum_bps(), kLowTargetBitrateBps); + fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps); // Not called on second frame. EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate)) .Times(0); video_source_.IncomingCapturedFrame( - CreateFrame(kStartTimeMs + 1, codec_width_, codec_height_)); - WaitForEncodedFrame(kStartTimeMs + 1); + CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_)); + WaitForEncodedFrame(rtc::TimeMillis()); + fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps); // Called after a process interval. const int64_t kProcessIntervalMs = vcm::VCMProcessTimer::kDefaultProcessIntervalMs; - fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec * - (kProcessIntervalMs + (1000 / kDefaultFps))); EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate)) .Times(1); - video_source_.IncomingCapturedFrame(CreateFrame( - kStartTimeMs + kProcessIntervalMs, codec_width_, codec_height_)); - WaitForEncodedFrame(kStartTimeMs + kProcessIntervalMs); + const int64_t start_time_ms = rtc::TimeMillis(); + while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) { + video_source_.IncomingCapturedFrame( + CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_)); + WaitForEncodedFrame(rtc::TimeMillis()); + fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps); + } + + // Since rates are unchanged, encoder should not be reconfigured. + EXPECT_FALSE(fake_encoder_.GetAndResetLastBitrateAllocation().has_value()); video_stream_encoder_->Stop(); } @@ -3429,4 +3494,93 @@ TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) { video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, SetsFrameTypes) { + video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + + // First frame is always keyframe. + video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr)); + WaitForEncodedFrame(1); + EXPECT_THAT(fake_encoder_.LastFrameTypes(), + testing::ElementsAre(FrameType{kVideoFrameKey})); + + // Insert delta frame. + video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr)); + WaitForEncodedFrame(2); + EXPECT_THAT(fake_encoder_.LastFrameTypes(), + testing::ElementsAre(FrameType{kVideoFrameDelta})); + + // Request next frame be a key-frame. + video_stream_encoder_->SendKeyFrame(); + video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr)); + WaitForEncodedFrame(3); + EXPECT_THAT(fake_encoder_.LastFrameTypes(), + testing::ElementsAre(FrameType{kVideoFrameKey})); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) { + // Setup simulcast with three streams. + ResetEncoder("VP8", 3, 1, 1, false); + video_stream_encoder_->OnBitrateUpdated(kSimulcastTargetBitrateBps, 0, 0); + // Wait for all three layers before triggering event. + sink_.SetNumExpectedLayers(3); + + // First frame is always keyframe. + video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr)); + WaitForEncodedFrame(1); + EXPECT_THAT(fake_encoder_.LastFrameTypes(), + testing::ElementsAreArray( + {kVideoFrameKey, kVideoFrameKey, kVideoFrameKey})); + + // Insert delta frame. + video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr)); + WaitForEncodedFrame(2); + EXPECT_THAT(fake_encoder_.LastFrameTypes(), + testing::ElementsAreArray( + {kVideoFrameDelta, kVideoFrameDelta, kVideoFrameDelta})); + + // Request next frame be a key-frame. + // Only first stream is configured to produce key-frame. + video_stream_encoder_->SendKeyFrame(); + video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr)); + WaitForEncodedFrame(3); + EXPECT_THAT(fake_encoder_.LastFrameTypes(), + testing::ElementsAreArray( + {kVideoFrameKey, kVideoFrameDelta, kVideoFrameDelta})); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) { + // Configure internal source factory and setup test again. + encoder_factory_.SetHasInternalSource(true); + ResetEncoder("VP8", 1, 1, 1, false); + video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + + // Call encoder directly, simulating internal source where encoded frame + // callback in VideoStreamEncoder is called despite no OnFrame(). + fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true); + EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs)); + EXPECT_THAT(fake_encoder_.LastFrameTypes(), + testing::ElementsAre(FrameType{kVideoFrameKey})); + + const std::vector kDeltaFrame = {kVideoFrameDelta}; + // Need to set timestamp manually since manually for injected frame. + VideoFrame frame = CreateFrame(101, nullptr); + frame.set_timestamp(101); + fake_encoder_.InjectFrame(frame, false); + EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs)); + EXPECT_THAT(fake_encoder_.LastFrameTypes(), + testing::ElementsAre(FrameType{kVideoFrameDelta})); + + // Request key-frame. The forces a dummy frame down into the encoder. + fake_encoder_.ExpectNullFrame(); + video_stream_encoder_->SendKeyFrame(); + EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs)); + EXPECT_THAT(fake_encoder_.LastFrameTypes(), + testing::ElementsAre(FrameType{kVideoFrameKey})); + + video_stream_encoder_->Stop(); +} } // namespace webrtc