diff --git a/api/video_codecs/vp8_frame_buffer_controller.h b/api/video_codecs/vp8_frame_buffer_controller.h index f3f6745504..c506fa4ca5 100644 --- a/api/video_codecs/vp8_frame_buffer_controller.h +++ b/api/video_codecs/vp8_frame_buffer_controller.h @@ -117,15 +117,10 @@ class Vp8FrameBufferController { // parameter use in the UpdateLayerConfig() call. // |is_keyframe| must be true iff the encoder decided to encode this frame as // a keyframe. - // If the encoder decided to drop this frame, |size_bytes| must be set to 0, - // otherwise it should indicate the size in bytes of the encoded frame. - // If |size_bytes| > 0, and |info| is not null, the TemporalLayers - // instance my update |info| with codec specific data such as temporal id. - // Some fields of this struct may have already been populated by the encoder, - // check before overwriting. - // If |size_bytes| > 0, |qp| should indicate the frame-level QP this frame was - // encoded at. If the encoder does not support extracting this, |qp| should be - // set to 0. + // If |info| is not null, the TemporalLayers instance may update |info| with + // codec specific data such as temporal id. + // |qp| should indicate the frame-level QP this frame was encoded at. If the + // encoder does not support extracting this, |qp| should be set to 0. virtual void OnEncodeDone(size_t stream_index, uint32_t rtp_timestamp, size_t size_bytes, @@ -133,6 +128,9 @@ class Vp8FrameBufferController { int qp, CodecSpecificInfo* info) = 0; + // Called when a frame is dropped by the encoder. + virtual void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) = 0; + // Called by the encoder when the packet loss rate changes. // |packet_loss_rate| runs between 0.0 (no loss) and 1.0 (everything lost). virtual void OnPacketLossRateUpdate(float packet_loss_rate) = 0; diff --git a/api/video_codecs/vp8_temporal_layers.cc b/api/video_codecs/vp8_temporal_layers.cc index 4c552852fc..004b10fb84 100644 --- a/api/video_codecs/vp8_temporal_layers.cc +++ b/api/video_codecs/vp8_temporal_layers.cc @@ -70,6 +70,12 @@ void Vp8TemporalLayers::OnEncodeDone(size_t stream_index, is_keyframe, qp, info); } +void Vp8TemporalLayers::OnFrameDropped(size_t stream_index, + uint32_t rtp_timestamp) { + RTC_DCHECK_LT(stream_index, controllers_.size()); + controllers_[stream_index]->OnFrameDropped(stream_index, rtp_timestamp); +} + void Vp8TemporalLayers::OnPacketLossRateUpdate(float packet_loss_rate) { for (auto& controller : controllers_) { controller->OnPacketLossRateUpdate(packet_loss_rate); diff --git a/api/video_codecs/vp8_temporal_layers.h b/api/video_codecs/vp8_temporal_layers.h index c46699bbdd..dcf04c618b 100644 --- a/api/video_codecs/vp8_temporal_layers.h +++ b/api/video_codecs/vp8_temporal_layers.h @@ -55,6 +55,8 @@ class Vp8TemporalLayers final : public Vp8FrameBufferController { int qp, CodecSpecificInfo* info) override; + void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) override; + void OnPacketLossRateUpdate(float packet_loss_rate) override; void OnRttUpdate(int64_t rtt_ms) override; diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/modules/video_coding/codecs/vp8/default_temporal_layers.cc index a00571785f..702a4bc250 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -454,14 +454,15 @@ void DefaultTemporalLayers::OnEncodeDone(size_t stream_index, RTC_DCHECK_LT(stream_index, StreamCount()); RTC_DCHECK_GT(num_layers_, 0); - auto pending_frame = pending_frames_.find(rtp_timestamp); - RTC_DCHECK(pending_frame != pending_frames_.end()); - if (size_bytes == 0) { - pending_frames_.erase(pending_frame); + RTC_LOG(LS_WARNING) << "Empty frame; treating as dropped."; + OnFrameDropped(stream_index, rtp_timestamp); return; } + auto pending_frame = pending_frames_.find(rtp_timestamp); + RTC_DCHECK(pending_frame != pending_frames_.end()); + PendingFrame& frame = pending_frame->second; const Vp8FrameConfig& frame_config = frame.dependency_info.frame_config; #if RTC_DCHECK_IS_ON @@ -539,6 +540,13 @@ void DefaultTemporalLayers::OnEncodeDone(size_t stream_index, } } +void DefaultTemporalLayers::OnFrameDropped(size_t stream_index, + uint32_t rtp_timestamp) { + auto pending_frame = pending_frames_.find(rtp_timestamp); + RTC_DCHECK(pending_frame != pending_frames_.end()); + pending_frames_.erase(pending_frame); +} + void DefaultTemporalLayers::OnPacketLossRateUpdate(float packet_loss_rate) {} void DefaultTemporalLayers::OnRttUpdate(int64_t rtt_ms) {} diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.h b/modules/video_coding/codecs/vp8/default_temporal_layers.h index 1667b5cd39..67839f6c94 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -57,6 +57,8 @@ class DefaultTemporalLayers final : public Vp8FrameBufferController { int qp, CodecSpecificInfo* info) override; + void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) override; + void OnPacketLossRateUpdate(float packet_loss_rate) override; void OnRttUpdate(int64_t rtt_ms) override; diff --git a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc index bc7d4da245..3e43275fdb 100644 --- a/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc +++ b/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc @@ -1105,8 +1105,8 @@ int LibvpxVp8Encoder::GetEncodedPartitions(const VideoFrame& input_image) { result = WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT; if (encoded_images_[encoder_idx].size() == 0) { // Dropped frame that will be re-encoded. - frame_buffer_controller_->OnEncodeDone( - stream_idx, input_image.timestamp(), 0, false, 0, nullptr); + frame_buffer_controller_->OnFrameDropped(stream_idx, + input_image.timestamp()); } } } diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.cc b/modules/video_coding/codecs/vp8/screenshare_layers.cc index 2c1ccd0ebd..65a445e50b 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -280,8 +280,8 @@ void ScreenshareLayers::OnEncodeDone(size_t stream_index, RTC_DCHECK_LT(stream_index, StreamCount()); if (size_bytes == 0) { - layers_[active_layer_].state = TemporalLayer::State::kDropped; - ++stats_.num_overshoots_; + RTC_LOG(LS_WARNING) << "Empty frame; treating as dropped."; + OnFrameDropped(stream_index, rtp_timestamp); return; } @@ -383,6 +383,12 @@ void ScreenshareLayers::OnEncodeDone(size_t stream_index, } } +void ScreenshareLayers::OnFrameDropped(size_t stream_index, + uint32_t rtp_timestamp) { + layers_[active_layer_].state = TemporalLayer::State::kDropped; + ++stats_.num_overshoots_; +} + void ScreenshareLayers::OnPacketLossRateUpdate(float packet_loss_rate) {} void ScreenshareLayers::OnRttUpdate(int64_t rtt_ms) {} diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.h b/modules/video_coding/codecs/vp8/screenshare_layers.h index 111b1c1322..69a1737e47 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -61,6 +61,8 @@ class ScreenshareLayers final : public Vp8FrameBufferController { int qp, CodecSpecificInfo* info) override; + void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) override; + void OnPacketLossRateUpdate(float packet_loss_rate) override; void OnRttUpdate(int64_t rtt_ms) override; diff --git a/test/fake_vp8_encoder.cc b/test/fake_vp8_encoder.cc index bd92bc6f07..b714b4d8f3 100644 --- a/test/fake_vp8_encoder.cc +++ b/test/fake_vp8_encoder.cc @@ -79,9 +79,13 @@ void FakeVP8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, codec_specific->codecType = kVideoCodecVP8; codec_specific->codecSpecific.VP8.keyIdx = kNoKeyIdx; codec_specific->codecSpecific.VP8.nonReference = false; - frame_buffer_controller_->OnEncodeDone( - stream_idx, timestamp, size_bytes, - frame_type == VideoFrameType::kVideoFrameKey, -1, codec_specific); + if (size_bytes > 0) { + frame_buffer_controller_->OnEncodeDone( + stream_idx, timestamp, size_bytes, + frame_type == VideoFrameType::kVideoFrameKey, -1, codec_specific); + } else { + frame_buffer_controller_->OnFrameDropped(stream_idx, timestamp); + } } std::unique_ptr FakeVP8Encoder::EncodeHook(