Add OnFrameDropped() to Vp8FrameBufferController

Prior to this CL, this was indicated by passing |size_bytes| = 0
to the method.

Bug: webrtc:10501
Change-Id: Icff3bb83344834dc62d62bde5ec5d05096a08e11
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/132712
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Commit-Queue: Elad Alon <eladalon@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27620}

api/video_codecs/vp8_frame_buffer_controller.h

@@ -117,15 +117,10 @@ class Vp8FrameBufferController {
   // parameter use in the UpdateLayerConfig() call.
   // |is_keyframe| must be true iff the encoder decided to encode this frame as
   // a keyframe.
-  // If the encoder decided to drop this frame, |size_bytes| must be set to 0,
+  // If |info| is not null, the TemporalLayers instance may update |info| with
-  // otherwise it should indicate the size in bytes of the encoded frame.
+  // codec specific data such as temporal id.
-  // If |size_bytes| > 0, and |info| is not null, the TemporalLayers
+  // |qp| should indicate the frame-level QP this frame was encoded at. If the
-  // instance my update |info| with codec specific data such as temporal id.
+  // encoder does not support extracting this, |qp| should be set to 0.
-  // Some fields of this struct may have already been populated by the encoder,
-  // check before overwriting.
-  // If |size_bytes| > 0, |qp| should indicate the frame-level QP this frame was
-  // encoded at. If the encoder does not support extracting this, |qp| should be
-  // set to 0.
   virtual void OnEncodeDone(size_t stream_index,
                             uint32_t rtp_timestamp,
                             size_t size_bytes,
@@ -133,6 +128,9 @@ class Vp8FrameBufferController {
                             int qp,
                             CodecSpecificInfo* info) = 0;
 
+  // Called when a frame is dropped by the encoder.
+  virtual void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) = 0;
+
   // Called by the encoder when the packet loss rate changes.
   // |packet_loss_rate| runs between 0.0 (no loss) and 1.0 (everything lost).
   virtual void OnPacketLossRateUpdate(float packet_loss_rate) = 0;

api/video_codecs/vp8_temporal_layers.cc

@@ -70,6 +70,12 @@ void Vp8TemporalLayers::OnEncodeDone(size_t stream_index,
                                                   is_keyframe, qp, info);
 }
 
+void Vp8TemporalLayers::OnFrameDropped(size_t stream_index,
+                                       uint32_t rtp_timestamp) {
+  RTC_DCHECK_LT(stream_index, controllers_.size());
+  controllers_[stream_index]->OnFrameDropped(stream_index, rtp_timestamp);
+}
+
 void Vp8TemporalLayers::OnPacketLossRateUpdate(float packet_loss_rate) {
   for (auto& controller : controllers_) {
     controller->OnPacketLossRateUpdate(packet_loss_rate);

api/video_codecs/vp8_temporal_layers.h

@@ -55,6 +55,8 @@ class Vp8TemporalLayers final : public Vp8FrameBufferController {
                     int qp,
                     CodecSpecificInfo* info) override;
 
+  void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) override;
+
   void OnPacketLossRateUpdate(float packet_loss_rate) override;
 
   void OnRttUpdate(int64_t rtt_ms) override;

modules/video_coding/codecs/vp8/default_temporal_layers.cc

@@ -454,14 +454,15 @@ void DefaultTemporalLayers::OnEncodeDone(size_t stream_index,
   RTC_DCHECK_LT(stream_index, StreamCount());
   RTC_DCHECK_GT(num_layers_, 0);
 
-  auto pending_frame = pending_frames_.find(rtp_timestamp);
-  RTC_DCHECK(pending_frame != pending_frames_.end());
-
   if (size_bytes == 0) {
-    pending_frames_.erase(pending_frame);
+    RTC_LOG(LS_WARNING) << "Empty frame; treating as dropped.";
+    OnFrameDropped(stream_index, rtp_timestamp);
     return;
   }
 
+  auto pending_frame = pending_frames_.find(rtp_timestamp);
+  RTC_DCHECK(pending_frame != pending_frames_.end());
+
   PendingFrame& frame = pending_frame->second;
   const Vp8FrameConfig& frame_config = frame.dependency_info.frame_config;
 #if RTC_DCHECK_IS_ON
@@ -539,6 +540,13 @@ void DefaultTemporalLayers::OnEncodeDone(size_t stream_index,
   }
 }
 
+void DefaultTemporalLayers::OnFrameDropped(size_t stream_index,
+                                           uint32_t rtp_timestamp) {
+  auto pending_frame = pending_frames_.find(rtp_timestamp);
+  RTC_DCHECK(pending_frame != pending_frames_.end());
+  pending_frames_.erase(pending_frame);
+}
+
 void DefaultTemporalLayers::OnPacketLossRateUpdate(float packet_loss_rate) {}
 
 void DefaultTemporalLayers::OnRttUpdate(int64_t rtt_ms) {}

modules/video_coding/codecs/vp8/default_temporal_layers.h

@@ -57,6 +57,8 @@ class DefaultTemporalLayers final : public Vp8FrameBufferController {
                     int qp,
                     CodecSpecificInfo* info) override;
 
+  void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) override;
+
   void OnPacketLossRateUpdate(float packet_loss_rate) override;
 
   void OnRttUpdate(int64_t rtt_ms) override;

modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc

@@ -1105,8 +1105,8 @@ int LibvpxVp8Encoder::GetEncodedPartitions(const VideoFrame& input_image) {
         result = WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT;
         if (encoded_images_[encoder_idx].size() == 0) {
           // Dropped frame that will be re-encoded.
-          frame_buffer_controller_->OnEncodeDone(
+          frame_buffer_controller_->OnFrameDropped(stream_idx,
-              stream_idx, input_image.timestamp(), 0, false, 0, nullptr);
+                                                   input_image.timestamp());
         }
       }
     }

modules/video_coding/codecs/vp8/screenshare_layers.cc

@@ -280,8 +280,8 @@ void ScreenshareLayers::OnEncodeDone(size_t stream_index,
   RTC_DCHECK_LT(stream_index, StreamCount());
 
   if (size_bytes == 0) {
-    layers_[active_layer_].state = TemporalLayer::State::kDropped;
+    RTC_LOG(LS_WARNING) << "Empty frame; treating as dropped.";
-    ++stats_.num_overshoots_;
+    OnFrameDropped(stream_index, rtp_timestamp);
     return;
   }
 
@@ -383,6 +383,12 @@ void ScreenshareLayers::OnEncodeDone(size_t stream_index,
   }
 }
 
+void ScreenshareLayers::OnFrameDropped(size_t stream_index,
+                                       uint32_t rtp_timestamp) {
+  layers_[active_layer_].state = TemporalLayer::State::kDropped;
+  ++stats_.num_overshoots_;
+}
+
 void ScreenshareLayers::OnPacketLossRateUpdate(float packet_loss_rate) {}
 
 void ScreenshareLayers::OnRttUpdate(int64_t rtt_ms) {}

modules/video_coding/codecs/vp8/screenshare_layers.h

@@ -61,6 +61,8 @@ class ScreenshareLayers final : public Vp8FrameBufferController {
                     int qp,
                     CodecSpecificInfo* info) override;
 
+  void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) override;
+
   void OnPacketLossRateUpdate(float packet_loss_rate) override;
 
   void OnRttUpdate(int64_t rtt_ms) override;

test/fake_vp8_encoder.cc

@@ -79,9 +79,13 @@ void FakeVP8Encoder::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
   codec_specific->codecType = kVideoCodecVP8;
   codec_specific->codecSpecific.VP8.keyIdx = kNoKeyIdx;
   codec_specific->codecSpecific.VP8.nonReference = false;
-  frame_buffer_controller_->OnEncodeDone(
+  if (size_bytes > 0) {
-      stream_idx, timestamp, size_bytes,
+    frame_buffer_controller_->OnEncodeDone(
-      frame_type == VideoFrameType::kVideoFrameKey, -1, codec_specific);
+        stream_idx, timestamp, size_bytes,
+        frame_type == VideoFrameType::kVideoFrameKey, -1, codec_specific);
+  } else {
+    frame_buffer_controller_->OnFrameDropped(stream_idx, timestamp);
+  }
 }
 
 std::unique_ptr<RTPFragmentationHeader> FakeVP8Encoder::EncodeHook(