Add OnLossNotification() to VideoEncoder and Vp8FrameBufferController

Bug: webrtc:10501
Change-Id: I33e8bfcf16cf24aadcfdf214d7d9bcd495bf9348
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/131021
Commit-Queue: Elad Alon <eladalon@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27449}

api/video_codecs/video_encoder.cc

@@ -121,6 +121,9 @@ void VideoEncoder::OnPacketLossRateUpdate(float packet_loss_rate) {}
 
 void VideoEncoder::OnRttUpdate(int64_t rtt_ms) {}
 
+void VideoEncoder::OnLossNotification(
+    const LossNotification& loss_notification) {}
+
 // TODO(webrtc:9722): Remove and make pure virtual.
 VideoEncoder::EncoderInfo VideoEncoder::GetEncoderInfo() const {
   return EncoderInfo();

api/video_codecs/video_encoder.h

@@ -208,6 +208,27 @@ class RTC_EXPORT VideoEncoder {
     DataRate bandwidth_allocation;
   };
 
+  struct LossNotification {
+    // The timestamp of the last decodable frame *prior* to the last received.
+    // (The last received - described below - might itself be decodable or not.)
+    uint32_t timestamp_of_last_decodable;
+    // The timestamp of the last received frame.
+    uint32_t timestamp_of_last_received;
+    // Describes whether the dependencies of the last received frame were
+    // all decodable.
+    // |false| if some dependencies were undecodable, |true| if all dependencies
+    // were decodable, and |nullopt| if the dependencies are unknown.
+    absl::optional<bool> is_last_received_dependencies_decodable;
+    // Describes whether the received frame was decodable.
+    // |false| if some dependency was undecodable or if some packet belonging
+    // to the last received frame was missed.
+    // |true| if all dependencies were decodable and all packets belonging
+    // to the last received frame were received.
+    // |nullopt| if no packet belonging to the last frame was missed, but the
+    // last packet in the frame was not yet received.
+    absl::optional<bool> is_last_received_decodable;
+  };
+
   static VideoCodecVP8 GetDefaultVp8Settings();
   static VideoCodecVP9 GetDefaultVp9Settings();
   static VideoCodecH264 GetDefaultH264Settings();
@@ -291,6 +312,9 @@ class RTC_EXPORT VideoEncoder {
   // Input:   - rtt_ms            : The new RTT, in milliseconds.
   virtual void OnRttUpdate(int64_t rtt_ms);
 
+  // Called when a loss notification is received.
+  virtual void OnLossNotification(const LossNotification& loss_notification);
+
   // Returns meta-data about the encoder, such as implementation name.
   // The output of this method may change during runtime. For instance if a
   // hardware encoder fails, it may fall back to doing software encoding using

api/video_codecs/vp8_frame_buffer_controller.h

@@ -14,7 +14,9 @@
 #include <memory>
 #include <vector>
 
+#include "absl/types/optional.h"
 #include "api/video_codecs/video_codec.h"
+#include "api/video_codecs/video_encoder.h"
 #include "api/video_codecs/vp8_frame_config.h"
 
 namespace webrtc {
@@ -137,6 +139,10 @@ class Vp8FrameBufferController {
 
   // Called by the encoder when the round trip time changes.
   virtual void OnRttUpdate(int64_t rtt_ms) = 0;
+
+  // Called when a loss notification is received.
+  virtual void OnLossNotification(
+      const VideoEncoder::LossNotification loss_notification) = 0;
 };
 
 // Interface for a factory of Vp8FrameBufferController instances.

api/video_codecs/vp8_temporal_layers.cc

@@ -82,4 +82,11 @@ void Vp8TemporalLayers::OnRttUpdate(int64_t rtt_ms) {
   }
 }
 
+void Vp8TemporalLayers::OnLossNotification(
+    const VideoEncoder::LossNotification loss_notification) {
+  for (auto& controller : controllers_) {
+    controller->OnLossNotification(loss_notification);
+  }
+}
+
 }  // namespace webrtc

api/video_codecs/vp8_temporal_layers.h

@@ -59,6 +59,9 @@ class Vp8TemporalLayers final : public Vp8FrameBufferController {
 
   void OnRttUpdate(int64_t rtt_ms) override;
 
+  void OnLossNotification(
+      const VideoEncoder::LossNotification loss_notification) override;
+
  private:
   std::vector<std::unique_ptr<Vp8FrameBufferController>> controllers_;
 };

modules/video_coding/codecs/vp8/default_temporal_layers.cc

@@ -542,6 +542,9 @@ void DefaultTemporalLayers::OnPacketLossRateUpdate(float packet_loss_rate) {}
 
 void DefaultTemporalLayers::OnRttUpdate(int64_t rtt_ms) {}
 
+void DefaultTemporalLayers::OnLossNotification(
+    const VideoEncoder::LossNotification loss_notification) {}
+
 TemplateStructure DefaultTemporalLayers::GetTemplateStructure(
     int num_layers) const {
   RTC_CHECK_LT(num_layers, 5);

modules/video_coding/codecs/vp8/default_temporal_layers.h

@@ -61,6 +61,9 @@ class DefaultTemporalLayers final : public Vp8FrameBufferController {
 
   void OnRttUpdate(int64_t rtt_ms) override;
 
+  void OnLossNotification(
+      const VideoEncoder::LossNotification loss_notification) override;
+
  private:
   struct DependencyInfo {
     DependencyInfo() = default;

modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc

@@ -330,6 +330,13 @@ void LibvpxVp8Encoder::OnRttUpdate(int64_t rtt_ms) {
   }
 }
 
+void LibvpxVp8Encoder::OnLossNotification(
+    const LossNotification& loss_notification) {
+  if (frame_buffer_controller_) {
+    frame_buffer_controller_->OnLossNotification(loss_notification);
+  }
+}
+
 void LibvpxVp8Encoder::SetStreamState(bool send_stream, int stream_idx) {
   if (send_stream && !send_stream_[stream_idx]) {
     // Need a key frame if we have not sent this stream before.

modules/video_coding/codecs/vp8/libvpx_vp8_encoder.h

@@ -61,6 +61,8 @@ class LibvpxVp8Encoder : public VideoEncoder {
 
   void OnRttUpdate(int64_t rtt_ms) override;
 
+  void OnLossNotification(const LossNotification& loss_notification) override;
+
   EncoderInfo GetEncoderInfo() const override;
 
   static vpx_enc_frame_flags_t EncodeFlags(const Vp8FrameConfig& references);

modules/video_coding/codecs/vp8/screenshare_layers.cc

@@ -388,6 +388,9 @@ void ScreenshareLayers::OnPacketLossRateUpdate(float packet_loss_rate) {}
 
 void ScreenshareLayers::OnRttUpdate(int64_t rtt_ms) {}
 
+void ScreenshareLayers::OnLossNotification(
+    const VideoEncoder::LossNotification loss_notification) {}
+
 TemplateStructure ScreenshareLayers::GetTemplateStructure(
     int num_layers) const {
   RTC_CHECK_LT(num_layers, 3);

modules/video_coding/codecs/vp8/screenshare_layers.h

@@ -65,6 +65,9 @@ class ScreenshareLayers final : public Vp8FrameBufferController {
 
   void OnRttUpdate(int64_t rtt_ms) override;
 
+  void OnLossNotification(
+      const VideoEncoder::LossNotification loss_notification) override;
+
  private:
   enum class TemporalLayerState : int { kDrop, kTl0, kTl1, kTl1Sync };