Add ProtectionBitrateCalculator as an abstract class. ProtectionBitrateCalculatorDefault implements ProtectionBitrateCalculator. Register VideoSendStream to packet feedback

Bug: webrtc:8656
Change-Id: Iab4f6ab8997cb082762218afc8580e9985ac2522
Reviewed-on: https://webrtc-review.googlesource.com/33010
Commit-Queue: Ying Wang <yinwa@webrtc.org>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#21348}

modules/video_coding/BUILD.gn

@@ -57,8 +57,9 @@ rtc_static_library("video_coding") {
     "packet.h",
     "packet_buffer.cc",
     "packet_buffer.h",
-    "protection_bitrate_calculator.cc",
     "protection_bitrate_calculator.h",
+    "protection_bitrate_calculator_default.cc",
+    "protection_bitrate_calculator_default.h",
     "qp_parser.cc",
     "qp_parser.h",
     "receiver.cc",

modules/video_coding/protection_bitrate_calculator.h

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
@@ -13,6 +13,7 @@
 
 #include <list>
 #include <memory>
+#include <vector>
 
 #include "modules/include/module_common_types.h"
 #include "modules/video_coding/include/video_coding.h"
@@ -22,57 +23,38 @@
 
 namespace webrtc {
 
-// ProtectionBitrateCalculator calculates how much of the allocated network
-// capacity that can be used by an encoder and how much that
-// is needed for redundant packets such as FEC and NACK. It uses an
-// implementation of |VCMProtectionCallback| to set new FEC parameters and get
-// the bitrate currently used for FEC and NACK.
-// Usage:
-// Setup by calling SetProtectionMethod and SetEncodingData.
-// For each encoded image, call UpdateWithEncodedData.
-// Each time the bandwidth estimate change, call SetTargetRates. SetTargetRates
-// will return the bitrate that can be used by an encoder.
-// A lock is used to protect internal states, so methods can be called on an
-// arbitrary thread.
 class ProtectionBitrateCalculator {
  public:
-  ProtectionBitrateCalculator(Clock* clock,
+  virtual ~ProtectionBitrateCalculator() {}
-                              VCMProtectionCallback* protection_callback);
-  ~ProtectionBitrateCalculator();
 
-  void SetProtectionMethod(bool enable_fec, bool enable_nack);
+  virtual void SetProtectionMethod(bool enable_fec, bool enable_nack) = 0;
 
   // Informs media optimization of initial encoding state.
-  void SetEncodingData(size_t width,
+  virtual void SetEncodingData(size_t width,
-                       size_t height,
+                               size_t height,
-                       size_t num_temporal_layers,
+                               size_t num_temporal_layers,
-                       size_t max_payload_size);
+                               size_t max_payload_size) = 0;
 
   // Returns target rate for the encoder given the channel parameters.
   // Inputs:  estimated_bitrate_bps - the estimated network bitrate in bits/s.
   //          actual_framerate - encoder frame rate.
   //          fraction_lost - packet loss rate in % in the network.
   //          round_trip_time_ms - round trip time in milliseconds.
-  uint32_t SetTargetRates(uint32_t estimated_bitrate_bps,
+  virtual uint32_t SetTargetRates(uint32_t estimated_bitrate_bps,
-                          int actual_framerate,
+                                  int actual_framerate,
-                          uint8_t fraction_lost,
+                                  uint8_t fraction_lost,
-                          int64_t round_trip_time_ms);
+                                  int64_t round_trip_time_ms) = 0;
+
+  virtual uint32_t SetTargetRates(uint32_t estimated_bitrate_bps,
+                                  std::vector<uint8_t> loss_mask_vector,
+                                  int64_t round_trip_time_ms) = 0;
+
   // Informs of encoded output.
-  void UpdateWithEncodedData(const EncodedImage& encoded_image);
+  virtual void UpdateWithEncodedData(const EncodedImage& encoded_image) = 0;
 
- private:
+  virtual void OnLossMaskVector(const std::vector<bool> loss_mask_vector) = 0;
-  enum { kBitrateAverageWinMs = 1000 };
-
-  Clock* const clock_;
-  VCMProtectionCallback* const protection_callback_;
-
-  rtc::CriticalSection crit_sect_;
-  std::unique_ptr<media_optimization::VCMLossProtectionLogic> loss_prot_logic_
-      RTC_GUARDED_BY(crit_sect_);
-  size_t max_payload_size_ RTC_GUARDED_BY(crit_sect_);
-
-  RTC_DISALLOW_COPY_AND_ASSIGN(ProtectionBitrateCalculator);
 };
 
 }  // namespace webrtc
+
 #endif  // MODULES_VIDEO_CODING_PROTECTION_BITRATE_CALCULATOR_H_

modules/video_coding/protection_bitrate_calculator_default.cc

@@ -8,13 +8,13 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-#include "modules/video_coding/protection_bitrate_calculator.h"
+#include "modules/video_coding/protection_bitrate_calculator_default.h"
 
 namespace webrtc {
 
 using rtc::CritScope;
 
-ProtectionBitrateCalculator::ProtectionBitrateCalculator(
+ProtectionBitrateCalculatorDefault::ProtectionBitrateCalculatorDefault(
     Clock* clock,
     VCMProtectionCallback* protection_callback)
     : clock_(clock),
@@ -23,21 +23,22 @@ ProtectionBitrateCalculator::ProtectionBitrateCalculator(
           clock_->TimeInMilliseconds())),
       max_payload_size_(1460) {}
 
-ProtectionBitrateCalculator::~ProtectionBitrateCalculator(void) {
+ProtectionBitrateCalculatorDefault::~ProtectionBitrateCalculatorDefault(void) {
   loss_prot_logic_->Release();
 }
 
-void ProtectionBitrateCalculator::SetEncodingData(size_t width,
+void ProtectionBitrateCalculatorDefault::SetEncodingData(
-                                                  size_t height,
+    size_t width,
-                                                  size_t num_temporal_layers,
+    size_t height,
-                                                  size_t max_payload_size) {
+    size_t num_temporal_layers,
+    size_t max_payload_size) {
   CritScope lock(&crit_sect_);
   loss_prot_logic_->UpdateFrameSize(width, height);
   loss_prot_logic_->UpdateNumLayers(num_temporal_layers);
   max_payload_size_ = max_payload_size;
 }
 
-uint32_t ProtectionBitrateCalculator::SetTargetRates(
+uint32_t ProtectionBitrateCalculatorDefault::SetTargetRates(
     uint32_t estimated_bitrate_bps,
     int actual_framerate_fps,
     uint8_t fraction_lost,
@@ -138,8 +139,8 @@ uint32_t ProtectionBitrateCalculator::SetTargetRates(
   return estimated_bitrate_bps * (1.0 - protection_overhead_rate);
 }
 
-void ProtectionBitrateCalculator::SetProtectionMethod(bool enable_fec,
+void ProtectionBitrateCalculatorDefault::SetProtectionMethod(bool enable_fec,
-                                                      bool enable_nack) {
+                                                             bool enable_nack) {
   media_optimization::VCMProtectionMethodEnum method(media_optimization::kNone);
   if (enable_fec && enable_nack) {
     method = media_optimization::kNackFec;
@@ -152,7 +153,7 @@ void ProtectionBitrateCalculator::SetProtectionMethod(bool enable_fec,
   loss_prot_logic_->SetMethod(method);
 }
 
-void ProtectionBitrateCalculator::UpdateWithEncodedData(
+void ProtectionBitrateCalculatorDefault::UpdateWithEncodedData(
     const EncodedImage& encoded_image) {
   const size_t encoded_length = encoded_image._length;
   CritScope lock(&crit_sect_);

modules/video_coding/protection_bitrate_calculator_default.h

@@ -0,0 +1,87 @@
+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_PROTECTION_BITRATE_CALCULATOR_DEFAULT_H_
+#define MODULES_VIDEO_CODING_PROTECTION_BITRATE_CALCULATOR_DEFAULT_H_
+
+#include <list>
+#include <memory>
+#include <vector>
+
+#include "modules/include/module_common_types.h"
+#include "modules/video_coding/include/video_coding.h"
+#include "modules/video_coding/media_opt_util.h"
+#include "modules/video_coding/protection_bitrate_calculator.h"
+#include "rtc_base/criticalsection.h"
+#include "system_wrappers/include/clock.h"
+
+namespace webrtc {
+
+// ProtectionBitrateCalculator calculates how much of the allocated network
+// capacity that can be used by an encoder and how much that
+// is needed for redundant packets such as FEC and NACK. It uses an
+// implementation of |VCMProtectionCallback| to set new FEC parameters and get
+// the bitrate currently used for FEC and NACK.
+// Usage:
+// Setup by calling SetProtectionMethod and SetEncodingData.
+// For each encoded image, call UpdateWithEncodedData.
+// Each time the bandwidth estimate change, call SetTargetRates. SetTargetRates
+// will return the bitrate that can be used by an encoder.
+// A lock is used to protect internal states, so methods can be called on an
+// arbitrary thread.
+class ProtectionBitrateCalculatorDefault : public ProtectionBitrateCalculator {
+ public:
+  ProtectionBitrateCalculatorDefault(
+      Clock* clock,
+      VCMProtectionCallback* protection_callback);
+  ~ProtectionBitrateCalculatorDefault();
+
+  void SetProtectionMethod(bool enable_fec, bool enable_nack) override;
+
+  // Informs media optimization of initial encoding state.
+  void SetEncodingData(size_t width,
+                       size_t height,
+                       size_t num_temporal_layers,
+                       size_t max_payload_size) override;
+
+  uint32_t SetTargetRates(uint32_t estimated_bitrate_bps,
+                          std::vector<uint8_t> loss_mask_vector,
+                          int64_t round_trip_time_ms) override {
+    return 0;
+  }
+
+  void OnLossMaskVector(const std::vector<bool> loss_mask_vector) override {}
+
+  // Returns target rate for the encoder given the channel parameters.
+  // Inputs:  estimated_bitrate_bps - the estimated network bitrate in bits/s.
+  //          actual_framerate - encoder frame rate.
+  //          fraction_lost - packet loss rate in % in the network.
+  //          round_trip_time_ms - round trip time in milliseconds.
+  uint32_t SetTargetRates(uint32_t estimated_bitrate_bps,
+                          int actual_framerate,
+                          uint8_t fraction_lost,
+                          int64_t round_trip_time_ms) override;
+  // Informs of encoded output.
+  void UpdateWithEncodedData(const EncodedImage& encoded_image) override;
+
+ private:
+  enum { kBitrateAverageWinMs = 1000 };
+  Clock* const clock_;
+  VCMProtectionCallback* const protection_callback_;
+  rtc::CriticalSection crit_sect_;
+  std::unique_ptr<media_optimization::VCMLossProtectionLogic> loss_prot_logic_
+      RTC_GUARDED_BY(crit_sect_);
+  size_t max_payload_size_ RTC_GUARDED_BY(crit_sect_);
+
+  RTC_DISALLOW_COPY_AND_ASSIGN(ProtectionBitrateCalculatorDefault);
+};
+
+}  // namespace webrtc
+#endif  // MODULES_VIDEO_CODING_PROTECTION_BITRATE_CALCULATOR_DEFAULT_H_

modules/video_coding/protection_bitrate_calculator_unittest.cc

@@ -8,7 +8,7 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-#include "modules/video_coding/protection_bitrate_calculator.h"
+#include "modules/video_coding/protection_bitrate_calculator_default.h"
 #include "system_wrappers/include/clock.h"
 #include "test/gtest.h"
 
@@ -46,7 +46,7 @@ class ProtectionBitrateCalculatorTest : public ::testing::Test {
 
   SimulatedClock clock_;
   ProtectionCallback protection_callback_;
-  ProtectionBitrateCalculator media_opt_;
+  ProtectionBitrateCalculatorDefault media_opt_;
 };
 
 TEST_F(ProtectionBitrateCalculatorTest, ProtectsUsingFecBitrate) {

video/video_send_stream.cc

@@ -11,12 +11,14 @@
 
 #include <algorithm>
 #include <cmath>
+#include <set>
 #include <sstream>
 #include <string>
 #include <utility>
 #include <vector>
 
 #include "call/rtp_transport_controller_send_interface.h"
+#include "call/video_send_stream.h"
 #include "common_types.h"  // NOLINT(build/include)
 #include "common_video/include/video_bitrate_allocator.h"
 #include "modules/bitrate_controller/include/bitrate_controller.h"
@@ -26,6 +28,7 @@
 #include "modules/rtp_rtcp/include/rtp_rtcp.h"
 #include "modules/rtp_rtcp/source/rtp_sender.h"
 #include "modules/utility/include/process_thread.h"
+#include "modules/video_coding/protection_bitrate_calculator_default.h"
 #include "modules/video_coding/utility/ivf_file_writer.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/file.h"
@@ -36,11 +39,12 @@
 #include "system_wrappers/include/field_trial.h"
 #include "video/call_stats.h"
 #include "video/payload_router.h"
-#include "call/video_send_stream.h"
 
 namespace webrtc {
 
 static const int kMinSendSidePacketHistorySize = 600;
+static const int kSendSideSeqNumSetMaxSize = 15000;
+
 namespace {
 
 // We don't do MTU discovery, so assume that we have the standard ethernet MTU.
@@ -244,7 +248,8 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
                             public webrtc::OverheadObserver,
                             public webrtc::VCMProtectionCallback,
                             public VideoStreamEncoder::EncoderSink,
-                            public VideoBitrateAllocationObserver {
+                            public VideoBitrateAllocationObserver,
+                            public webrtc::PacketFeedbackObserver {
  public:
   VideoSendStreamImpl(
       SendStatisticsProxy* stats_proxy,
@@ -283,6 +288,11 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
 
   void SetTransportOverhead(size_t transport_overhead_per_packet);
 
+  // From PacketFeedbackObserver.
+  void OnPacketAdded(uint32_t ssrc, uint16_t seq_num) override;
+  void OnPacketFeedbackVector(
+      const std::vector<PacketFeedback>& packet_feedback_vector) override;
+
  private:
   class CheckEncoderActivityTask;
   class EncoderReconfiguredTask;
@@ -323,7 +333,6 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
   void SignalEncoderActive();
 
   const bool send_side_bwe_with_overhead_;
-
   SendStatisticsProxy* const stats_proxy_;
   const VideoSendStream::Config* const config_;
   std::map<uint32_t, RtpState> suspended_ssrcs_;
@@ -354,7 +363,8 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
 
   VideoStreamEncoder* const video_stream_encoder_;
   EncoderRtcpFeedback encoder_feedback_;
-  ProtectionBitrateCalculator protection_bitrate_calculator_;
+  const std::unique_ptr<ProtectionBitrateCalculator>
+      protection_bitrate_calculator_;
 
   RtcpBandwidthObserver* const bandwidth_observer_;
   // RtpRtcp modules, declared here as they use other members on construction.
@@ -373,6 +383,10 @@ class VideoSendStreamImpl : public webrtc::BitrateAllocatorObserver,
   size_t overhead_bytes_per_packet_
       RTC_GUARDED_BY(overhead_bytes_per_packet_crit_);
   size_t transport_overhead_bytes_per_packet_;
+
+  rtc::CriticalSection feedback_packet_seq_num_set_cs_;
+  std::set<uint16_t> feedback_packet_seq_num_set_
+      RTC_GUARDED_BY(&feedback_packet_seq_num_set_cs_);
 };
 
 // TODO(tommi): See if there's a more elegant way to create a task that creates
@@ -585,7 +599,6 @@ VideoSendStream::VideoSendStream(
     // Only signal target bitrate for screenshare streams, for now.
     video_stream_encoder_->SetBitrateObserver(send_stream_.get());
   }
-
   ReconfigureVideoEncoder(std::move(encoder_config));
 }
 
@@ -714,7 +727,10 @@ VideoSendStreamImpl::VideoSendStreamImpl(
       encoder_feedback_(Clock::GetRealTimeClock(),
                         config_->rtp.ssrcs,
                         video_stream_encoder),
-      protection_bitrate_calculator_(Clock::GetRealTimeClock(), this),
+      protection_bitrate_calculator_(
+          rtc::MakeUnique<ProtectionBitrateCalculatorDefault>(
+              Clock::GetRealTimeClock(),
+              this)),
       bandwidth_observer_(transport->send_side_cc()->GetBandwidthObserver()),
       rtp_rtcp_modules_(CreateRtpRtcpModules(
           config_->send_transport,
@@ -811,6 +827,8 @@ VideoSendStreamImpl::VideoSendStreamImpl(
         config_->encoder_settings.payload_type,
         config_->encoder_settings.payload_name.c_str());
   }
+  // Signal congestion controller this object is ready for OnPacket* callbacks.
+  transport_->send_side_cc()->RegisterPacketFeedbackObserver(this);
 
   RTC_DCHECK(config_->encoder_settings.encoder);
   RTC_DCHECK_GE(config_->encoder_settings.payload_type, 0);
@@ -854,7 +872,7 @@ VideoSendStreamImpl::~VideoSendStreamImpl() {
   RTC_DCHECK(!payload_router_.IsActive())
       << "VideoSendStreamImpl::Stop not called";
   RTC_LOG(LS_INFO) << "~VideoSendStreamInternal: " << config_->ToString();
-
+  transport_->send_side_cc()->DeRegisterPacketFeedbackObserver(this);
   for (RtpRtcp* rtp_rtcp : rtp_rtcp_modules_) {
     transport_->packet_router()->RemoveSendRtpModule(rtp_rtcp);
     delete rtp_rtcp;
@@ -966,7 +984,7 @@ void VideoSendStreamImpl::OnEncoderConfigurationChanged(
 
   size_t number_of_temporal_layers =
       streams.back().temporal_layer_thresholds_bps.size() + 1;
-  protection_bitrate_calculator_.SetEncodingData(
+  protection_bitrate_calculator_->SetEncodingData(
       streams[0].width, streams[0].height, number_of_temporal_layers,
       config_->rtp.max_packet_size);
 
@@ -1002,7 +1020,7 @@ EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage(
       check_encoder_activity_task_->UpdateEncoderActivity();
   }
 
-  protection_bitrate_calculator_.UpdateWithEncodedData(encoded_image);
+  protection_bitrate_calculator_->UpdateWithEncodedData(encoded_image);
   EncodedImageCallback::Result result = payload_router_.OnEncodedImage(
       encoded_image, codec_specific_info, fragmentation);
 
@@ -1108,7 +1126,7 @@ void VideoSendStreamImpl::ConfigureProtection() {
 
   // Currently, both ULPFEC and FlexFEC use the same FEC rate calculation logic,
   // so enable that logic if either of those FEC schemes are enabled.
-  protection_bitrate_calculator_.SetProtectionMethod(
+  protection_bitrate_calculator_->SetProtectionMethod(
       flexfec_enabled || IsUlpfecEnabled(), nack_enabled);
 }
 
@@ -1216,7 +1234,7 @@ uint32_t VideoSendStreamImpl::OnBitrateUpdated(uint32_t bitrate_bps,
 
   // Get the encoder target rate. It is the estimated network rate -
   // protection overhead.
-  encoder_target_rate_bps_ = protection_bitrate_calculator_.SetTargetRates(
+  encoder_target_rate_bps_ = protection_bitrate_calculator_->SetTargetRates(
       payload_bitrate_bps, stats_proxy_->GetSendFrameRate(), fraction_loss,
       rtt);
 
@@ -1317,5 +1335,34 @@ void VideoSendStreamImpl::SetTransportOverhead(
   }
 }
 
+void VideoSendStreamImpl::OnPacketAdded(uint32_t ssrc, uint16_t seq_num) {
+  const auto& ssrcs = config_->rtp.ssrcs;
+  if (std::find(ssrcs.begin(), ssrcs.end(), ssrc) != ssrcs.end()) {
+    rtc::CritScope lock(&feedback_packet_seq_num_set_cs_);
+    feedback_packet_seq_num_set_.insert(seq_num);
+    if (feedback_packet_seq_num_set_.size() > kSendSideSeqNumSetMaxSize) {
+      feedback_packet_seq_num_set_.erase(feedback_packet_seq_num_set_.begin());
+    }
+  }
+}
+
+void VideoSendStreamImpl::OnPacketFeedbackVector(
+    const std::vector<PacketFeedback>& packet_feedback_vector) {
+  // Lost feedbacks are not considered to be lost packets.
+  std::vector<bool> loss_mask_vector;
+  rtc::CritScope lock(&feedback_packet_seq_num_set_cs_);
+  for (const PacketFeedback& packet : packet_feedback_vector) {
+    if (auto it = feedback_packet_seq_num_set_.find(packet.sequence_number) !=
+                  feedback_packet_seq_num_set_.end()) {
+      const bool lost = packet.arrival_time_ms == PacketFeedback::kNotReceived;
+      loss_mask_vector.push_back(lost);
+      feedback_packet_seq_num_set_.erase(it);
+    }
+  }
+  if (loss_mask_vector.empty())
+    return;
+  protection_bitrate_calculator_->OnLossMaskVector(loss_mask_vector);
+}
+
 }  // namespace internal
 }  // namespace webrtc

video/video_send_stream.h

@@ -19,7 +19,6 @@
 #include "call/video_receive_stream.h"
 #include "call/video_send_stream.h"
 #include "common_video/libyuv/include/webrtc_libyuv.h"
-#include "modules/video_coding/protection_bitrate_calculator.h"
 #include "rtc_base/criticalsection.h"
 #include "rtc_base/event.h"
 #include "rtc_base/task_queue.h"