Moving src/webrtc into src/.

In order to eliminate the WebRTC Subtree mirror in Chromium, 
WebRTC is moving the content of the src/webrtc directory up
to the src/ directory.

NOPRESUBMIT=true
NOTREECHECKS=true
NOTRY=true
TBR=tommi@webrtc.org

Bug: chromium:611808
Change-Id: Iac59c5b51b950f174119565bac87955a7994bc38
Reviewed-on: https://webrtc-review.googlesource.com/1560
Commit-Queue: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Henrik Kjellander <kjellander@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#19845}

modules/video_coding/frame_buffer2.cc

@@ -0,0 +1,573 @@
+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/video_coding/frame_buffer2.h"
+
+#include <algorithm>
+#include <cstring>
+#include <queue>
+
+#include "webrtc/modules/video_coding/include/video_coding_defines.h"
+#include "webrtc/modules/video_coding/jitter_estimator.h"
+#include "webrtc/modules/video_coding/timing.h"
+#include "webrtc/rtc_base/checks.h"
+#include "webrtc/rtc_base/logging.h"
+#include "webrtc/rtc_base/trace_event.h"
+#include "webrtc/system_wrappers/include/clock.h"
+#include "webrtc/system_wrappers/include/metrics.h"
+
+namespace webrtc {
+namespace video_coding {
+
+namespace {
+// Max number of frames the buffer will hold.
+constexpr int kMaxFramesBuffered = 600;
+
+// Max number of decoded frame info that will be saved.
+constexpr int kMaxFramesHistory = 50;
+
+constexpr int64_t kLogNonDecodedIntervalMs = 5000;
+}  // namespace
+
+FrameBuffer::FrameBuffer(Clock* clock,
+                         VCMJitterEstimator* jitter_estimator,
+                         VCMTiming* timing,
+                         VCMReceiveStatisticsCallback* stats_callback)
+    : clock_(clock),
+      new_continuous_frame_event_(false, false),
+      jitter_estimator_(jitter_estimator),
+      timing_(timing),
+      inter_frame_delay_(clock_->TimeInMilliseconds()),
+      last_decoded_frame_timestamp_(0),
+      last_decoded_frame_it_(frames_.end()),
+      last_continuous_frame_it_(frames_.end()),
+      num_frames_history_(0),
+      num_frames_buffered_(0),
+      stopped_(false),
+      protection_mode_(kProtectionNack),
+      stats_callback_(stats_callback),
+      last_log_non_decoded_ms_(-kLogNonDecodedIntervalMs) {}
+
+FrameBuffer::~FrameBuffer() {}
+
+FrameBuffer::ReturnReason FrameBuffer::NextFrame(
+    int64_t max_wait_time_ms,
+    std::unique_ptr<FrameObject>* frame_out,
+    bool keyframe_required) {
+  TRACE_EVENT0("webrtc", "FrameBuffer::NextFrame");
+  int64_t latest_return_time_ms =
+      clock_->TimeInMilliseconds() + max_wait_time_ms;
+  int64_t wait_ms = max_wait_time_ms;
+  int64_t now_ms = 0;
+
+  do {
+    now_ms = clock_->TimeInMilliseconds();
+    {
+      rtc::CritScope lock(&crit_);
+      new_continuous_frame_event_.Reset();
+      if (stopped_)
+        return kStopped;
+
+      wait_ms = max_wait_time_ms;
+
+      // Need to hold |crit_| in order to use |frames_|, therefore we
+      // set it here in the loop instead of outside the loop in order to not
+      // acquire the lock unnecesserily.
+      next_frame_it_ = frames_.end();
+
+      // |frame_it| points to the first frame after the
+      // |last_decoded_frame_it_|.
+      auto frame_it = frames_.end();
+      if (last_decoded_frame_it_ == frames_.end()) {
+        frame_it = frames_.begin();
+      } else {
+        frame_it = last_decoded_frame_it_;
+        ++frame_it;
+      }
+
+      // |continuous_end_it| points to the first frame after the
+      // |last_continuous_frame_it_|.
+      auto continuous_end_it = last_continuous_frame_it_;
+      if (continuous_end_it != frames_.end())
+        ++continuous_end_it;
+
+      for (; frame_it != continuous_end_it && frame_it != frames_.end();
+           ++frame_it) {
+        if (!frame_it->second.continuous ||
+            frame_it->second.num_missing_decodable > 0) {
+          continue;
+        }
+
+        FrameObject* frame = frame_it->second.frame.get();
+
+        if (keyframe_required && !frame->is_keyframe())
+          continue;
+
+        next_frame_it_ = frame_it;
+        if (frame->RenderTime() == -1)
+          frame->SetRenderTime(timing_->RenderTimeMs(frame->timestamp, now_ms));
+        wait_ms = timing_->MaxWaitingTime(frame->RenderTime(), now_ms);
+
+        // This will cause the frame buffer to prefer high framerate rather
+        // than high resolution in the case of the decoder not decoding fast
+        // enough and the stream has multiple spatial and temporal layers.
+        if (wait_ms == 0)
+          continue;
+
+        break;
+      }
+    }  // rtc::Critscope lock(&crit_);
+
+    wait_ms = std::min<int64_t>(wait_ms, latest_return_time_ms - now_ms);
+    wait_ms = std::max<int64_t>(wait_ms, 0);
+  } while (new_continuous_frame_event_.Wait(wait_ms));
+
+  {
+    rtc::CritScope lock(&crit_);
+    now_ms = clock_->TimeInMilliseconds();
+    if (next_frame_it_ != frames_.end()) {
+      std::unique_ptr<FrameObject> frame =
+          std::move(next_frame_it_->second.frame);
+
+      if (!frame->delayed_by_retransmission()) {
+        int64_t frame_delay;
+
+        if (inter_frame_delay_.CalculateDelay(frame->timestamp, &frame_delay,
+                                              frame->ReceivedTime())) {
+          jitter_estimator_->UpdateEstimate(frame_delay, frame->size());
+        }
+
+        float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0;
+        timing_->SetJitterDelay(jitter_estimator_->GetJitterEstimate(rtt_mult));
+        timing_->UpdateCurrentDelay(frame->RenderTime(), now_ms);
+      }
+
+      // Gracefully handle bad RTP timestamps and render time issues.
+      if (HasBadRenderTiming(*frame, now_ms)) {
+        jitter_estimator_->Reset();
+        timing_->Reset();
+        frame->SetRenderTime(timing_->RenderTimeMs(frame->timestamp, now_ms));
+      }
+
+      UpdateJitterDelay();
+      UpdateTimingFrameInfo();
+      PropagateDecodability(next_frame_it_->second);
+
+      // Sanity check for RTP timestamp monotonicity.
+      if (last_decoded_frame_it_ != frames_.end()) {
+        const FrameKey& last_decoded_frame_key = last_decoded_frame_it_->first;
+        const FrameKey& frame_key = next_frame_it_->first;
+
+        const bool frame_is_higher_spatial_layer_of_last_decoded_frame =
+            last_decoded_frame_timestamp_ == frame->timestamp &&
+            last_decoded_frame_key.picture_id == frame_key.picture_id &&
+            last_decoded_frame_key.spatial_layer < frame_key.spatial_layer;
+
+        if (AheadOrAt(last_decoded_frame_timestamp_, frame->timestamp) &&
+            !frame_is_higher_spatial_layer_of_last_decoded_frame) {
+          // TODO(brandtr): Consider clearing the entire buffer when we hit
+          // these conditions.
+          LOG(LS_WARNING) << "Frame with (timestamp:picture_id:spatial_id) ("
+                          << frame->timestamp << ":" << frame->picture_id << ":"
+                          << static_cast<int>(frame->spatial_layer) << ")"
+                          << " sent to decoder after frame with"
+                          << " (timestamp:picture_id:spatial_id) ("
+                          << last_decoded_frame_timestamp_ << ":"
+                          << last_decoded_frame_key.picture_id << ":"
+                          << static_cast<int>(
+                                 last_decoded_frame_key.spatial_layer)
+                          << ").";
+        }
+      }
+
+      AdvanceLastDecodedFrame(next_frame_it_);
+      last_decoded_frame_timestamp_ = frame->timestamp;
+      *frame_out = std::move(frame);
+      return kFrameFound;
+    }
+  }
+
+  if (latest_return_time_ms - now_ms > 0) {
+    // If |next_frame_it_ == frames_.end()| and there is still time left, it
+    // means that the frame buffer was cleared as the thread in this function
+    // was waiting to acquire |crit_| in order to return. Wait for the
+    // remaining time and then return.
+    return NextFrame(latest_return_time_ms - now_ms, frame_out);
+  }
+
+  return kTimeout;
+}
+
+bool FrameBuffer::HasBadRenderTiming(const FrameObject& frame, int64_t now_ms) {
+  // Assume that render timing errors are due to changes in the video stream.
+  int64_t render_time_ms = frame.RenderTimeMs();
+  const int64_t kMaxVideoDelayMs = 10000;
+  if (render_time_ms < 0) {
+    return true;
+  }
+  if (std::abs(render_time_ms - now_ms) > kMaxVideoDelayMs) {
+    int frame_delay = static_cast<int>(std::abs(render_time_ms - now_ms));
+    LOG(LS_WARNING) << "A frame about to be decoded is out of the configured "
+                    << "delay bounds (" << frame_delay << " > "
+                    << kMaxVideoDelayMs
+                    << "). Resetting the video jitter buffer.";
+    return true;
+  }
+  if (static_cast<int>(timing_->TargetVideoDelay()) > kMaxVideoDelayMs) {
+    LOG(LS_WARNING) << "The video target delay has grown larger than "
+                    << kMaxVideoDelayMs << " ms.";
+    return true;
+  }
+  return false;
+}
+
+void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) {
+  TRACE_EVENT0("webrtc", "FrameBuffer::SetProtectionMode");
+  rtc::CritScope lock(&crit_);
+  protection_mode_ = mode;
+}
+
+void FrameBuffer::Start() {
+  TRACE_EVENT0("webrtc", "FrameBuffer::Start");
+  rtc::CritScope lock(&crit_);
+  stopped_ = false;
+}
+
+void FrameBuffer::Stop() {
+  TRACE_EVENT0("webrtc", "FrameBuffer::Stop");
+  rtc::CritScope lock(&crit_);
+  stopped_ = true;
+  new_continuous_frame_event_.Set();
+}
+
+bool FrameBuffer::ValidReferences(const FrameObject& frame) const {
+  if (frame.picture_id < 0)
+    return false;
+
+  for (size_t i = 0; i < frame.num_references; ++i) {
+    if (frame.references[i] < 0 || frame.references[i] >= frame.picture_id)
+      return false;
+
+    for (size_t j = i + 1; j < frame.num_references; ++j) {
+      if (frame.references[i] == frame.references[j])
+        return false;
+    }
+  }
+
+  if (frame.inter_layer_predicted && frame.spatial_layer == 0)
+    return false;
+
+  return true;
+}
+
+void FrameBuffer::UpdatePlayoutDelays(const FrameObject& frame) {
+  TRACE_EVENT0("webrtc", "FrameBuffer::UpdatePlayoutDelays");
+  PlayoutDelay playout_delay = frame.EncodedImage().playout_delay_;
+  if (playout_delay.min_ms >= 0)
+    timing_->set_min_playout_delay(playout_delay.min_ms);
+
+  if (playout_delay.max_ms >= 0)
+    timing_->set_max_playout_delay(playout_delay.max_ms);
+}
+
+int FrameBuffer::InsertFrame(std::unique_ptr<FrameObject> frame) {
+  TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame");
+  RTC_DCHECK(frame);
+  if (stats_callback_)
+    stats_callback_->OnCompleteFrame(frame->is_keyframe(), frame->size(),
+                                     frame->contentType());
+  FrameKey key(frame->picture_id, frame->spatial_layer);
+
+  rtc::CritScope lock(&crit_);
+
+  int last_continuous_picture_id =
+      last_continuous_frame_it_ == frames_.end()
+          ? -1
+          : last_continuous_frame_it_->first.picture_id;
+
+  if (!ValidReferences(*frame)) {
+    LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" << key.picture_id
+                    << ":" << static_cast<int>(key.spatial_layer)
+                    << ") has invalid frame references, dropping frame.";
+    return last_continuous_picture_id;
+  }
+
+  if (num_frames_buffered_ >= kMaxFramesBuffered) {
+    LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" << key.picture_id
+                    << ":" << static_cast<int>(key.spatial_layer)
+                    << ") could not be inserted due to the frame "
+                    << "buffer being full, dropping frame.";
+    return last_continuous_picture_id;
+  }
+
+  if (last_decoded_frame_it_ != frames_.end() &&
+      key <= last_decoded_frame_it_->first) {
+    if (AheadOf(frame->timestamp, last_decoded_frame_timestamp_) &&
+        frame->is_keyframe()) {
+      // If this frame has a newer timestamp but an earlier picture id then we
+      // assume there has been a jump in the picture id due to some encoder
+      // reconfiguration or some other reason. Even though this is not according
+      // to spec we can still continue to decode from this frame if it is a
+      // keyframe.
+      LOG(LS_WARNING) << "A jump in picture id was detected, clearing buffer.";
+      ClearFramesAndHistory();
+      last_continuous_picture_id = -1;
+    } else {
+      LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
+                      << key.picture_id << ":"
+                      << static_cast<int>(key.spatial_layer)
+                      << ") inserted after frame ("
+                      << last_decoded_frame_it_->first.picture_id << ":"
+                      << static_cast<int>(
+                             last_decoded_frame_it_->first.spatial_layer)
+                      << ") was handed off for decoding, dropping frame.";
+      return last_continuous_picture_id;
+    }
+  }
+
+  // Test if inserting this frame would cause the order of the frames to become
+  // ambiguous (covering more than half the interval of 2^16). This can happen
+  // when the picture id make large jumps mid stream.
+  if (!frames_.empty() &&
+      key < frames_.begin()->first &&
+      frames_.rbegin()->first < key) {
+    LOG(LS_WARNING) << "A jump in picture id was detected, clearing buffer.";
+    ClearFramesAndHistory();
+    last_continuous_picture_id = -1;
+  }
+
+  auto info = frames_.insert(std::make_pair(key, FrameInfo())).first;
+
+  if (info->second.frame) {
+    LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" << key.picture_id
+                    << ":" << static_cast<int>(key.spatial_layer)
+                    << ") already inserted, dropping frame.";
+    return last_continuous_picture_id;
+  }
+
+  if (!UpdateFrameInfoWithIncomingFrame(*frame, info))
+    return last_continuous_picture_id;
+  UpdatePlayoutDelays(*frame);
+  info->second.frame = std::move(frame);
+  ++num_frames_buffered_;
+
+  if (info->second.num_missing_continuous == 0) {
+    info->second.continuous = true;
+    PropagateContinuity(info);
+    last_continuous_picture_id = last_continuous_frame_it_->first.picture_id;
+
+    // Since we now have new continuous frames there might be a better frame
+    // to return from NextFrame. Signal that thread so that it again can choose
+    // which frame to return.
+    new_continuous_frame_event_.Set();
+  }
+
+  return last_continuous_picture_id;
+}
+
+void FrameBuffer::PropagateContinuity(FrameMap::iterator start) {
+  TRACE_EVENT0("webrtc", "FrameBuffer::PropagateContinuity");
+  RTC_DCHECK(start->second.continuous);
+  if (last_continuous_frame_it_ == frames_.end())
+    last_continuous_frame_it_ = start;
+
+  std::queue<FrameMap::iterator> continuous_frames;
+  continuous_frames.push(start);
+
+  // A simple BFS to traverse continuous frames.
+  while (!continuous_frames.empty()) {
+    auto frame = continuous_frames.front();
+    continuous_frames.pop();
+
+    if (last_continuous_frame_it_->first < frame->first)
+      last_continuous_frame_it_ = frame;
+
+    // Loop through all dependent frames, and if that frame no longer has
+    // any unfulfilled dependencies then that frame is continuous as well.
+    for (size_t d = 0; d < frame->second.num_dependent_frames; ++d) {
+      auto frame_ref = frames_.find(frame->second.dependent_frames[d]);
+      RTC_DCHECK(frame_ref != frames_.end());
+
+      // TODO(philipel): Look into why we've seen this happen.
+      if (frame_ref != frames_.end()) {
+        --frame_ref->second.num_missing_continuous;
+        if (frame_ref->second.num_missing_continuous == 0) {
+          frame_ref->second.continuous = true;
+          continuous_frames.push(frame_ref);
+        }
+      }
+    }
+  }
+}
+
+void FrameBuffer::PropagateDecodability(const FrameInfo& info) {
+  TRACE_EVENT0("webrtc", "FrameBuffer::PropagateDecodability");
+  RTC_CHECK(info.num_dependent_frames < FrameInfo::kMaxNumDependentFrames);
+  for (size_t d = 0; d < info.num_dependent_frames; ++d) {
+    auto ref_info = frames_.find(info.dependent_frames[d]);
+    RTC_DCHECK(ref_info != frames_.end());
+    // TODO(philipel): Look into why we've seen this happen.
+    if (ref_info != frames_.end()) {
+      RTC_DCHECK_GT(ref_info->second.num_missing_decodable, 0U);
+      --ref_info->second.num_missing_decodable;
+    }
+  }
+}
+
+void FrameBuffer::AdvanceLastDecodedFrame(FrameMap::iterator decoded) {
+  TRACE_EVENT0("webrtc", "FrameBuffer::AdvanceLastDecodedFrame");
+  if (last_decoded_frame_it_ == frames_.end()) {
+    last_decoded_frame_it_ = frames_.begin();
+  } else {
+    RTC_DCHECK(last_decoded_frame_it_->first < decoded->first);
+    ++last_decoded_frame_it_;
+  }
+  --num_frames_buffered_;
+  ++num_frames_history_;
+
+  // First, delete non-decoded frames from the history.
+  while (last_decoded_frame_it_ != decoded) {
+    if (last_decoded_frame_it_->second.frame)
+      --num_frames_buffered_;
+    last_decoded_frame_it_ = frames_.erase(last_decoded_frame_it_);
+  }
+
+  // Then remove old history if we have too much history saved.
+  if (num_frames_history_ > kMaxFramesHistory) {
+    frames_.erase(frames_.begin());
+    --num_frames_history_;
+  }
+}
+
+bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const FrameObject& frame,
+                                                   FrameMap::iterator info) {
+  TRACE_EVENT0("webrtc", "FrameBuffer::UpdateFrameInfoWithIncomingFrame");
+  FrameKey key(frame.picture_id, frame.spatial_layer);
+  info->second.num_missing_continuous = frame.num_references;
+  info->second.num_missing_decodable = frame.num_references;
+
+  RTC_DCHECK(last_decoded_frame_it_ == frames_.end() ||
+             last_decoded_frame_it_->first < info->first);
+
+  // Check how many dependencies that have already been fulfilled.
+  for (size_t i = 0; i < frame.num_references; ++i) {
+    FrameKey ref_key(frame.references[i], frame.spatial_layer);
+    auto ref_info = frames_.find(ref_key);
+
+    // Does |frame| depend on a frame earlier than the last decoded frame?
+    if (last_decoded_frame_it_ != frames_.end() &&
+        ref_key <= last_decoded_frame_it_->first) {
+      if (ref_info == frames_.end()) {
+        int64_t now_ms = clock_->TimeInMilliseconds();
+        if (last_log_non_decoded_ms_ + kLogNonDecodedIntervalMs < now_ms) {
+          LOG(LS_WARNING)
+              << "Frame with (picture_id:spatial_id) (" << key.picture_id << ":"
+              << static_cast<int>(key.spatial_layer)
+              << ") depends on a non-decoded frame more previous than"
+              << " the last decoded frame, dropping frame.";
+          last_log_non_decoded_ms_ = now_ms;
+        }
+        return false;
+      }
+
+      --info->second.num_missing_continuous;
+      --info->second.num_missing_decodable;
+    } else {
+      if (ref_info == frames_.end())
+        ref_info = frames_.insert(std::make_pair(ref_key, FrameInfo())).first;
+
+      if (ref_info->second.continuous)
+        --info->second.num_missing_continuous;
+
+      // Add backwards reference so |frame| can be updated when new
+      // frames are inserted or decoded.
+      ref_info->second.dependent_frames[ref_info->second.num_dependent_frames] =
+          key;
+      RTC_DCHECK_LT(ref_info->second.num_dependent_frames,
+                    (FrameInfo::kMaxNumDependentFrames - 1));
+      // TODO(philipel): Look into why this could happen and handle
+      // appropriately.
+      if (ref_info->second.num_dependent_frames <
+          (FrameInfo::kMaxNumDependentFrames - 1)) {
+        ++ref_info->second.num_dependent_frames;
+      }
+    }
+    RTC_DCHECK_LE(ref_info->second.num_missing_continuous,
+                  ref_info->second.num_missing_decodable);
+  }
+
+  // Check if we have the lower spatial layer frame.
+  if (frame.inter_layer_predicted) {
+    ++info->second.num_missing_continuous;
+    ++info->second.num_missing_decodable;
+
+    FrameKey ref_key(frame.picture_id, frame.spatial_layer - 1);
+    // Gets or create the FrameInfo for the referenced frame.
+    auto ref_info = frames_.insert(std::make_pair(ref_key, FrameInfo())).first;
+    if (ref_info->second.continuous)
+      --info->second.num_missing_continuous;
+
+    if (ref_info == last_decoded_frame_it_) {
+      --info->second.num_missing_decodable;
+    } else {
+      ref_info->second.dependent_frames[ref_info->second.num_dependent_frames] =
+          key;
+      ++ref_info->second.num_dependent_frames;
+    }
+    RTC_DCHECK_LE(ref_info->second.num_missing_continuous,
+                  ref_info->second.num_missing_decodable);
+  }
+
+  RTC_DCHECK_LE(info->second.num_missing_continuous,
+                info->second.num_missing_decodable);
+
+  return true;
+}
+
+void FrameBuffer::UpdateJitterDelay() {
+  TRACE_EVENT0("webrtc", "FrameBuffer::UpdateJitterDelay");
+  if (!stats_callback_)
+    return;
+
+  int decode_ms;
+  int max_decode_ms;
+  int current_delay_ms;
+  int target_delay_ms;
+  int jitter_buffer_ms;
+  int min_playout_delay_ms;
+  int render_delay_ms;
+  if (timing_->GetTimings(&decode_ms, &max_decode_ms, &current_delay_ms,
+                          &target_delay_ms, &jitter_buffer_ms,
+                          &min_playout_delay_ms, &render_delay_ms)) {
+    stats_callback_->OnFrameBufferTimingsUpdated(
+        decode_ms, max_decode_ms, current_delay_ms, target_delay_ms,
+        jitter_buffer_ms, min_playout_delay_ms, render_delay_ms);
+  }
+}
+
+void FrameBuffer::UpdateTimingFrameInfo() {
+  TRACE_EVENT0("webrtc", "FrameBuffer::UpdateTimingFrameInfo");
+  rtc::Optional<TimingFrameInfo> info = timing_->GetTimingFrameInfo();
+  if (info)
+    stats_callback_->OnTimingFrameInfoUpdated(*info);
+}
+
+void FrameBuffer::ClearFramesAndHistory() {
+  TRACE_EVENT0("webrtc", "FrameBuffer::ClearFramesAndHistory");
+  frames_.clear();
+  last_decoded_frame_it_ = frames_.end();
+  last_continuous_frame_it_ = frames_.end();
+  next_frame_it_ = frames_.end();
+  num_frames_history_ = 0;
+  num_frames_buffered_ = 0;
+}
+
+}  // namespace video_coding
+}  // namespace webrtc