Introduce DecodedFramesHistory class and use it in FrameBuffer

This is a space efficient way to store more records about decoded frames,
which is needed for long term references.

Bug: webrtc:9710
Change-Id: I051d59d34a966d48db011142466d9cd15304b7d9
Reviewed-on: https://webrtc-review.googlesource.com/c/116792
Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26240}

modules/video_coding/utility/decoded_frames_history.cc

@@ -0,0 +1,108 @@
+/*
+ *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/utility/decoded_frames_history.h"
+
+#include <algorithm>
+
+#include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
+
+namespace webrtc {
+namespace video_coding {
+
+DecodedFramesHistory::LayerHistory::LayerHistory() = default;
+DecodedFramesHistory::LayerHistory::~LayerHistory() = default;
+
+DecodedFramesHistory::DecodedFramesHistory(size_t window_size)
+    : window_size_(window_size) {}
+
+DecodedFramesHistory::~DecodedFramesHistory() = default;
+
+void DecodedFramesHistory::InsertDecoded(const VideoLayerFrameId& frameid,
+                                         uint32_t timestamp) {
+  last_decoded_frame_ = frameid;
+  last_decoded_frame_timestamp_ = timestamp;
+  if (static_cast<int>(layers_.size()) < frameid.spatial_layer + 1) {
+    layers_.resize(frameid.spatial_layer + 1);
+    layers_[frameid.spatial_layer].buffer.resize(window_size_);
+    layers_[frameid.spatial_layer].last_stored_index = frameid.picture_id;
+    layers_[frameid.spatial_layer].buffer[frameid.picture_id % window_size_] =
+        true;
+    return;
+  }
+
+  LayerHistory& history = layers_[frameid.spatial_layer];
+
+  RTC_DCHECK_LT(history.last_stored_index, frameid.picture_id);
+
+  int64_t id_jump = frameid.picture_id - history.last_stored_index;
+  int last_index = history.last_stored_index % window_size_;
+  int new_index = frameid.picture_id % window_size_;
+
+  // Need to clear indexes between last_index + 1 and new_index as they fall
+  // out of the history window and now represent new unseen picture ids.
+  if (id_jump >= window_size_) {
+    // Wraps around whole buffer - clear it all.
+    std::fill(history.buffer.begin(), history.buffer.end(), false);
+  } else if (new_index > last_index) {
+    std::fill(history.buffer.begin() + last_index + 1,
+              history.buffer.begin() + new_index, false);
+  } else {
+    std::fill(history.buffer.begin() + last_index + 1, history.buffer.end(),
+              false);
+    std::fill(history.buffer.begin(), history.buffer.begin() + new_index,
+              false);
+  }
+
+  history.buffer[new_index] = true;
+  history.last_stored_index = frameid.picture_id;
+}
+
+bool DecodedFramesHistory::WasDecoded(const VideoLayerFrameId& frameid) {
+  // Unseen before spatial layer.
+  if (static_cast<int>(layers_.size()) < frameid.spatial_layer + 1)
+    return false;
+
+  LayerHistory& history = layers_[frameid.spatial_layer];
+
+  // Reference to the picture_id out of the stored history should happen.
+  if (frameid.picture_id <= history.last_stored_index - window_size_) {
+    RTC_LOG(LS_WARNING) << "Referencing a frame out of the history window. "
+                           "Assuming it was undecoded to avoid artifacts.";
+    return false;
+  }
+
+  if (frameid.picture_id > history.last_stored_index)
+    return false;
+
+  return history.buffer[frameid.picture_id % window_size_];
+}
+
+void DecodedFramesHistory::Clear() {
+  for (LayerHistory& layer : layers_) {
+    std::fill(layer.buffer.begin(), layer.buffer.end(), false);
+    layer.last_stored_index = 0;
+  }
+  last_decoded_frame_timestamp_.reset();
+  last_decoded_frame_.reset();
+}
+
+absl::optional<VideoLayerFrameId>
+DecodedFramesHistory::GetLastDecodedFrameId() {
+  return last_decoded_frame_;
+}
+
+absl::optional<uint32_t> DecodedFramesHistory::GetLastDecodedFrameTimestamp() {
+  return last_decoded_frame_timestamp_;
+}
+
+}  // namespace video_coding
+}  // namespace webrtc

modules/video_coding/utility/decoded_frames_history.h

@@ -0,0 +1,60 @@
+/*
+ *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_UTILITY_DECODED_FRAMES_HISTORY_H_
+#define MODULES_VIDEO_CODING_UTILITY_DECODED_FRAMES_HISTORY_H_
+
+#include <stdint.h>
+#include <bitset>
+#include <vector>
+
+#include "absl/types/optional.h"
+#include "api/video/encoded_frame.h"
+
+namespace webrtc {
+namespace video_coding {
+
+class DecodedFramesHistory {
+ public:
+  // window_size - how much frames back to the past are actually remembered.
+  explicit DecodedFramesHistory(size_t window_size);
+  ~DecodedFramesHistory();
+  // Called for each decoded frame. Assumes picture id's are non-decreasing.
+  void InsertDecoded(const VideoLayerFrameId& frameid, uint32_t timestamp);
+  // Query if the following (picture_id, spatial_id) pair was inserted before.
+  // Should be at most less by window_size-1 than the last inserted picture id.
+  bool WasDecoded(const VideoLayerFrameId& frameid);
+
+  void Clear();
+
+  absl::optional<VideoLayerFrameId> GetLastDecodedFrameId();
+  absl::optional<uint32_t> GetLastDecodedFrameTimestamp();
+
+ private:
+  struct LayerHistory {
+    LayerHistory();
+    ~LayerHistory();
+    // Cyclic bitset buffer. Stores last known |window_size| bits.
+    // last_stored_index is the last actually stored bit. Previous
+    // |window_size-1| bits are also in the memory. Value for i-th bit is at
+    // buffer[i % window_size].
+    std::vector<bool> buffer;
+    int64_t last_stored_index;
+  };
+
+  const int window_size_;
+  std::vector<LayerHistory> layers_;
+  absl::optional<VideoLayerFrameId> last_decoded_frame_;
+  absl::optional<uint32_t> last_decoded_frame_timestamp_;
+};
+
+}  // namespace video_coding
+}  // namespace webrtc
+#endif  // MODULES_VIDEO_CODING_UTILITY_DECODED_FRAMES_HISTORY_H_

modules/video_coding/utility/decoded_frames_history_unittest.cc

@@ -0,0 +1,116 @@
+/*
+ *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/utility/decoded_frames_history.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+namespace video_coding {
+namespace {
+
+constexpr int kHistorySize = 1 << 13;
+
+TEST(DecodedFramesHistory, RequestOnEmptyHistory) {
+  DecodedFramesHistory history(kHistorySize);
+  EXPECT_EQ(history.WasDecoded({1234, 0}), false);
+}
+
+TEST(DecodedFramesHistory, FindsLastDecodedFrame) {
+  DecodedFramesHistory history(kHistorySize);
+  history.InsertDecoded({1234, 0}, 0);
+  EXPECT_EQ(history.WasDecoded({1234, 0}), true);
+}
+
+TEST(DecodedFramesHistory, FindsPreviousFrame) {
+  DecodedFramesHistory history(kHistorySize);
+  history.InsertDecoded({1234, 0}, 0);
+  history.InsertDecoded({1235, 0}, 0);
+  EXPECT_EQ(history.WasDecoded({1234, 0}), true);
+}
+
+TEST(DecodedFramesHistory, ReportsMissingFrame) {
+  DecodedFramesHistory history(kHistorySize);
+  history.InsertDecoded({1234, 0}, 0);
+  history.InsertDecoded({1236, 0}, 0);
+  EXPECT_EQ(history.WasDecoded({1235, 0}), false);
+}
+
+TEST(DecodedFramesHistory, ClearsHistory) {
+  DecodedFramesHistory history(kHistorySize);
+  history.InsertDecoded({1234, 0}, 0);
+  history.Clear();
+  EXPECT_EQ(history.WasDecoded({1234, 0}), false);
+  EXPECT_EQ(history.GetLastDecodedFrameId(), absl::nullopt);
+  EXPECT_EQ(history.GetLastDecodedFrameTimestamp(), absl::nullopt);
+}
+
+TEST(DecodedFramesHistory, HandlesMultipleLayers) {
+  DecodedFramesHistory history(kHistorySize);
+  history.InsertDecoded({1234, 0}, 0);
+  history.InsertDecoded({1234, 1}, 0);
+  history.InsertDecoded({1235, 0}, 0);
+  history.InsertDecoded({1236, 0}, 0);
+  history.InsertDecoded({1236, 1}, 0);
+  EXPECT_EQ(history.WasDecoded({1235, 0}), true);
+  EXPECT_EQ(history.WasDecoded({1235, 1}), false);
+}
+
+TEST(DecodedFramesHistory, HandlesNewLayer) {
+  DecodedFramesHistory history(kHistorySize);
+  history.InsertDecoded({1234, 0}, 0);
+  history.InsertDecoded({1234, 1}, 0);
+  history.InsertDecoded({1235, 0}, 0);
+  history.InsertDecoded({1235, 1}, 0);
+  history.InsertDecoded({1236, 0}, 0);
+  history.InsertDecoded({1236, 1}, 0);
+  EXPECT_EQ(history.WasDecoded({1234, 2}), false);
+}
+
+TEST(DecodedFramesHistory, HandlesBigJumpInPictureId) {
+  DecodedFramesHistory history(kHistorySize);
+  history.InsertDecoded({1234, 0}, 0);
+  history.InsertDecoded({1235, 0}, 0);
+  history.InsertDecoded({1236, 0}, 0);
+  history.InsertDecoded({1236 + kHistorySize / 2, 0}, 0);
+  EXPECT_EQ(history.WasDecoded({1234, 0}), true);
+  EXPECT_EQ(history.WasDecoded({1237, 0}), false);
+}
+
+TEST(DecodedFramesHistory, ForgetsTooOldHistory) {
+  DecodedFramesHistory history(kHistorySize);
+  history.InsertDecoded({1234, 0}, 0);
+  history.InsertDecoded({1235, 0}, 0);
+  history.InsertDecoded({1236, 0}, 0);
+  history.InsertDecoded({1236 + kHistorySize * 2, 0}, 0);
+  EXPECT_EQ(history.WasDecoded({1234, 0}), false);
+  EXPECT_EQ(history.WasDecoded({1237, 0}), false);
+}
+
+TEST(DecodedFramesHistory, ReturnsLastDecodedFrameId) {
+  DecodedFramesHistory history(kHistorySize);
+  EXPECT_EQ(history.GetLastDecodedFrameId(), absl::nullopt);
+  history.InsertDecoded({1234, 0}, 0);
+  EXPECT_EQ(history.GetLastDecodedFrameId(), VideoLayerFrameId(1234, 0));
+  history.InsertDecoded({1235, 0}, 0);
+  EXPECT_EQ(history.GetLastDecodedFrameId(), VideoLayerFrameId(1235, 0));
+}
+
+TEST(DecodedFramesHistory, ReturnsLastDecodedFrameTimestamp) {
+  DecodedFramesHistory history(kHistorySize);
+  EXPECT_EQ(history.GetLastDecodedFrameTimestamp(), absl::nullopt);
+  history.InsertDecoded({1234, 0}, 12345);
+  EXPECT_EQ(history.GetLastDecodedFrameTimestamp(), 12345u);
+  history.InsertDecoded({1235, 0}, 12366);
+  EXPECT_EQ(history.GetLastDecodedFrameTimestamp(), 12366u);
+}
+
+}  // namespace
+}  // namespace video_coding
+}  // namespace webrtc