Move TimestampExtrapolator closer to its single user

The `TimestampExtrapolator` is only used by the `VCMTiming`
class, despite there being references to it from both
`modules/rtp_rtcp/BUILD.gn` and `modules/video_coding/BUILD.gn`.

Bug: webrtc:14111
Change-Id: If1a02a56a0c83b13d619ca08dc76c884fa829369
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/275482
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Johannes Kron <kron@webrtc.org>
Commit-Queue: Rasmus Brandt <brandtr@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38093}

modules/video_coding/BUILD.gn

@@ -292,7 +292,6 @@ rtc_library("video_coding") {
     "../../rtc_base/system:no_unique_address",
     "../../rtc_base/task_utils:repeating_task",
     "../../rtc_base/third_party/base64",
-    "../../rtc_base/time:timestamp_extrapolator",
     "../../system_wrappers",
     "../../system_wrappers:field_trial",
     "../../system_wrappers:metrics",

modules/video_coding/timing/BUILD.gn

@@ -84,6 +84,18 @@ rtc_library("rtt_filter") {
   ]
 }
 
+rtc_library("timestamp_extrapolator") {
+  sources = [
+    "timestamp_extrapolator.cc",
+    "timestamp_extrapolator.h",
+  ]
+  deps = [
+    "../../../api/units:timestamp",
+    "../../../modules:module_api_public",
+  ]
+  absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
+}
+
 rtc_library("timing_module") {
   sources = [
     "timing.cc",
@@ -91,6 +103,7 @@ rtc_library("timing_module") {
   ]
   deps = [
     ":codec_timer",
+    ":timestamp_extrapolator",
     "../../../api:field_trials_view",
     "../../../api/units:time_delta",
     "../../../api/video:video_frame",
@@ -100,7 +113,6 @@ rtc_library("timing_module") {
     "../../../rtc_base:rtc_numerics",
     "../../../rtc_base/experiments:field_trial_parser",
     "../../../rtc_base/synchronization:mutex",
-    "../../../rtc_base/time:timestamp_extrapolator",
     "../../../system_wrappers",
   ]
   absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
@@ -113,6 +125,7 @@ rtc_library("timing_unittests") {
     "inter_frame_delay_unittest.cc",
     "jitter_estimator_unittest.cc",
     "rtt_filter_unittest.cc",
+    "timestamp_extrapolator_unittest.cc",
     "timing_unittest.cc",
   ]
   deps = [
@@ -120,6 +133,7 @@ rtc_library("timing_unittests") {
     ":inter_frame_delay",
     ":jitter_estimator",
     ":rtt_filter",
+    ":timestamp_extrapolator",
     ":timing_module",
     "../../../api:array_view",
     "../../../api:field_trials",

modules/video_coding/timing/timestamp_extrapolator.cc

@@ -0,0 +1,162 @@
+/*
+ *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/timing/timestamp_extrapolator.h"
+
+#include <algorithm>
+
+#include "absl/types/optional.h"
+#include "modules/include/module_common_types_public.h"
+
+namespace webrtc {
+
+namespace {
+
+constexpr double kLambda = 1;
+constexpr uint32_t kStartUpFilterDelayInPackets = 2;
+constexpr double kAlarmThreshold = 60e3;
+// in timestamp ticks, i.e. 15 ms
+constexpr double kAccDrift = 6600;
+constexpr double kAccMaxError = 7000;
+constexpr double kP11 = 1e10;
+
+}  // namespace
+
+TimestampExtrapolator::TimestampExtrapolator(Timestamp start)
+    : start_(Timestamp::Zero()),
+      prev_(Timestamp::Zero()),
+      packet_count_(0),
+      detector_accumulator_pos_(0),
+      detector_accumulator_neg_(0) {
+  Reset(start);
+}
+
+void TimestampExtrapolator::Reset(Timestamp start) {
+  start_ = start;
+  prev_ = start_;
+  first_unwrapped_timestamp_ = absl::nullopt;
+  w_[0] = 90.0;
+  w_[1] = 0;
+  p_[0][0] = 1;
+  p_[1][1] = kP11;
+  p_[0][1] = p_[1][0] = 0;
+  unwrapper_ = TimestampUnwrapper();
+  packet_count_ = 0;
+  detector_accumulator_pos_ = 0;
+  detector_accumulator_neg_ = 0;
+}
+
+void TimestampExtrapolator::Update(Timestamp now, uint32_t ts90khz) {
+  if (now - prev_ > TimeDelta::Seconds(10)) {
+    // Ten seconds without a complete frame.
+    // Reset the extrapolator
+    Reset(now);
+  } else {
+    prev_ = now;
+  }
+
+  // Remove offset to prevent badly scaled matrices
+  const TimeDelta offset = now - start_;
+  double t_ms = offset.ms();
+
+  int64_t unwrapped_ts90khz = unwrapper_.Unwrap(ts90khz);
+
+  if (!first_unwrapped_timestamp_) {
+    // Make an initial guess of the offset,
+    // should be almost correct since t_ms - start
+    // should about zero at this time.
+    w_[1] = -w_[0] * t_ms;
+    first_unwrapped_timestamp_ = unwrapped_ts90khz;
+  }
+
+  double residual =
+      (static_cast<double>(unwrapped_ts90khz) - *first_unwrapped_timestamp_) -
+      t_ms * w_[0] - w_[1];
+  if (DelayChangeDetection(residual) &&
+      packet_count_ >= kStartUpFilterDelayInPackets) {
+    // A sudden change of average network delay has been detected.
+    // Force the filter to adjust its offset parameter by changing
+    // the offset uncertainty. Don't do this during startup.
+    p_[1][1] = kP11;
+  }
+
+  if (prev_unwrapped_timestamp_ &&
+      unwrapped_ts90khz < prev_unwrapped_timestamp_) {
+    // Drop reordered frames.
+    return;
+  }
+
+  // T = [t(k) 1]';
+  // that = T'*w;
+  // K = P*T/(lambda + T'*P*T);
+  double K[2];
+  K[0] = p_[0][0] * t_ms + p_[0][1];
+  K[1] = p_[1][0] * t_ms + p_[1][1];
+  double TPT = kLambda + t_ms * K[0] + K[1];
+  K[0] /= TPT;
+  K[1] /= TPT;
+  // w = w + K*(ts(k) - that);
+  w_[0] = w_[0] + K[0] * residual;
+  w_[1] = w_[1] + K[1] * residual;
+  // P = 1/lambda*(P - K*T'*P);
+  double p00 =
+      1 / kLambda * (p_[0][0] - (K[0] * t_ms * p_[0][0] + K[0] * p_[1][0]));
+  double p01 =
+      1 / kLambda * (p_[0][1] - (K[0] * t_ms * p_[0][1] + K[0] * p_[1][1]));
+  p_[1][0] =
+      1 / kLambda * (p_[1][0] - (K[1] * t_ms * p_[0][0] + K[1] * p_[1][0]));
+  p_[1][1] =
+      1 / kLambda * (p_[1][1] - (K[1] * t_ms * p_[0][1] + K[1] * p_[1][1]));
+  p_[0][0] = p00;
+  p_[0][1] = p01;
+  prev_unwrapped_timestamp_ = unwrapped_ts90khz;
+  if (packet_count_ < kStartUpFilterDelayInPackets) {
+    packet_count_++;
+  }
+}
+
+absl::optional<Timestamp> TimestampExtrapolator::ExtrapolateLocalTime(
+    uint32_t timestamp90khz) const {
+  int64_t unwrapped_ts90khz = unwrapper_.UnwrapWithoutUpdate(timestamp90khz);
+
+  if (!first_unwrapped_timestamp_) {
+    return absl::nullopt;
+  } else if (packet_count_ < kStartUpFilterDelayInPackets) {
+    constexpr double kRtpTicksPerMs = 90;
+    TimeDelta diff = TimeDelta::Millis(
+        (unwrapped_ts90khz - *prev_unwrapped_timestamp_) / kRtpTicksPerMs);
+    return prev_ + diff;
+  } else if (w_[0] < 1e-3) {
+    return start_;
+  } else {
+    double timestampDiff = unwrapped_ts90khz - *first_unwrapped_timestamp_;
+    auto diff_ms = static_cast<int64_t>((timestampDiff - w_[1]) / w_[0] + 0.5);
+    return start_ + TimeDelta::Millis(diff_ms);
+  }
+}
+
+bool TimestampExtrapolator::DelayChangeDetection(double error) {
+  // CUSUM detection of sudden delay changes
+  error = (error > 0) ? std::min(error, kAccMaxError)
+                      : std::max(error, -kAccMaxError);
+  detector_accumulator_pos_ =
+      std::max(detector_accumulator_pos_ + error - kAccDrift, double{0});
+  detector_accumulator_neg_ =
+      std::min(detector_accumulator_neg_ + error + kAccDrift, double{0});
+  if (detector_accumulator_pos_ > kAlarmThreshold ||
+      detector_accumulator_neg_ < -kAlarmThreshold) {
+    // Alarm
+    detector_accumulator_pos_ = detector_accumulator_neg_ = 0;
+    return true;
+  }
+  return false;
+}
+
+}  // namespace webrtc

modules/video_coding/timing/timestamp_extrapolator.h

@@ -0,0 +1,48 @@
+/*
+ *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_TIMING_TIMESTAMP_EXTRAPOLATOR_H_
+#define MODULES_VIDEO_CODING_TIMING_TIMESTAMP_EXTRAPOLATOR_H_
+
+#include <stdint.h>
+
+#include "absl/types/optional.h"
+#include "api/units/timestamp.h"
+#include "modules/include/module_common_types_public.h"
+
+namespace webrtc {
+
+// Not thread safe.
+class TimestampExtrapolator {
+ public:
+  explicit TimestampExtrapolator(Timestamp start);
+  void Update(Timestamp now, uint32_t ts90khz);
+  absl::optional<Timestamp> ExtrapolateLocalTime(uint32_t timestamp90khz) const;
+  void Reset(Timestamp start);
+
+ private:
+  void CheckForWrapArounds(uint32_t ts90khz);
+  bool DelayChangeDetection(double error);
+
+  double w_[2];
+  double p_[2][2];
+  Timestamp start_;
+  Timestamp prev_;
+  absl::optional<int64_t> first_unwrapped_timestamp_;
+  TimestampUnwrapper unwrapper_;
+  absl::optional<int64_t> prev_unwrapped_timestamp_;
+  uint32_t packet_count_;
+  double detector_accumulator_pos_;
+  double detector_accumulator_neg_;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_TIMING_TIMESTAMP_EXTRAPOLATOR_H_

modules/video_coding/timing/timestamp_extrapolator_unittest.cc

@@ -0,0 +1,208 @@
+/*
+ *  Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/timing/timestamp_extrapolator.h"
+
+#include <stdint.h>
+
+#include <limits>
+
+#include "absl/types/optional.h"
+#include "api/units/frequency.h"
+#include "api/units/time_delta.h"
+#include "api/units/timestamp.h"
+#include "system_wrappers/include/clock.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+
+namespace webrtc {
+
+using ::testing::Eq;
+using ::testing::Optional;
+
+namespace {
+
+constexpr Frequency kRtpHz = Frequency::KiloHertz(90);
+constexpr Frequency k25Fps = Frequency::Hertz(25);
+constexpr TimeDelta k25FpsDelay = 1 / k25Fps;
+
+}  // namespace
+
+TEST(TimestampExtrapolatorTest, ExtrapolationOccursAfter2Packets) {
+  SimulatedClock clock(Timestamp::Millis(1337));
+  TimestampExtrapolator ts_extrapolator(clock.CurrentTime());
+
+  // No packets so no timestamp.
+  EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(90000), Eq(absl::nullopt));
+
+  uint32_t rtp = 90000;
+  clock.AdvanceTime(k25FpsDelay);
+  // First result is a bit confusing since it is based off the "start" time,
+  // which is arbitrary.
+  ts_extrapolator.Update(clock.CurrentTime(), rtp);
+  EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+              Optional(clock.CurrentTime()));
+
+  rtp += kRtpHz / k25Fps;
+  clock.AdvanceTime(k25FpsDelay);
+  ts_extrapolator.Update(clock.CurrentTime(), rtp);
+  EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+              Optional(clock.CurrentTime()));
+  EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp + 90000),
+              Optional(clock.CurrentTime() + TimeDelta::Seconds(1)));
+}
+
+TEST(TimestampExtrapolatorTest, ResetsAfter10SecondPause) {
+  SimulatedClock clock(Timestamp::Millis(1337));
+  TimestampExtrapolator ts_extrapolator(clock.CurrentTime());
+
+  uint32_t rtp = 90000;
+  ts_extrapolator.Update(clock.CurrentTime(), rtp);
+  EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+              Optional(clock.CurrentTime()));
+
+  rtp += kRtpHz / k25Fps;
+  clock.AdvanceTime(k25FpsDelay);
+  ts_extrapolator.Update(clock.CurrentTime(), rtp);
+  EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+              Optional(clock.CurrentTime()));
+
+  rtp += 10 * kRtpHz.hertz();
+  clock.AdvanceTime(TimeDelta::Seconds(10) + TimeDelta::Micros(1));
+  ts_extrapolator.Update(clock.CurrentTime(), rtp);
+  EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+              Optional(clock.CurrentTime()));
+}
+
+TEST(TimestampExtrapolatorTest, TimestampExtrapolatesMultipleRtpWrapArounds) {
+  SimulatedClock clock(Timestamp::Millis(1337));
+  TimestampExtrapolator ts_extrapolator(clock.CurrentTime());
+
+  uint32_t rtp = std::numeric_limits<uint32_t>::max();
+  ts_extrapolator.Update(clock.CurrentTime(), rtp);
+  EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+              Optional(clock.CurrentTime()));
+
+  // One overflow. Static cast to avoid undefined behaviour with +=.
+  rtp += static_cast<uint32_t>(kRtpHz / k25Fps);
+  clock.AdvanceTime(k25FpsDelay);
+  ts_extrapolator.Update(clock.CurrentTime(), rtp);
+  EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+              Optional(clock.CurrentTime()));
+
+  // Assert that extrapolation works across the boundary as expected.
+  EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp + 90000),
+              Optional(clock.CurrentTime() + TimeDelta::Seconds(1)));
+  // This is not quite 1s since the math always rounds up.
+  EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp - 90000),
+              Optional(clock.CurrentTime() - TimeDelta::Millis(999)));
+
+  // In order to avoid a wrap arounds reset, add a packet every 10s until we
+  // overflow twice.
+  constexpr TimeDelta kRtpOverflowDelay =
+      std::numeric_limits<uint32_t>::max() / kRtpHz;
+  const Timestamp overflow_time = clock.CurrentTime() + kRtpOverflowDelay * 2;
+
+  while (clock.CurrentTime() < overflow_time) {
+    clock.AdvanceTime(TimeDelta::Seconds(10));
+    // Static-cast before += to avoid undefined behaviour of overflow.
+    rtp += static_cast<uint32_t>(kRtpHz * TimeDelta::Seconds(10));
+    ts_extrapolator.Update(clock.CurrentTime(), rtp);
+    EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+                Optional(clock.CurrentTime()));
+  }
+}
+
+TEST(TimestampExtrapolatorTest, Slow90KHzClock) {
+  // This simulates a slow camera, which produces frames at 24Hz instead of
+  // 25Hz. The extrapolator should be able to resolve this with enough data.
+  SimulatedClock clock(Timestamp::Millis(1337));
+  TimestampExtrapolator ts_extrapolator(clock.CurrentTime());
+
+  constexpr TimeDelta k24FpsDelay = 1 / Frequency::Hertz(24);
+  uint32_t rtp = 90000;
+  ts_extrapolator.Update(clock.CurrentTime(), rtp);
+
+  // Slow camera will increment RTP at 25 FPS rate even though its producing at
+  // 24 FPS. After 25 frames the extrapolator should settle at this rate.
+  for (int i = 0; i < 25; ++i) {
+    rtp += kRtpHz / k25Fps;
+    clock.AdvanceTime(k24FpsDelay);
+    ts_extrapolator.Update(clock.CurrentTime(), rtp);
+  }
+
+  // The camera would normally produce 25 frames in 90K ticks, but is slow
+  // so takes 1s + k24FpsDelay for 90K ticks.
+  constexpr Frequency kSlowRtpHz = 90000 / (25 * k24FpsDelay);
+  // The extrapolator will be predicting that time at millisecond precision.
+  auto ts = ts_extrapolator.ExtrapolateLocalTime(rtp + kSlowRtpHz.hertz());
+  ASSERT_TRUE(ts.has_value());
+  EXPECT_EQ(ts->ms(), clock.TimeInMilliseconds() + 1000);
+}
+
+TEST(TimestampExtrapolatorTest, Fast90KHzClock) {
+  // This simulates a fast camera, which produces frames at 26Hz instead of
+  // 25Hz. The extrapolator should be able to resolve this with enough data.
+  SimulatedClock clock(Timestamp::Millis(1337));
+  TimestampExtrapolator ts_extrapolator(clock.CurrentTime());
+
+  constexpr TimeDelta k26FpsDelay = 1 / Frequency::Hertz(26);
+  uint32_t rtp = 90000;
+  ts_extrapolator.Update(clock.CurrentTime(), rtp);
+
+  // Fast camera will increment RTP at 25 FPS rate even though its producing at
+  // 26 FPS. After 25 frames the extrapolator should settle at this rate.
+  for (int i = 0; i < 25; ++i) {
+    rtp += kRtpHz / k25Fps;
+    clock.AdvanceTime(k26FpsDelay);
+    ts_extrapolator.Update(clock.CurrentTime(), rtp);
+  }
+
+  // The camera would normally produce 25 frames in 90K ticks, but is slow
+  // so takes 1s + k24FpsDelay for 90K ticks.
+  constexpr Frequency kSlowRtpHz = 90000 / (25 * k26FpsDelay);
+  // The extrapolator will be predicting that time at millisecond precision.
+  auto ts = ts_extrapolator.ExtrapolateLocalTime(rtp + kSlowRtpHz.hertz());
+  ASSERT_TRUE(ts.has_value());
+  EXPECT_EQ(ts->ms(), clock.TimeInMilliseconds() + 1000);
+}
+
+TEST(TimestampExtrapolatorTest, TimestampJump) {
+  // This simulates a jump in RTP timestamp, which could occur if a camera was
+  // swapped for example.
+  SimulatedClock clock(Timestamp::Millis(1337));
+  TimestampExtrapolator ts_extrapolator(clock.CurrentTime());
+
+  uint32_t rtp = 90000;
+  clock.AdvanceTime(k25FpsDelay);
+  ts_extrapolator.Update(clock.CurrentTime(), rtp);
+  rtp += kRtpHz / k25Fps;
+  clock.AdvanceTime(k25FpsDelay);
+  ts_extrapolator.Update(clock.CurrentTime(), rtp);
+  rtp += kRtpHz / k25Fps;
+  clock.AdvanceTime(k25FpsDelay);
+  ts_extrapolator.Update(clock.CurrentTime(), rtp);
+  EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp),
+              Optional(clock.CurrentTime()));
+  EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(rtp + 90000),
+              Optional(clock.CurrentTime() + TimeDelta::Seconds(1)));
+
+  // Jump RTP.
+  uint32_t new_rtp = 1337 * 90000;
+  clock.AdvanceTime(k25FpsDelay);
+  ts_extrapolator.Update(clock.CurrentTime(), new_rtp);
+  new_rtp += kRtpHz / k25Fps;
+  clock.AdvanceTime(k25FpsDelay);
+  ts_extrapolator.Update(clock.CurrentTime(), new_rtp);
+  EXPECT_THAT(ts_extrapolator.ExtrapolateLocalTime(new_rtp),
+              Optional(clock.CurrentTime()));
+}
+
+}  // namespace webrtc

modules/video_coding/timing/timing.cc

@@ -13,9 +13,9 @@
 #include <algorithm>
 
 #include "api/units/time_delta.h"
+#include "modules/video_coding/timing/timestamp_extrapolator.h"
 #include "rtc_base/experiments/field_trial_parser.h"
 #include "rtc_base/logging.h"
-#include "rtc_base/time/timestamp_extrapolator.h"
 #include "system_wrappers/include/clock.h"
 
 namespace webrtc {

modules/video_coding/timing/timing.h

@@ -19,10 +19,10 @@
 #include "api/video/video_frame.h"
 #include "api/video/video_timing.h"
 #include "modules/video_coding/timing/codec_timer.h"
+#include "modules/video_coding/timing/timestamp_extrapolator.h"
 #include "rtc_base/experiments/field_trial_parser.h"
 #include "rtc_base/synchronization/mutex.h"
 #include "rtc_base/thread_annotations.h"
-#include "rtc_base/time/timestamp_extrapolator.h"
 
 namespace webrtc {