Android: Support externally aligned timestamps

This support is needed if there is a big delay between the creation of
frames and the time they are delivered to the WebRTC C++ layer in
AndroidVideoTrackSource. This is the case if e.g. some heavy video
processing is applied to the frames that takes a couple of hundred
milliseconds. Currently, timestamps coming from Android video sources
are aligned to rtc::TimeMicros() once they reach the WebRTC C++ layer in
AndroidVideoTrackSource. At this point, we "forget" any latency that
might occur before this point, and audio/video sync consequently
suffers.

Bug: webrtc:9991
Change-Id: I7b1aaca9a60a978b9195dd5e5eed4779a0055607
Reviewed-on: https://webrtc-review.googlesource.com/c/110783
Commit-Queue: Magnus Jedvert <magjed@webrtc.org>
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25654}

sdk/android/api/org/webrtc/PeerConnectionFactory.java

@@ -440,9 +440,25 @@ public class PeerConnectionFactory {
     return new MediaStream(nativeCreateLocalMediaStream(nativeFactory, label));
   }
 
-  public VideoSource createVideoSource(boolean isScreencast) {
+  /**
+   * Create video source with given parameters. If alignTimestamps is false, the caller is
+   * responsible for aligning the frame timestamps to rtc::TimeNanos(). This can be used to achieve
+   * higher accuracy if there is a big delay between frame creation and frames being delivered to
+   * the returned video source. If alignTimestamps is true, timestamps will be aligned to
+   * rtc::TimeNanos() when they arrive to the returned video source.
+   */
+  public VideoSource createVideoSource(boolean isScreencast, boolean alignTimestamps) {
     checkPeerConnectionFactoryExists();
-    return new VideoSource(nativeCreateVideoSource(nativeFactory, isScreencast));
+    return new VideoSource(nativeCreateVideoSource(nativeFactory, isScreencast, alignTimestamps));
+  }
+
+  /**
+   * Same as above with alignTimestamps set to true.
+   *
+   * @see #createVideoSource(boolean, boolean)
+   */
+  public VideoSource createVideoSource(boolean isScreencast) {
+    return createVideoSource(isScreencast, /* alignTimestamps= */ true);
   }
 
   public VideoTrack createVideoTrack(String id, VideoSource source) {
@@ -567,7 +583,8 @@ public class PeerConnectionFactory {
       PeerConnection.RTCConfiguration rtcConfig, MediaConstraints constraints, long nativeObserver,
       SSLCertificateVerifier sslCertificateVerifier);
   private static native long nativeCreateLocalMediaStream(long factory, String label);
-  private static native long nativeCreateVideoSource(long factory, boolean is_screencast);
+  private static native long nativeCreateVideoSource(
+      long factory, boolean is_screencast, boolean alignTimestamps);
   private static native long nativeCreateVideoTrack(
       long factory, String id, long nativeVideoSource);
   private static native long nativeCreateAudioSource(long factory, MediaConstraints constraints);

sdk/android/api/org/webrtc/SurfaceTextureHelper.java

@@ -36,10 +36,14 @@ public class SurfaceTextureHelper {
   /**
    * Construct a new SurfaceTextureHelper sharing OpenGL resources with |sharedContext|. A dedicated
    * thread and handler is created for handling the SurfaceTexture. May return null if EGL fails to
-   * initialize a pixel buffer surface and make it current.
+   * initialize a pixel buffer surface and make it current. If alignTimestamps is true, the frame
+   * timestamps will be aligned to rtc::TimeNanos(). If frame timestamps are aligned to
+   * rtc::TimeNanos() there is no need for aligning timestamps again in
+   * PeerConnectionFactory.createVideoSource(). This makes the timestamps more accurate and
+   * closer to actual creation time.
    */
   public static SurfaceTextureHelper create(
-      final String threadName, final EglBase.Context sharedContext) {
+      final String threadName, final EglBase.Context sharedContext, boolean alignTimestamps) {
     final HandlerThread thread = new HandlerThread(threadName);
     thread.start();
     final Handler handler = new Handler(thread.getLooper());
@@ -53,7 +57,7 @@ public class SurfaceTextureHelper {
       @Override
       public SurfaceTextureHelper call() {
         try {
-          return new SurfaceTextureHelper(sharedContext, handler);
+          return new SurfaceTextureHelper(sharedContext, handler, alignTimestamps);
         } catch (RuntimeException e) {
           Logging.e(TAG, threadName + " create failure", e);
           return null;
@@ -62,11 +66,22 @@ public class SurfaceTextureHelper {
     });
   }
 
+  /**
+   * Same as above with alignTimestamps set to false.
+   *
+   * @see #create(String, EglBase.Context, boolean)
+   */
+  public static SurfaceTextureHelper create(
+      final String threadName, final EglBase.Context sharedContext) {
+    return create(threadName, sharedContext, /* alignTimestamps= */ false);
+  }
+
   private final Handler handler;
   private final EglBase eglBase;
   private final SurfaceTexture surfaceTexture;
   private final int oesTextureId;
   private final YuvConverter yuvConverter = new YuvConverter();
+  @Nullable private final TimestampAligner timestampAligner;
 
   // These variables are only accessed from the |handler| thread.
   @Nullable private VideoSink listener;
@@ -95,11 +110,13 @@ public class SurfaceTextureHelper {
     }
   };
 
-  private SurfaceTextureHelper(EglBase.Context sharedContext, Handler handler) {
+  private SurfaceTextureHelper(
+      EglBase.Context sharedContext, Handler handler, boolean alignTimestamps) {
     if (handler.getLooper().getThread() != Thread.currentThread()) {
       throw new IllegalStateException("SurfaceTextureHelper must be created on the handler thread");
     }
     this.handler = handler;
+    this.timestampAligner = alignTimestamps ? new TimestampAligner() : null;
 
     eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_BUFFER);
     try {
@@ -264,7 +281,10 @@ public class SurfaceTextureHelper {
 
     final float[] transformMatrix = new float[16];
     surfaceTexture.getTransformMatrix(transformMatrix);
-    final long timestampNs = surfaceTexture.getTimestamp();
+    long timestampNs = surfaceTexture.getTimestamp();
+    if (timestampAligner != null) {
+      timestampNs = timestampAligner.translateTimestamp(timestampNs);
+    }
     if (textureWidth == 0 || textureHeight == 0) {
       throw new RuntimeException("Texture size has not been set.");
     }
@@ -289,5 +309,8 @@ public class SurfaceTextureHelper {
     surfaceTexture.release();
     eglBase.release();
     handler.getLooper().quit();
+    if (timestampAligner != null) {
+      timestampAligner.dispose();
+    }
   }
 }

sdk/android/api/org/webrtc/TimestampAligner.java

@@ -0,0 +1,59 @@
+/*
+ *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+/**
+ * The TimestampAligner class helps translating camera timestamps into the same timescale as is
+ * used by rtc::TimeNanos(). Some cameras have built in timestamping which is more accurate than
+ * reading the system clock, but using a different epoch and unknown clock drift. Frame timestamps
+ * in webrtc should use rtc::TimeNanos (system monotonic time), and this class provides a filter
+ * which lets us use the rtc::TimeNanos timescale, and at the same time take advantage of higher
+ * accuracy of the camera clock. This class is a wrapper on top of rtc::TimestampAligner.
+ */
+public class TimestampAligner {
+  /**
+   * Wrapper around rtc::TimeNanos(). This is normally same as System.nanoTime(), but call this
+   * function to be safe.
+   */
+  public static long getRtcTimeNanos() {
+    return nativeRtcTimeNanos();
+  }
+
+  private volatile long nativeTimestampAligner = nativeCreateTimestampAligner();
+
+  /**
+   * Translates camera timestamps to the same timescale as is used by rtc::TimeNanos().
+   * |cameraTimeNs| is assumed to be accurate, but with an unknown epoch and clock drift. Returns
+   * the translated timestamp.
+   */
+  public long translateTimestamp(long cameraTimeNs) {
+    checkNativeAlignerExists();
+    return nativeTranslateTimestamp(nativeTimestampAligner, cameraTimeNs);
+  }
+
+  /** Dispose native timestamp aligner. */
+  public void dispose() {
+    checkNativeAlignerExists();
+    nativeReleaseTimestampAligner(nativeTimestampAligner);
+    nativeTimestampAligner = 0;
+  }
+
+  private void checkNativeAlignerExists() {
+    if (nativeTimestampAligner == 0) {
+      throw new IllegalStateException("TimestampAligner has been disposed.");
+    }
+  }
+
+  private static native long nativeRtcTimeNanos();
+  private static native long nativeCreateTimestampAligner();
+  private static native void nativeReleaseTimestampAligner(long timestampAligner);
+  private static native long nativeTranslateTimestamp(long timestampAligner, long cameraTimeNs);
+}