Android: Support externally aligned timestamps

This support is needed if there is a big delay between the creation of
frames and the time they are delivered to the WebRTC C++ layer in
AndroidVideoTrackSource. This is the case if e.g. some heavy video
processing is applied to the frames that takes a couple of hundred
milliseconds. Currently, timestamps coming from Android video sources
are aligned to rtc::TimeMicros() once they reach the WebRTC C++ layer in
AndroidVideoTrackSource. At this point, we "forget" any latency that
might occur before this point, and audio/video sync consequently
suffers.

Bug: webrtc:9991
Change-Id: I7b1aaca9a60a978b9195dd5e5eed4779a0055607
Reviewed-on: https://webrtc-review.googlesource.com/c/110783
Commit-Queue: Magnus Jedvert <magjed@webrtc.org>
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25654}

sdk/android/api/org/webrtc/SurfaceTextureHelper.java

@@ -36,10 +36,14 @@ public class SurfaceTextureHelper {
   /**
    * Construct a new SurfaceTextureHelper sharing OpenGL resources with |sharedContext|. A dedicated
    * thread and handler is created for handling the SurfaceTexture. May return null if EGL fails to
-   * initialize a pixel buffer surface and make it current.
+   * initialize a pixel buffer surface and make it current. If alignTimestamps is true, the frame
+   * timestamps will be aligned to rtc::TimeNanos(). If frame timestamps are aligned to
+   * rtc::TimeNanos() there is no need for aligning timestamps again in
+   * PeerConnectionFactory.createVideoSource(). This makes the timestamps more accurate and
+   * closer to actual creation time.
    */
   public static SurfaceTextureHelper create(
-      final String threadName, final EglBase.Context sharedContext) {
+      final String threadName, final EglBase.Context sharedContext, boolean alignTimestamps) {
     final HandlerThread thread = new HandlerThread(threadName);
     thread.start();
     final Handler handler = new Handler(thread.getLooper());
@@ -53,7 +57,7 @@ public class SurfaceTextureHelper {
       @Override
       public SurfaceTextureHelper call() {
         try {
-          return new SurfaceTextureHelper(sharedContext, handler);
+          return new SurfaceTextureHelper(sharedContext, handler, alignTimestamps);
         } catch (RuntimeException e) {
           Logging.e(TAG, threadName + " create failure", e);
           return null;
@@ -62,11 +66,22 @@ public class SurfaceTextureHelper {
     });
   }
 
+  /**
+   * Same as above with alignTimestamps set to false.
+   *
+   * @see #create(String, EglBase.Context, boolean)
+   */
+  public static SurfaceTextureHelper create(
+      final String threadName, final EglBase.Context sharedContext) {
+    return create(threadName, sharedContext, /* alignTimestamps= */ false);
+  }
+
   private final Handler handler;
   private final EglBase eglBase;
   private final SurfaceTexture surfaceTexture;
   private final int oesTextureId;
   private final YuvConverter yuvConverter = new YuvConverter();
+  @Nullable private final TimestampAligner timestampAligner;
 
   // These variables are only accessed from the |handler| thread.
   @Nullable private VideoSink listener;
@@ -95,11 +110,13 @@ public class SurfaceTextureHelper {
     }
   };
 
-  private SurfaceTextureHelper(EglBase.Context sharedContext, Handler handler) {
+  private SurfaceTextureHelper(
+      EglBase.Context sharedContext, Handler handler, boolean alignTimestamps) {
     if (handler.getLooper().getThread() != Thread.currentThread()) {
       throw new IllegalStateException("SurfaceTextureHelper must be created on the handler thread");
     }
     this.handler = handler;
+    this.timestampAligner = alignTimestamps ? new TimestampAligner() : null;
 
     eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_BUFFER);
     try {
@@ -264,7 +281,10 @@ public class SurfaceTextureHelper {
 
     final float[] transformMatrix = new float[16];
     surfaceTexture.getTransformMatrix(transformMatrix);
-    final long timestampNs = surfaceTexture.getTimestamp();
+    long timestampNs = surfaceTexture.getTimestamp();
+    if (timestampAligner != null) {
+      timestampNs = timestampAligner.translateTimestamp(timestampNs);
+    }
     if (textureWidth == 0 || textureHeight == 0) {
       throw new RuntimeException("Texture size has not been set.");
     }
@@ -289,5 +309,8 @@ public class SurfaceTextureHelper {
     surfaceTexture.release();
     eglBase.release();
     handler.getLooper().quit();
+    if (timestampAligner != null) {
+      timestampAligner.dispose();
+    }
   }
 }