Move frame adaptation inside video processor.

Bug: webrtc:10530
Change-Id: Iba6a91bf3e1ec4b2821b554e9e28fd2ead662723
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/131947
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Commit-Queue: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27542}

sdk/android/api/org/webrtc/VideoProcessor.java

@@ -17,9 +17,60 @@ import android.support.annotation.Nullable;
  * on to another object. This object is also allowed to observe capturer start/stop.
  */
 public interface VideoProcessor extends CapturerObserver {
+  public static class FrameAdaptationParameters {
+    public final int cropX;
+    public final int cropY;
+    public final int cropWidth;
+    public final int cropHeight;
+    public final int scaleWidth;
+    public final int scaleHeight;
+    public final long timestampNs;
+    public final boolean drop;
+
+    public FrameAdaptationParameters(int cropX, int cropY, int cropWidth, int cropHeight,
+        int scaleWidth, int scaleHeight, long timestampNs, boolean drop) {
+      this.cropX = cropX;
+      this.cropY = cropY;
+      this.cropWidth = cropWidth;
+      this.cropHeight = cropHeight;
+      this.scaleWidth = scaleWidth;
+      this.scaleHeight = scaleHeight;
+      this.timestampNs = timestampNs;
+      this.drop = drop;
+    }
+  }
+
+  /**
+   * This is a chance to access an unadapted frame. The default implementation applies the
+   * adaptation and forwards the frame to {@link #onFrameCaptured(VideoFrame)}.
+   */
+  default void onFrameCaptured(VideoFrame frame, FrameAdaptationParameters parameters) {
+    VideoFrame adaptedFrame = applyFrameAdaptationParameters(frame, parameters);
+    if (adaptedFrame != null) {
+      onFrameCaptured(adaptedFrame);
+      adaptedFrame.release();
+    }
+  }
+
   /**
    * Set the sink that receives the output from this processor. Null can be passed in to unregister
    * a sink. After this call returns, no frames should be delivered to an unregistered sink.
    */
   void setSink(@Nullable VideoSink sink);
+
+  /**
+   * Applies the frame adaptation parameters to a frame. Returns null if the frame is meant to be
+   * dropped. Returns a new frame. The caller is responsible for releasing the returned frame.
+   */
+  public static @Nullable VideoFrame applyFrameAdaptationParameters(
+      VideoFrame frame, FrameAdaptationParameters parameters) {
+    if (parameters.drop) {
+      return null;
+    }
+
+    final VideoFrame.Buffer adaptedBuffer =
+        frame.getBuffer().cropAndScale(parameters.cropX, parameters.cropY, parameters.cropWidth,
+            parameters.cropHeight, parameters.scaleWidth, parameters.scaleHeight);
+    return new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs);
+  }
 }

sdk/android/api/org/webrtc/VideoSource.java

@@ -59,28 +59,20 @@ public class VideoSource extends MediaSource {
 
     @Override
     public void onFrameCaptured(VideoFrame frame) {
-      final NativeAndroidVideoTrackSource.FrameAdaptationParameters parameters =
+      final VideoProcessor.FrameAdaptationParameters parameters =
           nativeAndroidVideoTrackSource.adaptFrame(frame);
-      if (parameters == null) {
-        // Drop frame.
-        return;
-      }
-
-      final VideoFrame.Buffer adaptedBuffer =
-          frame.getBuffer().cropAndScale(parameters.cropX, parameters.cropY, parameters.cropWidth,
-              parameters.cropHeight, parameters.scaleWidth, parameters.scaleHeight);
-      final VideoFrame adaptedFrame =
-          new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs);
-
       synchronized (videoProcessorLock) {
         if (videoProcessor != null) {
-          videoProcessor.onFrameCaptured(adaptedFrame);
+          videoProcessor.onFrameCaptured(frame, parameters);
-          adaptedBuffer.release();
           return;
         }
       }
-      nativeAndroidVideoTrackSource.onFrameCaptured(adaptedFrame);
+
-      adaptedBuffer.release();
+      VideoFrame adaptedFrame = VideoProcessor.applyFrameAdaptationParameters(frame, parameters);
+      if (adaptedFrame != null) {
+        nativeAndroidVideoTrackSource.onFrameCaptured(adaptedFrame);
+        adaptedFrame.release();
+      }
     }
   };
 

sdk/android/src/java/org/webrtc/NativeAndroidVideoTrackSource.java

@@ -12,6 +12,7 @@ package org.webrtc;
 
 import android.support.annotation.Nullable;
 import org.webrtc.VideoFrame;
+import org.webrtc.VideoProcessor;
 
 /**
  * This class is meant to be a simple layer that only handles the JNI wrapping of a C++
@@ -25,28 +26,6 @@ class NativeAndroidVideoTrackSource {
   // Pointer to webrtc::jni::AndroidVideoTrackSource.
   private final long nativeAndroidVideoTrackSource;
 
-  public static class FrameAdaptationParameters {
-    public final int cropX;
-    public final int cropY;
-    public final int cropWidth;
-    public final int cropHeight;
-    public final int scaleWidth;
-    public final int scaleHeight;
-    public final long timestampNs;
-
-    @CalledByNative("FrameAdaptationParameters")
-    FrameAdaptationParameters(int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth,
-        int scaleHeight, long timestampNs) {
-      this.cropX = cropX;
-      this.cropY = cropY;
-      this.cropWidth = cropWidth;
-      this.cropHeight = cropHeight;
-      this.scaleWidth = scaleWidth;
-      this.scaleHeight = scaleHeight;
-      this.timestampNs = timestampNs;
-    }
-  }
-
   public NativeAndroidVideoTrackSource(long nativeAndroidVideoTrackSource) {
     this.nativeAndroidVideoTrackSource = nativeAndroidVideoTrackSource;
   }
@@ -66,7 +45,7 @@ class NativeAndroidVideoTrackSource {
    * adaptation parameters before calling onFrameCaptured().
    */
   @Nullable
-  public FrameAdaptationParameters adaptFrame(VideoFrame frame) {
+  public VideoProcessor.FrameAdaptationParameters adaptFrame(VideoFrame frame) {
     return nativeAdaptFrame(nativeAndroidVideoTrackSource, frame.getBuffer().getWidth(),
         frame.getBuffer().getHeight(), frame.getRotation(), frame.getTimestampNs());
   }
@@ -93,13 +72,21 @@ class NativeAndroidVideoTrackSource {
         targetPortraitAspectRatio.height, maxPortraitPixelCount, maxFps);
   }
 
+  @CalledByNative
+  static VideoProcessor.FrameAdaptationParameters createFrameAdaptationParameters(int cropX,
+      int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight, long timestampNs,
+      boolean drop) {
+    return new VideoProcessor.FrameAdaptationParameters(
+        cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight, timestampNs, drop);
+  }
+
   private static native void nativeSetState(long nativeAndroidVideoTrackSource, boolean isLive);
   private static native void nativeAdaptOutputFormat(long nativeAndroidVideoTrackSource,
       int landscapeWidth, int landscapeHeight, @Nullable Integer maxLandscapePixelCount,
       int portraitWidth, int portraitHeight, @Nullable Integer maxPortraitPixelCount,
       @Nullable Integer maxFps);
   @Nullable
-  private static native FrameAdaptationParameters nativeAdaptFrame(
+  private static native VideoProcessor.FrameAdaptationParameters nativeAdaptFrame(
       long nativeAndroidVideoTrackSource, int width, int height, int rotation, long timestampNs);
   private static native void nativeOnFrameCaptured(
       long nativeAndroidVideoTrackSource, int rotation, long timestampNs, VideoFrame.Buffer buffer);

sdk/android/src/java/org/webrtc/NativeCapturerObserver.java

@@ -36,7 +36,7 @@ class NativeCapturerObserver implements CapturerObserver {
 
   @Override
   public void onFrameCaptured(VideoFrame frame) {
-    final NativeAndroidVideoTrackSource.FrameAdaptationParameters parameters =
+    final VideoProcessor.FrameAdaptationParameters parameters =
         nativeAndroidVideoTrackSource.adaptFrame(frame);
     if (parameters == null) {
       // Drop frame.

sdk/android/src/jni/android_video_track_source.cc

@@ -102,33 +102,30 @@ ScopedJavaLocalRef<jobject> AndroidVideoTrackSource::AdaptFrame(
                                   camera_time_us, rtc::TimeMicros())
                         : j_timestamp_ns;
 
-  int adapted_width;
+  int adapted_width = 0;
-  int adapted_height;
+  int adapted_height = 0;
-  int crop_width;
+  int crop_width = 0;
-  int crop_height;
+  int crop_height = 0;
-  int crop_x;
+  int crop_x = 0;
-  int crop_y;
+  int crop_y = 0;
+  bool drop;
 
   // TODO(magjed): Move this logic to users of NativeAndroidVideoTrackSource
   // instead, in order to keep this native wrapping layer as thin as possible.
   if (rotation % 180 == 0) {
-    if (!rtc::AdaptedVideoTrackSource::AdaptFrame(
+    drop = !rtc::AdaptedVideoTrackSource::AdaptFrame(
-            j_width, j_height, camera_time_us, &adapted_width, &adapted_height,
+        j_width, j_height, camera_time_us, &adapted_width, &adapted_height,
-            &crop_width, &crop_height, &crop_x, &crop_y)) {
+        &crop_width, &crop_height, &crop_x, &crop_y);
-      return nullptr;
-    }
   } else {
     // Swap all width/height and x/y.
-    if (!rtc::AdaptedVideoTrackSource::AdaptFrame(
+    drop = !rtc::AdaptedVideoTrackSource::AdaptFrame(
-            j_height, j_width, camera_time_us, &adapted_height, &adapted_width,
+        j_height, j_width, camera_time_us, &adapted_height, &adapted_width,
-            &crop_height, &crop_width, &crop_y, &crop_x)) {
+        &crop_height, &crop_width, &crop_y, &crop_x);
-      return nullptr;
-    }
   }
 
-  return Java_FrameAdaptationParameters_Constructor(
+  return Java_NativeAndroidVideoTrackSource_createFrameAdaptationParameters(
       env, crop_x, crop_y, crop_width, crop_height, adapted_width,
-      adapted_height, aligned_timestamp_ns);
+      adapted_height, aligned_timestamp_ns, drop);
 }
 
 void AndroidVideoTrackSource::OnFrameCaptured(