Android: Add video processing interface

This CL adds an API for injecting video processing after the WebRTC
CPU and QP scaling step.

Bug: webrtc:10247
Change-Id: I776498e1e9113f50e953ee411bbb31f181863312
Reviewed-on: https://webrtc-review.googlesource.com/c/119953
Commit-Queue: Magnus Jedvert <magjed@webrtc.org>
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26740}

sdk/android/BUILD.gn

@@ -326,6 +326,7 @@ if (is_android) {
       "api/org/webrtc/StatsReport.java",
       "api/org/webrtc/TurnCustomizer.java",
       "api/org/webrtc/VideoSource.java",
+      "api/org/webrtc/VideoProcessor.java",
       "api/org/webrtc/VideoTrack.java",
       "src/java/org/webrtc/NativeLibrary.java",
       "src/java/org/webrtc/NativeCapturerObserver.java",

sdk/android/api/org/webrtc/VideoProcessor.java

@@ -0,0 +1,25 @@
+/*
+ *  Copyright 2019 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import android.support.annotation.Nullable;
+
+/**
+ * Lightweight abstraction for an object that can receive video frames, process them, and pass them
+ * on to another object. This object is also allowed to observe capturer start/stop.
+ */
+public interface VideoProcessor extends CapturerObserver {
+  /**
+   * Set the sink that receives the output from this processor. Null can be passed in to unregister
+   * a sink. After this call returns, no frames should be delivered to an unregistered sink.
+   */
+  void setSink(@Nullable VideoSink sink);
+}

sdk/android/api/org/webrtc/VideoSource.java

@@ -30,15 +30,31 @@ public class VideoSource extends MediaSource {
   }
 
   private final NativeAndroidVideoTrackSource nativeAndroidVideoTrackSource;
+  private final Object videoProcessorLock = new Object();
+  @Nullable private VideoProcessor videoProcessor;
+  private boolean isCapturerRunning;
+
   private final CapturerObserver capturerObserver = new CapturerObserver() {
     @Override
     public void onCapturerStarted(boolean success) {
       nativeAndroidVideoTrackSource.setState(success);
+      synchronized (videoProcessorLock) {
+        isCapturerRunning = success;
+        if (videoProcessor != null) {
+          videoProcessor.onCapturerStarted(success);
+        }
+      }
     }
 
     @Override
     public void onCapturerStopped() {
       nativeAndroidVideoTrackSource.setState(/* isLive= */ false);
+      synchronized (videoProcessorLock) {
+        isCapturerRunning = false;
+        if (videoProcessor != null) {
+          videoProcessor.onCapturerStopped();
+        }
+      }
     }
 
     @Override
@@ -53,9 +69,17 @@ public class VideoSource extends MediaSource {
       final VideoFrame.Buffer adaptedBuffer =
           frame.getBuffer().cropAndScale(parameters.cropX, parameters.cropY, parameters.cropWidth,
               parameters.cropHeight, parameters.scaleWidth, parameters.scaleHeight);
-      // TODO(magjed): Add video processing hook here.
+      final VideoFrame adaptedFrame =
-      nativeAndroidVideoTrackSource.onFrameCaptured(
+          new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs);
-          new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs));
+
+      synchronized (videoProcessorLock) {
+        if (videoProcessor != null) {
+          videoProcessor.onFrameCaptured(adaptedFrame);
+          adaptedBuffer.release();
+          return;
+        }
+      }
+      nativeAndroidVideoTrackSource.onFrameCaptured(adaptedFrame);
       adaptedBuffer.release();
     }
   };
@@ -98,6 +122,31 @@ public class VideoSource extends MediaSource {
         maxLandscapePixelCount, targetPortraitAspectRatio, maxPortraitPixelCount, maxFps);
   }
 
+  /**
+   * Hook for injecting a custom video processor before frames are passed onto WebRTC. The frames
+   * will be cropped and scaled depending on CPU and network conditions before they are passed to
+   * the video processor. Frames will be delivered to the video processor on the same thread they
+   * are passed to this object. The video processor is allowed to deliver the processed frames
+   * back on any thread.
+   */
+  public void setVideoProcessor(@Nullable VideoProcessor newVideoProcessor) {
+    synchronized (videoProcessorLock) {
+      if (videoProcessor != null) {
+        videoProcessor.setSink(/* sink= */ null);
+        if (isCapturerRunning) {
+          videoProcessor.onCapturerStopped();
+        }
+      }
+      videoProcessor = newVideoProcessor;
+      if (newVideoProcessor != null) {
+        newVideoProcessor.setSink(nativeAndroidVideoTrackSource::onFrameCaptured);
+        if (isCapturerRunning) {
+          newVideoProcessor.onCapturerStarted(/* success= */ true);
+        }
+      }
+    }
+  }
+
   public CapturerObserver getCapturerObserver() {
     return capturerObserver;
   }
@@ -106,4 +155,10 @@ public class VideoSource extends MediaSource {
   long getNativeVideoTrackSource() {
     return getNativeMediaSource();
   }
+
+  @Override
+  public void dispose() {
+    setVideoProcessor(/* newVideoProcessor= */ null);
+    super.dispose();
+  }
 }