VideoCapturerAndroid: Add function to change capture format while camera is running

Review URL: https://codereview.webrtc.org/1178703009

Cr-Commit-Position: refs/heads/master@{#9608}

talk/app/webrtc/androidtests/src/org/webrtc/VideoCapturerAndroidTest.java

@@ -82,8 +82,8 @@ public class VideoCapturerAndroidTest extends ActivityTestCase {
     }
 
     @Override
-    public void OnFrameCaptured(byte[] frame, int length, int rotation,
+    public void OnFrameCaptured(byte[] frame, int length, int width, int height,
-        long timeStamp) {
+        int rotation, long timeStamp) {
       synchronized (frameLock) {
         ++framesCaptured;
         frameSize = length;

talk/app/webrtc/androidvideocapturer.cc

@@ -67,10 +67,14 @@ class AndroidVideoCapturer::FrameFactory : public cricket::VideoFrameFactory {
 
   void UpdateCapturedFrame(void* frame_data,
                            int length,
+                           int width,
+                           int height,
                            int rotation,
                            int64 time_stamp_in_ns) {
     captured_frame_.fourcc = static_cast<uint32>(cricket::FOURCC_YV12);
     captured_frame_.data = frame_data;
+    captured_frame_.width = width;
+    captured_frame_.height = height;
     captured_frame_.elapsed_time = rtc::TimeNanos() - start_time_;
     captured_frame_.time_stamp = time_stamp_in_ns;
     captured_frame_.rotation = rotation;
@@ -235,10 +239,13 @@ void AndroidVideoCapturer::OnCapturerStarted(bool success) {
 
 void AndroidVideoCapturer::OnIncomingFrame(void* frame_data,
                                            int length,
+                                           int width,
+                                           int height,
                                            int rotation,
                                            int64 time_stamp) {
   CHECK(thread_checker_.CalledOnValidThread());
-  frame_factory_->UpdateCapturedFrame(frame_data, length, rotation, time_stamp);
+  frame_factory_->UpdateCapturedFrame(frame_data, length, width, height,
+                                      rotation, time_stamp);
   SignalFrameCaptured(this, frame_factory_->GetCapturedFrame());
 }
 

talk/app/webrtc/androidvideocapturer.h

@@ -71,6 +71,8 @@ class AndroidVideoCapturer : public cricket::VideoCapturer {
   // Called from JNI when a new frame has been captured.
   void OnIncomingFrame(void* video_frame,
                        int length,
+                       int width,
+                       int height,
                        int rotation,
                        int64 time_stamp);
 

talk/app/webrtc/java/jni/androidvideocapturer_jni.cc

@@ -174,12 +174,14 @@ void AndroidVideoCapturerJni::OnCapturerStarted(bool success) {
 
 void AndroidVideoCapturerJni::OnIncomingFrame(void* video_frame,
                                               int length,
+                                              int width,
+                                              int height,
                                               int rotation,
                                               int64 time_stamp) {
   invoker_.AsyncInvoke<void>(
       thread_,
-      rtc::Bind(&AndroidVideoCapturerJni::OnIncomingFrame_w,
+      rtc::Bind(&AndroidVideoCapturerJni::OnIncomingFrame_w, this, video_frame,
-                this, video_frame, length, rotation, time_stamp));
+                length, width, height, rotation, time_stamp));
 }
 
 void AndroidVideoCapturerJni::OnOutputFormatRequest(int width,
@@ -202,11 +204,14 @@ void AndroidVideoCapturerJni::OnCapturerStarted_w(bool success) {
 
 void AndroidVideoCapturerJni::OnIncomingFrame_w(void* video_frame,
                                                 int length,
+                                                int width,
+                                                int height,
                                                 int rotation,
                                                 int64 time_stamp) {
   CHECK(thread_checker_.CalledOnValidThread());
   if (capturer_) {
-    capturer_->OnIncomingFrame(video_frame, length, rotation, time_stamp);
+    capturer_->OnIncomingFrame(video_frame, length, width, height, rotation,
+                               time_stamp);
   } else {
     LOG(LS_INFO) <<
         "Frame arrived after camera has been stopped: " << time_stamp <<
@@ -230,12 +235,12 @@ JNIEnv* AndroidVideoCapturerJni::jni() { return AttachCurrentThreadIfNeeded(); }
 
 JOW(void, VideoCapturerAndroid_00024NativeObserver_nativeOnFrameCaptured)
     (JNIEnv* jni, jclass, jlong j_capturer, jbyteArray j_frame, jint length,
-        jint rotation, jlong ts) {
+        jint width, jint height, jint rotation, jlong ts) {
   jboolean is_copy = true;
   jbyte* bytes = jni->GetByteArrayElements(j_frame, &is_copy);
   if (!is_copy) {
-    reinterpret_cast<AndroidVideoCapturerJni*>(
+    reinterpret_cast<AndroidVideoCapturerJni*>(j_capturer)
-        j_capturer)->OnIncomingFrame(bytes, length, rotation, ts);
+        ->OnIncomingFrame(bytes, length, width, height, rotation, ts);
   } else {
     // If this is a copy of the original frame, it means that the memory
     // is not direct memory and thus VideoCapturerAndroid does not guarantee

talk/app/webrtc/java/jni/androidvideocapturer_jni.h

@@ -64,6 +64,8 @@ class AndroidVideoCapturerJni : public webrtc::AndroidVideoCapturerDelegate {
   void OnCapturerStarted(bool success);
   void OnIncomingFrame(void* video_frame,
                        int length,
+                       int width,
+                       int height,
                        int rotation,
                        int64 time_stamp);
   void OnOutputFormatRequest(int width, int height, int fps);
@@ -78,6 +80,8 @@ private:
   void OnCapturerStopped_w();
   void OnIncomingFrame_w(void* video_frame,
                          int length,
+                         int width,
+                         int height,
                          int rotation,
                          int64 time_stamp);
   void OnOutputFormatRequest_w(int width, int height, int fps);

talk/app/webrtc/java/src/org/webrtc/VideoCapturerAndroid.java

@@ -52,7 +52,11 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.IdentityHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.Exchanger;
 import java.util.concurrent.TimeUnit;
 
@@ -85,9 +89,12 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
   private SurfaceTexture cameraSurfaceTexture;
   private int[] cameraGlTextures = null;
   private final FramePool videoBuffers = new FramePool();
-  private int width;
+  // Remember the requested format in case we want to switch cameras.
-  private int height;
+  private int requestedWidth;
-  private int framerate;
+  private int requestedHeight;
+  private int requestedFramerate;
+  // The capture format will be the closest supported format to the requested format.
+  private CaptureFormat captureFormat;
   private int cameraFramesCount;
   private int captureBuffersCount;
   private volatile boolean pendingCameraSwitch;
@@ -129,7 +136,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
       }
       Log.d(TAG, "Camera fps: " + cameraFps + ". CaptureBuffers: " +
           String.format("%.1f", averageCaptureBuffersCount) +
-          ". Pending buffers: [" + videoBuffers.pendingFramesTimeStamps() + "]");
+          ". Pending buffers: " + videoBuffers.pendingFramesTimeStamps());
       if (cameraFramesCount == 0) {
         Log.e(TAG, "Camera freezed.");
         if (errorHandler != null) {
@@ -244,24 +251,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
       return false;
     }
 
-    int new_id = (id + 1) % Camera.getNumberOfCameras();
-
-    CaptureFormat formatToUse  = null;
-    List<CaptureFormat> formats = supportedFormats.get(new_id);
-    for (CaptureFormat format : formats) {
-      if (format.width == width && format.height == height) {
-        formatToUse = format;
-        break;
-      }
-    }
-
-    if (formatToUse == null) {
-      Log.d(TAG, "No valid format found to switch camera.");
-      return false;
-    }
-
     pendingCameraSwitch = true;
-    id = new_id;
+    id = (id + 1) % Camera.getNumberOfCameras();
     cameraThreadHandler.post(new Runnable() {
       @Override public void run() {
         switchCameraOnCameraThread(switchDoneEvent);
@@ -285,6 +276,25 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
     });
   }
 
+  // Reconfigure the camera to capture in a new format. This should only be called while the camera
+  // is running.
+  public synchronized void changeCaptureFormat(
+      final int width, final int height, final int framerate) {
+    if (cameraThreadHandler == null) {
+      Log.e(TAG, "Calling changeCaptureFormat() for already stopped camera.");
+      return;
+    }
+    cameraThreadHandler.post(new Runnable() {
+      @Override public void run() {
+        startPreviewOnCameraThread(width, height, framerate);
+      }
+    });
+  }
+
+  public synchronized List<CaptureFormat> getSupportedFormats() {
+    return supportedFormats.get(id);
+  }
+
   private VideoCapturerAndroid() {
     Log.d(TAG, "VideoCapturerAndroid");
   }
@@ -349,7 +359,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
     return getSupportedFormatsAsJson(id);
   }
 
-  static class CaptureFormat {
+  public static class CaptureFormat {
     public final int width;
     public final int height;
     public final int maxFramerate;
@@ -394,6 +404,21 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
     private static int roundUp(int x, int alignment) {
       return (int)ceil(x / (double)alignment) * alignment;
     }
+
+    @Override
+    public String toString() {
+      return width + "x" + height + "@[" + minFramerate + ":" + maxFramerate + "]";
+    }
+
+    @Override
+    public boolean equals(Object that) {
+      if (!(that instanceof CaptureFormat)) {
+        return false;
+      }
+      final CaptureFormat c = (CaptureFormat) that;
+      return width == c.width && height == c.height && maxFramerate == c.maxFramerate
+          && minFramerate == c.minFramerate;
+    }
   }
 
   private static String getSupportedFormatsAsJson(int id) throws JSONException {
@@ -479,9 +504,6 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
     if (cameraThreadHandler != null) {
       throw new RuntimeException("Camera has already been started.");
     }
-    this.width = width;
-    this.height = height;
-    this.framerate = framerate;
 
     Exchanger<Handler> handlerExchanger = new Exchanger<Handler>();
     cameraThread = new CameraThread(handlerExchanger);
@@ -539,40 +561,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
 
       Log.d(TAG, "Camera orientation: " + info.orientation +
           " .Device orientation: " + getDeviceOrientation());
-      Camera.Parameters parameters = camera.getParameters();
-      Log.d(TAG, "isVideoStabilizationSupported: " +
-          parameters.isVideoStabilizationSupported());
-      if (parameters.isVideoStabilizationSupported()) {
-        parameters.setVideoStabilization(true);
-      }
-
       camera.setErrorCallback(cameraErrorCallback);
-
+      startPreviewOnCameraThread(width, height, framerate);
-      int androidFramerate = framerate * 1000;
-      int[] range = getFramerateRange(parameters, androidFramerate);
-      if (range != null) {
-        Log.d(TAG, "Start capturing: " + width + "x" + height + "@[" +
-            range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX]  + ":" +
-            range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] + "]");
-        parameters.setPreviewFpsRange(
-            range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX],
-            range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]);
-      }
-      Camera.Size pictureSize = getPictureSize(parameters, width, height);
-      parameters.setPictureSize(pictureSize.width, pictureSize.height);
-      parameters.setPreviewSize(width, height);
-      // TODO(hbos): If other ImageFormats are to be supported then
-      // CaptureFormat needs to be updated (currently hard-coded to say YV12,
-      // getSupportedFormats only returns YV12).
-      int format = ImageFormat.YV12;
-      parameters.setPreviewFormat(format);
-      camera.setParameters(parameters);
-      // Note: setRecordingHint(true) actually decrease frame rate on N5.
-      // parameters.setRecordingHint(true);
-
-      videoBuffers.queueCameraBuffers(width, height, format, camera);
-      camera.setPreviewCallbackWithBuffer(this);
-      camera.startPreview();
       frameObserver.OnCapturerStarted(true);
 
       // Start camera observer.
@@ -593,6 +583,69 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
     return;
   }
 
+  // (Re)start preview with the closest supported format to |width| x |height| @ |framerate|.
+  private void startPreviewOnCameraThread(int width, int height, int framerate) {
+    Log.d(TAG, "startPreviewOnCameraThread requested: " + width + "x" + height + "@" + framerate);
+    if (camera == null) {
+      Log.e(TAG, "Calling startPreviewOnCameraThread on stopped camera.");
+      return;
+    }
+
+    requestedWidth = width;
+    requestedHeight = height;
+    requestedFramerate = framerate;
+
+    // Find closest supported format for |width| x |height| @ |framerate|.
+    final Camera.Parameters parameters = camera.getParameters();
+    final int[] range = getFramerateRange(parameters, framerate * 1000);
+    final Camera.Size previewSize =
+        getClosestSupportedSize(parameters.getSupportedPreviewSizes(), width, height);
+    final CaptureFormat captureFormat = new CaptureFormat(
+        previewSize.width, previewSize.height,
+        range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX],
+        range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]);
+
+    // Check if we are already using this capture format, then we don't need to do anything.
+    if (captureFormat.equals(this.captureFormat)) {
+      return;
+    }
+
+    // Update camera parameters.
+    Log.d(TAG, "isVideoStabilizationSupported: " +
+        parameters.isVideoStabilizationSupported());
+    if (parameters.isVideoStabilizationSupported()) {
+      parameters.setVideoStabilization(true);
+    }
+    // Note: setRecordingHint(true) actually decrease frame rate on N5.
+    // parameters.setRecordingHint(true);
+    if (captureFormat.maxFramerate > 0) {
+      parameters.setPreviewFpsRange(captureFormat.minFramerate, captureFormat.maxFramerate);
+    }
+    parameters.setPreviewSize(captureFormat.width, captureFormat.height);
+    parameters.setPreviewFormat(captureFormat.imageFormat);
+    // Picture size is for taking pictures and not for preview/video, but we need to set it anyway
+    // as a workaround for an aspect ratio problem on Nexus 7.
+    final Camera.Size pictureSize =
+        getClosestSupportedSize(parameters.getSupportedPictureSizes(), width, height);
+    parameters.setPictureSize(pictureSize.width, pictureSize.height);
+
+    // Temporarily stop preview if it's already running.
+    if (this.captureFormat != null) {
+      camera.stopPreview();
+      // Calling |setPreviewCallbackWithBuffer| with null should clear the internal camera buffer
+      // queue, but sometimes we receive a frame with the old resolution after this call anyway.
+      camera.setPreviewCallbackWithBuffer(null);
+    }
+
+    // (Re)start preview.
+    Log.d(TAG, "Start capturing: " + captureFormat);
+    this.captureFormat = captureFormat;
+    camera.setParameters(parameters);
+    videoBuffers.queueCameraBuffers(captureFormat.frameSize(), camera);
+    camera.setPreviewCallbackWithBuffer(this);
+    camera.startPreview();
+  }
+
   // Called by native code.  Returns true when camera is known to be stopped.
   synchronized void stopCapture() throws InterruptedException {
     if (cameraThreadHandler == null) {
@@ -627,6 +680,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
       camera.stopPreview();
       camera.setPreviewCallbackWithBuffer(null);
       videoBuffers.stopReturnBuffersToCamera();
+      captureFormat = null;
 
       camera.setPreviewTexture(null);
       cameraSurfaceTexture = null;
@@ -647,7 +701,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
     Log.d(TAG, "switchCameraOnCameraThread");
 
     doStopCaptureOnCameraThread();
-    startCaptureOnCameraThread(width, height, framerate, frameObserver,
+    startCaptureOnCameraThread(requestedWidth, requestedHeight, requestedFramerate, frameObserver,
         applicationContext);
     pendingCameraSwitch = false;
     Log.d(TAG, "switchCameraOnCameraThread done");
@@ -702,37 +756,40 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
     return orientation;
   }
 
-  private static int[] getFramerateRange(Camera.Parameters parameters,
+  // Helper class for finding the closest supported format for the two functions below.
-                                         int framerate) {
+  private static abstract class ClosestComparator<T> implements Comparator<T> {
-    List<int[]> listFpsRange = parameters.getSupportedPreviewFpsRange();
+    // Difference between supported and requested parameter.
-    int[] bestRange = null;
+    abstract int diff(T supportedParameter);
-    int bestRangeDiff = Integer.MAX_VALUE;
+
-    for (int[] range : listFpsRange) {
+    @Override
-      int rangeDiff =
+    public int compare(T t1, T t2) {
-          abs(framerate -range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX])
+      return diff(t1) - diff(t2);
-          + abs(range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] - framerate);
-      if (bestRangeDiff > rangeDiff) {
-        bestRange = range;
-        bestRangeDiff = rangeDiff;
     }
   }
-    return bestRange;
-  }
 
-  private static Camera.Size getPictureSize(Camera.Parameters parameters,
+  private static int[] getFramerateRange(Camera.Parameters parameters, final int framerate) {
-      int width, int height) {
+    List<int[]> listFpsRange = parameters.getSupportedPreviewFpsRange();
-    int bestAreaDiff = Integer.MAX_VALUE;
+    if (listFpsRange.isEmpty()) {
-    Camera.Size bestSize = null;
+      Log.w(TAG, "No supported preview fps range");
-    int requestedArea = width * height;
+      return new int[]{0, 0};
-    for (Camera.Size pictureSize : parameters.getSupportedPictureSizes()) {
-      int areaDiff = abs(requestedArea
-          - pictureSize.width * pictureSize.height);
-      if (areaDiff < bestAreaDiff) {
-        bestAreaDiff = areaDiff;
-        bestSize = pictureSize;
     }
+    return Collections.min(listFpsRange,
+        new ClosestComparator<int[]>() {
+          @Override int diff(int[] range) {
+            return abs(framerate - range[Camera.Parameters.PREVIEW_FPS_MIN_INDEX])
+                + abs(framerate - range[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]);
           }
-    return bestSize;
+     });
+  }
+
+  private static Camera.Size getClosestSupportedSize(
+      List<Camera.Size> supportedSizes, final int requestedWidth, final int requestedHeight) {
+    return Collections.min(supportedSizes,
+        new ClosestComparator<Camera.Size>() {
+          @Override int diff(Camera.Size size) {
+            return abs(requestedWidth - size.width) + abs(requestedHeight - size.height);
+          }
+     });
   }
 
   // Called on cameraThread so must not "synchronized".
@@ -748,11 +805,9 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
       throw new RuntimeException("Unexpected camera in callback!");
     }
 
-    long captureTimeNs =
+    final long captureTimeNs = SystemClock.elapsedRealtimeNanos();
-        TimeUnit.MILLISECONDS.toNanos(SystemClock.elapsedRealtime());
 
-    cameraFramesCount++;
+    captureBuffersCount += videoBuffers.numCaptureBuffersAvailable();
-    captureBuffersCount += videoBuffers.numCaptureBuffersAvailable;
     int rotation = getDeviceOrientation();
     if (info.facing == Camera.CameraInfo.CAMERA_FACING_BACK) {
       rotation = 360 - rotation;
@@ -761,9 +816,13 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
     // Mark the frame owning |data| as used.
     // Note that since data is directBuffer,
     // data.length >= videoBuffers.frameSize.
-    videoBuffers.reserveByteBuffer(data, captureTimeNs);
+    if (videoBuffers.reserveByteBuffer(data, captureTimeNs)) {
-    frameObserver.OnFrameCaptured(data, videoBuffers.frameSize, rotation,
+      cameraFramesCount++;
-        captureTimeNs);
+      frameObserver.OnFrameCaptured(data, videoBuffers.frameSize, captureFormat.width,
+          captureFormat.height, rotation, captureTimeNs);
+    } else {
+      Log.w(TAG, "reserveByteBuffer failed - dropping frame.");
+    }
   }
 
   // runCameraThreadUntilIdle make sure all posted messages to the cameraThread
@@ -799,132 +858,98 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
     // Arbitrary queue depth.  Higher number means more memory allocated & held,
     // lower number means more sensitivity to processing time in the client (and
     // potentially stalling the capturer if it runs out of buffers to write to).
-    private static int numCaptureBuffers = 3;
+    private static final int numCaptureBuffers = 3;
-    private final List<Frame> cameraFrames = new ArrayList<Frame>();
+    // This container tracks the buffers added as camera callback buffers. It is needed for finding
-    public int frameSize = 0;
+    // the corresponding ByteBuffer given a byte[].
-    public int numCaptureBuffersAvailable = 0;
+    private final Map<byte[], ByteBuffer> queuedBuffers = new IdentityHashMap<byte[], ByteBuffer>();
+    // This container tracks the frames that have been sent but not returned. It is needed for
+    // keeping the buffers alive and for finding the corresponding ByteBuffer given a timestamp.
+    private final Map<Long, ByteBuffer> pendingBuffers = new HashMap<Long, ByteBuffer>();
+    private int frameSize = 0;
     private Camera camera;
 
-    private static class Frame {
+    int numCaptureBuffersAvailable() {
-      private final ByteBuffer buffer;
+      return queuedBuffers.size();
-      public long timeStamp = -1;
-      public final int frameSize;
-
-      Frame(int frameSize) {
-        this.frameSize = frameSize;
-        buffer = ByteBuffer.allocateDirect(frameSize);
     }
 
-      byte[] data() {
+    // Discards previous queued buffers and adds new callback buffers to camera.
-        return buffer.array();
+    void queueCameraBuffers(int frameSize, Camera camera) {
-      }
-    }
-
-    // Adds frames as callback buffers to |camera|. If a new frame size is
-    // required, new buffers are allocated and added.
-    void queueCameraBuffers(int width, int height, int format, Camera camera) {
-      if (this.camera != null)
-        throw new RuntimeException("camera already set.");
-
       this.camera = camera;
-      int newFrameSize = CaptureFormat.frameSize(width, height, format);
+      this.frameSize = frameSize;
 
-      numCaptureBuffersAvailable = 0;
+      queuedBuffers.clear();
-      if (newFrameSize != frameSize) {
-        // Create new frames and add to the camera.
-        // The old frames will be released when frames are returned.
       for (int i = 0; i < numCaptureBuffers; ++i) {
-          Frame frame = new Frame(newFrameSize);
+        final ByteBuffer buffer = ByteBuffer.allocateDirect(frameSize);
-          cameraFrames.add(frame);
+        camera.addCallbackBuffer(buffer.array());
-          this.camera.addCallbackBuffer(frame.data());
+        queuedBuffers.put(buffer.array(), buffer);
       }
-        numCaptureBuffersAvailable = numCaptureBuffers;
+      Log.d(TAG, "queueCameraBuffers enqueued " + numCaptureBuffers
-      } else {
-        // Add all frames that have been returned.
-        for (Frame frame : cameraFrames) {
-          if (frame.timeStamp < 0) {
-            camera.addCallbackBuffer(frame.data());
-            numCaptureBuffersAvailable++;
-          }
-        }
-      }
-      frameSize = newFrameSize;
-      Log.d(TAG, "queueCameraBuffers enqued " + numCaptureBuffersAvailable
           + " buffers of size " + frameSize + ".");
     }
 
     String pendingFramesTimeStamps() {
-      String pendingTimeStamps = new String();
+      List<Long> timeStampsMs = new ArrayList<Long>();
-      for (Frame frame : cameraFrames) {
+      for (Long timeStampNs : pendingBuffers.keySet()) {
-        if (frame.timeStamp > -1) {
+        timeStampsMs.add(TimeUnit.NANOSECONDS.toMillis(timeStampNs));
-          pendingTimeStamps += " " +
-              TimeUnit.NANOSECONDS.toMillis(frame.timeStamp);
       }
-      }
+      return timeStampsMs.toString();
-      return pendingTimeStamps;
     }
 
     void stopReturnBuffersToCamera() {
       this.camera = null;
-      String pendingTimeStamps = pendingFramesTimeStamps();
       Log.d(TAG, "stopReturnBuffersToCamera called."
-            + (pendingTimeStamps.isEmpty() ?
+            + (pendingBuffers.isEmpty() ?
                    " All buffers have been returned."
-                   : " Pending buffers: [" + pendingTimeStamps + "]."));
+                   : " Pending buffers: " + pendingFramesTimeStamps() + "."));
-
     }
 
-    void reserveByteBuffer(byte[] data, long timeStamp) {
+    boolean reserveByteBuffer(byte[] data, long timeStamp) {
-      for (Frame frame : cameraFrames) {
+      final ByteBuffer buffer = queuedBuffers.remove(data);
-        if (data == frame.data()) {
+      if (buffer == null) {
-          if (frame.timeStamp > 0) {
+        // Frames might be posted to |onPreviewFrame| with the previous format while changing
-            throw new RuntimeException("Frame already in use !");
+        // capture format in |startPreviewOnCameraThread|. Drop these old frames.
+        Log.w(TAG, "Received callback buffer from previous configuration with length: "
+            + data.length);
+        return false;
       }
-          frame.timeStamp = timeStamp;
+      if (buffer.capacity() != frameSize) {
-          numCaptureBuffersAvailable--;
+        throw new IllegalStateException("Callback buffer has unexpected frame size");
-          if (numCaptureBuffersAvailable == 0) {
+      }
+      if (pendingBuffers.containsKey(timeStamp)) {
+        Log.e(TAG, "Timestamp already present in pending buffers - they need to be unique");
+        return false;
+      }
+      pendingBuffers.put(timeStamp, buffer);
+      if (queuedBuffers.isEmpty()) {
         Log.v(TAG, "Camera is running out of capture buffers."
-                + " Pending buffers: [" + pendingFramesTimeStamps() + "]");
+            + " Pending buffers: " + pendingFramesTimeStamps());
       }
-          return;
+      return true;
-        }
-      }
-      throw new RuntimeException("unknown data buffer?!?");
     }
 
     void returnBuffer(long timeStamp) {
-      Frame returnedFrame = null;
+      final ByteBuffer returnedFrame = pendingBuffers.remove(timeStamp);
-      for (Frame frame : cameraFrames) {
-        if (timeStamp == frame.timeStamp) {
-          frame.timeStamp = -1;
-          returnedFrame = frame;
-          break;
-        }
-      }
-
       if (returnedFrame == null) {
         throw new RuntimeException("unknown data buffer with time stamp "
             + timeStamp + "returned?!?");
       }
 
-      if (camera != null && returnedFrame.frameSize == frameSize) {
+      if (camera != null && returnedFrame.capacity() == frameSize) {
-        camera.addCallbackBuffer(returnedFrame.data());
+        camera.addCallbackBuffer(returnedFrame.array());
-        if (numCaptureBuffersAvailable == 0) {
+        if (queuedBuffers.isEmpty()) {
           Log.v(TAG, "Frame returned when camera is running out of capture"
               + " buffers for TS " + TimeUnit.NANOSECONDS.toMillis(timeStamp));
         }
-        numCaptureBuffersAvailable++;
+        queuedBuffers.put(returnedFrame.array(), returnedFrame);
         return;
       }
 
-      if (returnedFrame.frameSize != frameSize) {
+      if (returnedFrame.capacity() != frameSize) {
         Log.d(TAG, "returnBuffer with time stamp "
             + TimeUnit.NANOSECONDS.toMillis(timeStamp)
-            + " called with old frame size, " + returnedFrame.frameSize + ".");
+            + " called with old frame size, " + returnedFrame.capacity() + ".");
-        // Since this frame has the wrong size, remove it from the list. Frames
+        // Since this frame has the wrong size, don't requeue it. Frames with the correct size are
-        // with the correct size is created in queueCameraBuffers so this must
+        // created in queueCameraBuffers so this must be an old buffer.
-        // be an old buffer.
-        cameraFrames.remove(returnedFrame);
         return;
       }
 
@@ -942,8 +967,8 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
 
     // Delivers a captured frame. Called on a Java thread owned by
     // VideoCapturerAndroid.
-    abstract void OnFrameCaptured(byte[] data, int length, int rotation,
+    abstract void OnFrameCaptured(byte[] data, int length, int width, int height,
-        long timeStamp);
+        int rotation, long timeStamp);
 
     // Requests an output format from the video capturer. Captured frames
     // by the camera will be scaled/or dropped by the video capturer.
@@ -966,9 +991,9 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
     }
 
     @Override
-    public void OnFrameCaptured(byte[] data, int length, int rotation,
+    public void OnFrameCaptured(byte[] data, int length, int width, int height,
-        long timeStamp) {
+        int rotation, long timeStamp) {
-      nativeOnFrameCaptured(nativeCapturer, data, length, rotation, timeStamp);
+      nativeOnFrameCaptured(nativeCapturer, data, length, width, height, rotation, timeStamp);
     }
 
     @Override
@@ -979,7 +1004,7 @@ public class VideoCapturerAndroid extends VideoCapturer implements PreviewCallba
     private native void nativeCapturerStarted(long nativeCapturer,
         boolean success);
     private native void nativeOnFrameCaptured(long nativeCapturer,
-        byte[] data, int length, int rotation, long timeStamp);
+        byte[] data, int length, int width, int height, int rotation, long timeStamp);
     private native void nativeOnOutputFormatRequest(long nativeCapturer,
         int width, int height, int fps);
   }