Android: Add support for cropping textures

BUG=b/28622232
R=glaznev@webrtc.org, nisse@webrtc.org

Review URL: https://codereview.webrtc.org/1965953003 .

Cr-Commit-Position: refs/heads/master@{#12720}

webrtc/api/androidvideocapturer.cc

@@ -88,14 +88,12 @@ class AndroidVideoCapturer::FrameFactory : public cricket::VideoFrameFactory {
       int output_width,
       int output_height) const override {
     if (buffer_->native_handle() != nullptr) {
-      // TODO(perkj) Implement cropping.
-      RTC_CHECK_EQ(cropped_input_width, buffer_->width());
-      RTC_CHECK_EQ(cropped_input_height, buffer_->height());
       rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled_buffer(
           static_cast<webrtc_jni::AndroidTextureBuffer*>(buffer_.get())
-              ->ScaleAndRotate(output_width, output_height,
+              ->CropScaleAndRotate(cropped_input_width, cropped_input_height,
-                               apply_rotation_ ? input_frame->rotation :
+                                   output_width, output_height,
-                                   webrtc::kVideoRotation_0));
+                                   apply_rotation_ ? input_frame->rotation
+                                                   : webrtc::kVideoRotation_0));
       return new cricket::WebRtcVideoFrame(
           scaled_buffer, input_frame->time_stamp,
           apply_rotation_ ? webrtc::kVideoRotation_0 : input_frame->rotation);

webrtc/api/java/jni/androidmediaencoder_jni.cc

@@ -683,9 +683,9 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
       if (frame.video_frame_buffer()->native_handle() != nullptr) {
         rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled_buffer(
             static_cast<AndroidTextureBuffer*>(
-                frame.video_frame_buffer().get())->ScaleAndRotate(
+                frame.video_frame_buffer().get())->CropScaleAndRotate(
-                    scaled_resolution.width,
+                    frame.width(), frame.height(),
-                    scaled_resolution.height,
+                    scaled_resolution.width, scaled_resolution.height,
                     webrtc::kVideoRotation_0));
         input_frame.set_video_frame_buffer(scaled_buffer);
       } else {

webrtc/api/java/jni/native_handle_impl.cc

@@ -59,6 +59,37 @@ void RotateMatrix(float a[16], webrtc::VideoRotation rotation) {
   }
 }
 
+// Calculates result = a * b, in column-major order.
+void MultiplyMatrix(const float a[16], const float b[16], float result[16]) {
+  for (int i = 0; i < 4; ++i) {
+    for (int j = 0; j < 4; ++j) {
+      float sum = 0;
+      for (int k = 0; k < 4; ++k) {
+        sum += a[k * 4 + j] * b[i * 4 + k];
+      }
+      result[i * 4 + j] = sum;
+    }
+  }
+}
+
+// Center crop by keeping xFraction of the width and yFraction of the height,
+// so e.g. cropping from 640x480 to 640x360 would use
+// xFraction=1, yFraction=360/480.
+void CropMatrix(float a[16], float xFraction, float yFraction) {
+  // Move cropped area to the center of the frame by offsetting half the
+  // removed area.
+  const float xOffset = (1 - xFraction) / 2;
+  const float yOffset = (1 - yFraction) / 2;
+  const float crop_matrix[16] = {
+    xFraction, 0, 0, 0,
+    0, yFraction, 0, 0,
+    0, 0, 1, 0,
+    xOffset, yOffset, 0, 1};
+  float mul_result[16];
+  MultiplyMatrix(crop_matrix, a, mul_result);
+  memcpy(a, mul_result, sizeof(mul_result));
+}
+
 }  // anonymouse namespace
 
 namespace webrtc_jni {
@@ -146,15 +177,18 @@ AndroidTextureBuffer::NativeToI420Buffer() {
 }
 
 rtc::scoped_refptr<AndroidTextureBuffer>
-AndroidTextureBuffer::ScaleAndRotate(int dst_widht,
+AndroidTextureBuffer::CropScaleAndRotate(int cropped_width,
+                                         int cropped_height,
+                                         int dst_width,
                                          int dst_height,
                                          webrtc::VideoRotation rotation) {
-  if (width() == dst_widht && height() == dst_height &&
+  if (cropped_width == dst_width && cropped_height == dst_height &&
+      width() == dst_width && height() == dst_height &&
       rotation == webrtc::kVideoRotation_0) {
     return this;
   }
-  int rotated_width = (rotation % 180 == 0) ? dst_widht : dst_height;
+  int rotated_width = (rotation % 180 == 0) ? dst_width : dst_height;
-  int rotated_height = (rotation % 180 == 0) ? dst_height : dst_widht;
+  int rotated_height = (rotation % 180 == 0) ? dst_height : dst_width;
 
   // Here we use Bind magic to add a reference count to |this| until the newly
   // created AndroidTextureBuffer is destructed
@@ -163,6 +197,11 @@ AndroidTextureBuffer::ScaleAndRotate(int dst_widht,
           rotated_width, rotated_height, native_handle_,
           surface_texture_helper_, rtc::KeepRefUntilDone(this)));
 
+  if (cropped_width != width() || cropped_height != height()) {
+    CropMatrix(buffer->native_handle_.sampling_matrix,
+               cropped_width / static_cast<float>(width()),
+               cropped_height / static_cast<float>(height()));
+  }
   RotateMatrix(buffer->native_handle_.sampling_matrix, rotation);
   return buffer;
 }

webrtc/api/java/jni/native_handle_impl.h

@@ -38,8 +38,11 @@ class AndroidTextureBuffer : public webrtc::NativeHandleBuffer {
   ~AndroidTextureBuffer();
   rtc::scoped_refptr<VideoFrameBuffer> NativeToI420Buffer() override;
 
-  rtc::scoped_refptr<AndroidTextureBuffer> ScaleAndRotate(
+  // First crop, then scale to dst resolution, and then rotate.
-      int dst_widht,
+  rtc::scoped_refptr<AndroidTextureBuffer> CropScaleAndRotate(
+      int cropped_width,
+      int cropped_height,
+      int dst_width,
       int dst_height,
       webrtc::VideoRotation rotation);