zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use the underlying type of Java Video Buffer on Java -> C++ Frame Buffer

Browse Source 
Just like the C++ API, add a method in Java VideoFrame.Buffer that
describes the underlying implementation.
Use this method to properly select AndroidVideoBuffer
or AndroidVideoI420Buffer in Java -> C++ Video Frame Conversion.

Also, add a test case for WrappedNativeI420Buffer
in VideoFrameBufferTest for consistency.

Bug: webrtc:12602
Change-Id: I4c0444e8af6f6a1109bc514e7ab6c2214f1f6d60
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/223080
Commit-Queue: Byoungchan Lee <daniel.l@hpcnt.com>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Xavier Lepaul‎ <xalep@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#34545}
This commit is contained in:
Byoungchan Lee
2021-07-24 06:16:20 +09:00
committed by WebRTC LUCI CQ
parent d2b885fd91
commit f740c252e7
7 changed files with 151 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
api/video/BUILD.gn
View File

@ -69,6 +69,12 @@ rtc_library("video_frame") {
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
if (is_android) {
java_cpp_enum("video_frame_enums") {
sources = [ "video_frame_buffer.h" ]
}
}
rtc_library("video_frame_i010") {
visibility = [ "*" ]
sources = [

2
api/video/video_frame_buffer.h
View File

@ -46,6 +46,8 @@ class RTC_EXPORT VideoFrameBuffer : public rtc::RefCountInterface {
// New frame buffer types will be added conservatively when there is an
// opportunity to optimize the path between some pair of video source and
// video sink.
// GENERATED_JAVA_ENUM_PACKAGE: org.webrtc
// GENERATED_JAVA_CLASS_NAME_OVERRIDE: VideoFrameBufferType
enum class Type {
kNative,
kI420,

9
sdk/android/BUILD.gn
View File

@ -202,7 +202,9 @@ if (is_android) {
":base_java",
"//rtc_base:base_java",
"//third_party/android_deps:com_android_support_support_annotations_java",
"//third_party/androidx:androidx_annotation_annotation_java",
]
srcjar_deps = [ "//api/video:video_frame_enums" ]
}
rtc_android_library("video_java") {
@ -1486,11 +1488,16 @@ if (is_android) {
rtc_library("instrumentationtests_jni") {
testonly = true
sources = [ "instrumentationtests/loggable_test.cc" ]
sources = [
"instrumentationtests/loggable_test.cc",
"instrumentationtests/video_frame_buffer_test.cc",
]
deps = [
":base_jni",
":native_api_jni",
":videoframe_jni",
"../../api/video:video_frame",
"../../rtc_base:rtc_base_approved",
]
}

14
sdk/android/api/org/webrtc/VideoFrame.java
View File

@ -34,6 +34,15 @@ public class VideoFrame implements RefCounted {
* and the buffer needs to be returned to the VideoSource as soon as all references are gone.
*/
public interface Buffer extends RefCounted {
/**
* Representation of the underlying buffer. Currently, only NATIVE and I420 are supported.
*/
@CalledByNative("Buffer")
@VideoFrameBufferType
default int getBufferType() {
return VideoFrameBufferType.NATIVE;
}
/**
* Resolution of the buffer in pixels.
*/
@ -63,6 +72,11 @@ public class VideoFrame implements RefCounted {
* Interface for I420 buffers.
*/
public interface I420Buffer extends Buffer {
@Override
default int getBufferType() {
return VideoFrameBufferType.I420;
}
/**
* Returns a direct ByteBuffer containing Y-plane data. The buffer capacity is at least
* getStrideY() * getHeight() bytes. The position of the returned buffer is ignored and must

44
sdk/android/instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java
View File

@ -32,6 +32,7 @@ import org.chromium.base.test.params.ParameterizedRunner;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.webrtc.VideoFrame;
/**
* Test VideoFrame buffers of different kind of formats: I420, RGB, OES, NV12, NV21, and verify
@ -45,7 +46,7 @@ public class VideoFrameBufferTest {
/**
* These tests are parameterized on this enum which represents the different VideoFrame.Buffers.
*/
private static enum BufferType { I420, RGB_TEXTURE, OES_TEXTURE, NV21, NV12 }
private static enum BufferType { I420_JAVA, I420_NATIVE, RGB_TEXTURE, OES_TEXTURE, NV21, NV12 }
@ClassParameter private static List<ParameterSet> CLASS_PARAMS = new ArrayList<>();
@ -75,17 +76,39 @@ public class VideoFrameBufferTest {
*/
private static VideoFrame.Buffer createBufferWithType(
BufferType bufferType, VideoFrame.I420Buffer i420Buffer) {
VideoFrame.Buffer buffer;
switch (bufferType) {
case I420:
return i420Buffer.toI420();
case I420_JAVA:
buffer = i420Buffer;
buffer.retain();
assertEquals(VideoFrameBufferType.I420, buffer.getBufferType());
assertEquals(VideoFrameBufferType.I420, nativeGetBufferType(buffer));
return buffer;
case I420_NATIVE:
buffer = nativeGetNativeI420Buffer(i420Buffer);
assertEquals(VideoFrameBufferType.I420, buffer.getBufferType());
assertEquals(VideoFrameBufferType.I420, nativeGetBufferType(buffer));
return buffer;
case RGB_TEXTURE:
return createRgbTextureBuffer(/* eglContext= */ null, i420Buffer);
buffer = createRgbTextureBuffer(/* eglContext= */ null, i420Buffer);
assertEquals(VideoFrameBufferType.NATIVE, buffer.getBufferType());
assertEquals(VideoFrameBufferType.NATIVE, nativeGetBufferType(buffer));
return buffer;
case OES_TEXTURE:
return createOesTextureBuffer(/* eglContext= */ null, i420Buffer);
buffer = createOesTextureBuffer(/* eglContext= */ null, i420Buffer);
assertEquals(VideoFrameBufferType.NATIVE, buffer.getBufferType());
assertEquals(VideoFrameBufferType.NATIVE, nativeGetBufferType(buffer));
return buffer;
case NV21:
return createNV21Buffer(i420Buffer);
buffer = createNV21Buffer(i420Buffer);
assertEquals(VideoFrameBufferType.NATIVE, buffer.getBufferType());
assertEquals(VideoFrameBufferType.NATIVE, nativeGetBufferType(buffer));
return buffer;
case NV12:
return createNV12Buffer(i420Buffer);
buffer = createNV12Buffer(i420Buffer);
assertEquals(VideoFrameBufferType.NATIVE, buffer.getBufferType());
assertEquals(VideoFrameBufferType.NATIVE, nativeGetBufferType(buffer));
return buffer;
default:
throw new IllegalArgumentException("Unknown buffer type: " + bufferType);
}
@ -452,6 +475,7 @@ public class VideoFrameBufferTest {
final VideoFrame.I420Buffer outputI420Buffer = bufferToTest.toI420();
bufferToTest.release();
assertEquals(VideoFrameBufferType.I420, nativeGetBufferType(outputI420Buffer));
assertAlmostEqualI420Buffers(referenceI420Buffer, outputI420Buffer);
referenceI420Buffer.release();
outputI420Buffer.release();
@ -504,4 +528,10 @@ public class VideoFrameBufferTest {
testCropAndScale(4 /* cropX= */, 4 /* cropY= */, /* cropWidth= */ 12, /* cropHeight= */ 12,
/* scaleWidth= */ 8, /* scaleHeight= */ 8);
}
@VideoFrameBufferType private static native int nativeGetBufferType(VideoFrame.Buffer buffer);
/** Returns the copy of I420Buffer using WrappedNativeI420Buffer. */
private static native VideoFrame.Buffer nativeGetNativeI420Buffer(
VideoFrame.I420Buffer i420Buffer);
}

45
sdk/android/instrumentationtests/video_frame_buffer_test.cc Normal file
View File

@ -0,0 +1,45 @@
/*
* Copyright 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/video/i420_buffer.h"
#include "sdk/android/src/jni/jni_helpers.h"
#include "sdk/android/src/jni/video_frame.h"
#include "sdk/android/src/jni/wrapped_native_i420_buffer.h"
namespace webrtc {
namespace jni {
JNI_FUNCTION_DECLARATION(jint,
VideoFrameBufferTest_nativeGetBufferType,
JNIEnv* jni,
jclass,
jobject video_frame_buffer) {
const JavaParamRef<jobject> j_video_frame_buffer(video_frame_buffer);
rtc::scoped_refptr<VideoFrameBuffer> buffer =
JavaToNativeFrameBuffer(jni, j_video_frame_buffer);
return static_cast<jint>(buffer->type());
}
JNI_FUNCTION_DECLARATION(jobject,
VideoFrameBufferTest_nativeGetNativeI420Buffer,
JNIEnv* jni,
jclass,
jobject i420_buffer) {
const JavaParamRef<jobject> j_i420_buffer(i420_buffer);
rtc::scoped_refptr<VideoFrameBuffer> buffer =
JavaToNativeFrameBuffer(jni, j_i420_buffer);
const I420BufferInterface* inputBuffer = buffer->GetI420();
RTC_DCHECK(inputBuffer != nullptr);
rtc::scoped_refptr<I420Buffer> outputBuffer = I420Buffer::Copy(*inputBuffer);
return WrapI420Buffer(jni, outputBuffer).Release();
}
} // namespace jni
} // namespace webrtc

42
sdk/android/src/jni/video_frame.cc
View File

@ -70,6 +70,13 @@ class AndroidVideoBuffer : public VideoFrameBuffer {
class AndroidVideoI420Buffer : public I420BufferInterface {
public:
// Creates a native VideoFrameBuffer from a Java VideoFrame.I420Buffer.
static rtc::scoped_refptr<AndroidVideoI420Buffer> Create(
JNIEnv* jni,
int width,
int height,
const JavaRef<jobject>& j_video_frame_buffer);
// Adopts and takes ownership of the Java VideoFrame.Buffer. I.e. retain()
// will not be called, but release() will be called when the returned
// AndroidVideoBuffer is destroyed.
@ -113,11 +120,24 @@ class AndroidVideoI420Buffer : public I420BufferInterface {
int stride_v_;
};
rtc::scoped_refptr<AndroidVideoI420Buffer> AndroidVideoI420Buffer::Create(
JNIEnv* jni,
int width,
int height,
const JavaRef<jobject>& j_video_frame_buffer) {
Java_Buffer_retain(jni, j_video_frame_buffer);
return AndroidVideoI420Buffer::Adopt(jni, width, height,
j_video_frame_buffer);
}
rtc::scoped_refptr<AndroidVideoI420Buffer> AndroidVideoI420Buffer::Adopt(
JNIEnv* jni,
int width,
int height,
const JavaRef<jobject>& j_video_frame_buffer) {
RTC_DCHECK_EQ(
static_cast<Type>(Java_Buffer_getBufferType(jni, j_video_frame_buffer)),
Type::kI420);
return rtc::make_ref_counted<AndroidVideoI420Buffer>(jni, width, height,
j_video_frame_buffer);
}
@ -164,6 +184,9 @@ int64_t GetJavaVideoFrameTimestampNs(JNIEnv* jni,
rtc::scoped_refptr<AndroidVideoBuffer> AndroidVideoBuffer::Adopt(
JNIEnv* jni,
const JavaRef<jobject>& j_video_frame_buffer) {
RTC_DCHECK_EQ(
static_cast<Type>(Java_Buffer_getBufferType(jni, j_video_frame_buffer)),
Type::kNative);
return rtc::make_ref_counted<AndroidVideoBuffer>(jni, j_video_frame_buffer);
}
@ -233,7 +256,20 @@ rtc::scoped_refptr<I420BufferInterface> AndroidVideoBuffer::ToI420() {
rtc::scoped_refptr<VideoFrameBuffer> JavaToNativeFrameBuffer(
JNIEnv* jni,
const JavaRef<jobject>& j_video_frame_buffer) {
return AndroidVideoBuffer::Create(jni, j_video_frame_buffer);
VideoFrameBuffer::Type type = static_cast<VideoFrameBuffer::Type>(
Java_Buffer_getBufferType(jni, j_video_frame_buffer));
switch (type) {
case VideoFrameBuffer::Type::kI420: {
const int width = Java_Buffer_getWidth(jni, j_video_frame_buffer);
const int height = Java_Buffer_getHeight(jni, j_video_frame_buffer);
return AndroidVideoI420Buffer::Create(jni, width, height,
j_video_frame_buffer);
}
case VideoFrameBuffer::Type::kNative:
return AndroidVideoBuffer::Create(jni, j_video_frame_buffer);
default:
RTC_CHECK_NOTREACHED();
}
}
VideoFrame JavaToNativeFrame(JNIEnv* jni,
@ -243,8 +279,8 @@ VideoFrame JavaToNativeFrame(JNIEnv* jni,
Java_VideoFrame_getBuffer(jni, j_video_frame);
int rotation = Java_VideoFrame_getRotation(jni, j_video_frame);
int64_t timestamp_ns = Java_VideoFrame_getTimestampNs(jni, j_video_frame);
rtc::scoped_refptr<AndroidVideoBuffer> buffer =
AndroidVideoBuffer::Create(jni, j_video_frame_buffer);
rtc::scoped_refptr<VideoFrameBuffer> buffer =
JavaToNativeFrameBuffer(jni, j_video_frame_buffer);
return VideoFrame::Builder()
.set_video_frame_buffer(buffer)
.set_timestamp_rtp(timestamp_rtp)