zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Create AndroidVideoBuffer and allow renderers to consume it.

Browse Source 
Bug: webrtc:7760
Change-Id: I3e3fddf48090ae27b226c65ddbb51f2c3d8dc544
Reviewed-on: https://chromium-review.googlesource.com/535638
Commit-Queue: Sami Kalliomäki <sakal@webrtc.org>
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#18624}
This commit is contained in:
Sami Kalliomäki
2017-06-16 10:08:23 +02:00
committed by Commit Bot
parent af99b6d67a
commit bc061b4280
5 changed files with 290 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
webrtc/sdk/android/src/jni/classreferenceholder.cc
View File

@ -66,33 +66,33 @@ ClassReferenceHolder::ClassReferenceHolder(JNIEnv* jni) {
LoadClass(jni, "org/webrtc/EglBase$Context");
LoadClass(jni, "org/webrtc/EglBase14$Context");
LoadClass(jni, "org/webrtc/IceCandidate");
LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder");
LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer");
LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer");
LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$VideoCodecType");
LoadClass(jni, "org/webrtc/MediaCodecVideoEncoder");
LoadClass(jni, "org/webrtc/MediaCodecVideoEncoder$OutputBufferInfo");
LoadClass(jni, "org/webrtc/MediaCodecVideoEncoder$VideoCodecType");
LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder");
LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer");
LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer");
LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$VideoCodecType");
LoadClass(jni, "org/webrtc/MediaSource$State");
LoadClass(jni, "org/webrtc/MediaStream");
LoadClass(jni, "org/webrtc/MediaStreamTrack$State");
LoadClass(jni, "org/webrtc/MediaStreamTrack$MediaType");
LoadClass(jni, "org/webrtc/MediaStreamTrack$State");
LoadClass(jni, "org/webrtc/NetworkMonitor");
LoadClass(jni, "org/webrtc/NetworkMonitorAutoDetect$ConnectionType");
LoadClass(jni, "org/webrtc/NetworkMonitorAutoDetect$IPAddress");
LoadClass(jni, "org/webrtc/NetworkMonitorAutoDetect$NetworkInformation");
LoadClass(jni, "org/webrtc/PeerConnectionFactory");
LoadClass(jni, "org/webrtc/PeerConnection$BundlePolicy");
LoadClass(jni, "org/webrtc/PeerConnection$CandidateNetworkPolicy");
LoadClass(jni, "org/webrtc/PeerConnection$ContinualGatheringPolicy");
LoadClass(jni, "org/webrtc/PeerConnection$RtcpMuxPolicy");
LoadClass(jni, "org/webrtc/PeerConnection$IceConnectionState");
LoadClass(jni, "org/webrtc/PeerConnection$IceGatheringState");
LoadClass(jni, "org/webrtc/PeerConnection$IceTransportsType");
LoadClass(jni, "org/webrtc/PeerConnection$KeyType");
LoadClass(jni, "org/webrtc/PeerConnection$RtcpMuxPolicy");
LoadClass(jni, "org/webrtc/PeerConnection$SignalingState");
LoadClass(jni, "org/webrtc/PeerConnection$TcpCandidatePolicy");
LoadClass(jni, "org/webrtc/PeerConnection$TlsCertPolicy");
LoadClass(jni, "org/webrtc/PeerConnection$CandidateNetworkPolicy");
LoadClass(jni, "org/webrtc/PeerConnection$KeyType");
LoadClass(jni, "org/webrtc/PeerConnection$SignalingState");
LoadClass(jni, "org/webrtc/PeerConnectionFactory");
LoadClass(jni, "org/webrtc/RTCStats");
LoadClass(jni, "org/webrtc/RTCStatsReport");
LoadClass(jni, "org/webrtc/RtpReceiver");
@ -103,6 +103,9 @@ ClassReferenceHolder::ClassReferenceHolder(JNIEnv* jni) {
LoadClass(jni, "org/webrtc/StatsReport$Value");
LoadClass(jni, "org/webrtc/SurfaceTextureHelper");
LoadClass(jni, "org/webrtc/VideoCapturer");
LoadClass(jni, "org/webrtc/VideoFrame");
LoadClass(jni, "org/webrtc/VideoFrame$Buffer");
LoadClass(jni, "org/webrtc/VideoFrame$I420Buffer");
LoadClass(jni, "org/webrtc/VideoRenderer$I420Frame");
LoadClass(jni, "org/webrtc/VideoTrack");
LoadClass(jni, "org/webrtc/WrappedNativeI420Buffer");

141
webrtc/sdk/android/src/jni/native_handle_impl.cc
View File

@ -17,7 +17,9 @@
#include "webrtc/base/keep_ref_until_done.h"
#include "webrtc/base/logging.h"
#include "webrtc/base/scoped_ref_ptr.h"
#include "webrtc/base/timeutils.h"
#include "webrtc/common_video/include/video_frame_buffer.h"
#include "webrtc/sdk/android/src/jni/classreferenceholder.h"
#include "webrtc/sdk/android/src/jni/jni_helpers.h"
#include "webrtc/system_wrappers/include/aligned_malloc.h"
@ -32,7 +34,25 @@ Matrix::Matrix(JNIEnv* jni, jfloatArray a) {
jni->ReleaseFloatArrayElements(a, ptr, 0);
}
jfloatArray Matrix::ToJava(JNIEnv* jni) {
Matrix Matrix::fromAndroidGraphicsMatrix(JNIEnv* jni, jobject j_matrix) {
jfloatArray array_3x3 = jni->NewFloatArray(9);
jclass j_matrix_class = jni->FindClass("android/graphics/Matrix");
jni->CallVoidMethod(j_matrix,
GetMethodID(jni, j_matrix_class, "getValues", "([F)V"),
array_3x3);
jfloat* array_3x3_ptr = jni->GetFloatArrayElements(array_3x3, nullptr);
Matrix matrix;
memset(matrix.elem_, 0, sizeof(matrix.elem_));
for (int y = 0; y < 3; ++y) {
for (int x = 0; x < 3; ++x) {
matrix.elem_[y * 4 + x] = array_3x3_ptr[x + y * 3];
}
}
matrix.elem_[3 + 3 * 3] = 1; // Bottom-right corner should be 1.
return matrix;
}
jfloatArray Matrix::ToJava(JNIEnv* jni) const {
jfloatArray matrix = jni->NewFloatArray(16);
jni->SetFloatArrayRegion(matrix, 0, 16, elem_);
return matrix;
@ -195,4 +215,123 @@ rtc::scoped_refptr<webrtc::I420BufferInterface> AndroidTextureBuffer::ToI420() {
return copy;
}
AndroidVideoBuffer::AndroidVideoBuffer(JNIEnv* jni,
jmethodID j_retain_id,
jmethodID j_release_id,
int width,
int height,
const Matrix& matrix,
jobject j_video_frame_buffer)
: j_release_id_(j_release_id),
width_(width),
height_(height),
matrix_(matrix),
j_video_frame_buffer_(jni, j_video_frame_buffer) {
jni->CallVoidMethod(j_video_frame_buffer, j_retain_id);
}
AndroidVideoBuffer::~AndroidVideoBuffer() {
JNIEnv* jni = AttachCurrentThreadIfNeeded();
jni->CallVoidMethod(*j_video_frame_buffer_, j_release_id_);
}
jobject AndroidVideoBuffer::video_frame_buffer() const {
return *j_video_frame_buffer_;
}
webrtc::VideoFrameBuffer::Type AndroidVideoBuffer::type() const {
return Type::kNative;
}
int AndroidVideoBuffer::width() const {
return width_;
}
int AndroidVideoBuffer::height() const {
return height_;
}
rtc::scoped_refptr<webrtc::I420BufferInterface> AndroidVideoBuffer::ToI420() {
// TODO(magjed): Implement using Java ToI420.
return nullptr;
}
jobject AndroidVideoBuffer::ToJavaI420Frame(JNIEnv* jni,
int width,
int height,
int rotation) {
jclass j_byte_buffer_class = jni->FindClass("java/nio/ByteBuffer");
jclass j_i420_frame_class =
FindClass(jni, "org/webrtc/VideoRenderer$I420Frame");
jmethodID j_i420_frame_ctor_id =
GetMethodID(jni, j_i420_frame_class, "<init>",
"(III[FLorg/webrtc/VideoFrame$Buffer;J)V");
// Java code just uses the native frame to hold a reference to the buffer so
// this is okay.
webrtc::VideoFrame* native_frame = new webrtc::VideoFrame(
this, 0 /* timestamp */, 0 /* render_time_ms */,
webrtc::VideoRotation::kVideoRotation_0 /* rotation */);
return jni->NewObject(j_i420_frame_class, j_i420_frame_ctor_id, width, height,
rotation, matrix_.ToJava(jni), *j_video_frame_buffer_,
jlongFromPointer(native_frame));
}
AndroidVideoBufferFactory::AndroidVideoBufferFactory(JNIEnv* jni)
: j_video_frame_class_(jni, FindClass(jni, "org/webrtc/VideoFrame")),
j_get_buffer_id_(GetMethodID(jni,
*j_video_frame_class_,
"getBuffer",
"()Lorg/webrtc/VideoFrame$Buffer;")),
j_get_width_id_(
GetMethodID(jni, *j_video_frame_class_, "getWidth", "()I")),
j_get_height_id_(
GetMethodID(jni, *j_video_frame_class_, "getHeight", "()I")),
j_get_rotation_id_(
GetMethodID(jni, *j_video_frame_class_, "getRotation", "()I")),
j_get_transform_matrix_id_(GetMethodID(jni,
*j_video_frame_class_,
"getTransformMatrix",
"()Landroid/graphics/Matrix;")),
j_get_timestamp_ns_id_(
GetMethodID(jni, *j_video_frame_class_, "getTimestampNs", "()J")),
j_video_frame_buffer_class_(
jni,
FindClass(jni, "org/webrtc/VideoFrame$Buffer")),
j_retain_id_(
GetMethodID(jni, *j_video_frame_buffer_class_, "retain", "()V")),
j_release_id_(
GetMethodID(jni, *j_video_frame_buffer_class_, "release", "()V")) {}
webrtc::VideoFrame AndroidVideoBufferFactory::CreateFrame(
JNIEnv* jni,
jobject j_video_frame,
uint32_t timestamp_rtp) const {
jobject j_video_frame_buffer =
jni->CallObjectMethod(j_video_frame, j_get_buffer_id_);
int width = jni->CallIntMethod(j_video_frame, j_get_width_id_);
int height = jni->CallIntMethod(j_video_frame, j_get_height_id_);
int rotation = jni->CallIntMethod(j_video_frame, j_get_rotation_id_);
jobject j_matrix =
jni->CallObjectMethod(j_video_frame, j_get_transform_matrix_id_);
Matrix matrix = Matrix::fromAndroidGraphicsMatrix(jni, j_matrix);
uint32_t timestamp_ns =
jni->CallLongMethod(j_video_frame, j_get_timestamp_ns_id_);
rtc::scoped_refptr<AndroidVideoBuffer> buffer =
CreateBuffer(width, height, matrix, j_video_frame_buffer);
return webrtc::VideoFrame(buffer, timestamp_rtp,
timestamp_ns / rtc::kNumNanosecsPerMillisec,
static_cast<webrtc::VideoRotation>(rotation));
}
rtc::scoped_refptr<AndroidVideoBuffer> AndroidVideoBufferFactory::CreateBuffer(
int width,
int height,
const Matrix& matrix,
jobject j_video_frame_buffer) const {
JNIEnv* jni = AttachCurrentThreadIfNeeded();
return new rtc::RefCountedObject<AndroidVideoBuffer>(
jni, j_retain_id_, j_release_id_, width, height, matrix,
j_video_frame_buffer);
}
} // namespace webrtc_jni

84
webrtc/sdk/android/src/jni/native_handle_impl.h
View File

@ -13,9 +13,11 @@
#include <jni.h>
#include "webrtc/api/video/video_frame.h"
#include "webrtc/api/video/video_frame_buffer.h"
#include "webrtc/api/video/video_rotation.h"
#include "webrtc/base/callback.h"
#include "webrtc/sdk/android/src/jni/jni_helpers.h"
namespace webrtc_jni {
@ -25,7 +27,9 @@ class Matrix {
public:
Matrix(JNIEnv* jni, jfloatArray a);
jfloatArray ToJava(JNIEnv* jni);
static Matrix fromAndroidGraphicsMatrix(JNIEnv* jni, jobject j_matrix);
jfloatArray ToJava(JNIEnv* jni) const;
// Crop arguments are relative to original size.
void Crop(float cropped_width,
@ -36,6 +40,8 @@ class Matrix {
void Rotate(webrtc::VideoRotation rotation);
private:
Matrix() {}
static void Multiply(const float a[16], const float b[16], float result[16]);
float elem_[16];
};
@ -52,7 +58,18 @@ struct NativeHandleImpl {
Matrix sampling_matrix;
};
class AndroidTextureBuffer : public webrtc::VideoFrameBuffer {
// Base class to differentiate between the old texture frames and the new
// Java-based frames.
// TODO(sakal): Remove this and AndroidTextureBuffer once they are no longer
// needed.
class AndroidVideoFrameBuffer : public webrtc::VideoFrameBuffer {
public:
enum class AndroidType { kTextureBuffer, kJavaBuffer };
virtual AndroidType android_type() = 0;
};
class AndroidTextureBuffer : public AndroidVideoFrameBuffer {
public:
AndroidTextureBuffer(int width,
int height,
@ -70,6 +87,8 @@ class AndroidTextureBuffer : public webrtc::VideoFrameBuffer {
rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override;
AndroidType android_type() override { return AndroidType::kTextureBuffer; }
const int width_;
const int height_;
NativeHandleImpl native_handle_;
@ -82,6 +101,67 @@ class AndroidTextureBuffer : public webrtc::VideoFrameBuffer {
rtc::Callback0<void> no_longer_used_cb_;
};
class AndroidVideoBuffer : public AndroidVideoFrameBuffer {
public:
AndroidVideoBuffer(JNIEnv* jni,
jmethodID j_retain_id,
jmethodID j_release_id,
int width,
int height,
const Matrix& matrix,
jobject j_video_frame_buffer);
~AndroidVideoBuffer() override;
jobject video_frame_buffer() const;
// Returns an instance of VideoRenderer.I420Frame (deprecated)
jobject ToJavaI420Frame(JNIEnv* jni, int width, int height, int rotation);
private:
Type type() const override;
int width() const override;
int height() const override;
rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override;
AndroidType android_type() override { return AndroidType::kJavaBuffer; }
const jmethodID j_release_id_;
const int width_;
const int height_;
const Matrix matrix_;
// Holds a VideoFrame.Buffer.
ScopedGlobalRef<jobject> j_video_frame_buffer_;
};
class AndroidVideoBufferFactory {
public:
explicit AndroidVideoBufferFactory(JNIEnv* jni);
webrtc::VideoFrame CreateFrame(JNIEnv* jni,
jobject j_video_frame,
uint32_t timestamp_rtp) const;
rtc::scoped_refptr<AndroidVideoBuffer> CreateBuffer(
int width,
int height,
const Matrix& matrix,
jobject j_video_frame_buffer) const;
private:
ScopedGlobalRef<jclass> j_video_frame_class_;
jmethodID j_get_buffer_id_;
jmethodID j_get_width_id_;
jmethodID j_get_height_id_;
jmethodID j_get_rotation_id_;
jmethodID j_get_transform_matrix_id_;
jmethodID j_get_timestamp_ns_id_;
ScopedGlobalRef<jclass> j_video_frame_buffer_class_;
jmethodID j_retain_id_;
jmethodID j_release_id_;
};
} // namespace webrtc_jni
#endif // WEBRTC_SDK_ANDROID_SRC_JNI_NATIVE_HANDLE_IMPL_H_

27
webrtc/sdk/android/src/jni/peerconnection_jni.cc
View File

@ -871,10 +871,29 @@ class JavaVideoRendererWrapper
void OnFrame(const webrtc::VideoFrame& video_frame) override {
ScopedLocalRefFrame local_ref_frame(jni());
jobject j_frame = (video_frame.video_frame_buffer()->type() ==
webrtc::VideoFrameBuffer::Type::kNative)
? ToJavaTextureFrame(&video_frame)
: ToJavaI420Frame(&video_frame);
jobject j_frame;
if (video_frame.video_frame_buffer()->type() ==
webrtc::VideoFrameBuffer::Type::kNative) {
AndroidVideoFrameBuffer* android_buffer =
static_cast<AndroidVideoFrameBuffer*>(
video_frame.video_frame_buffer().get());
switch (android_buffer->android_type()) {
case AndroidVideoFrameBuffer::AndroidType::kTextureBuffer:
j_frame = ToJavaTextureFrame(&video_frame);
break;
case AndroidVideoFrameBuffer::AndroidType::kJavaBuffer:
j_frame = static_cast<AndroidVideoBuffer*>(android_buffer)
->ToJavaI420Frame(jni(), video_frame.width(),
video_frame.height(),
video_frame.rotation());
break;
default:
RTC_NOTREACHED();
}
} else {
j_frame = ToJavaI420Frame(&video_frame);
}
// |j_callbacks_| is responsible for releasing |j_frame| with
// VideoRenderer.renderFrameDone().
jni()->CallVoidMethod(*j_callbacks_, j_render_frame_id_, j_frame);