diff --git a/webrtc/sdk/android/api/org/webrtc/VideoRenderer.java b/webrtc/sdk/android/api/org/webrtc/VideoRenderer.java index bfa4f67d4b..9dde095242 100644 --- a/webrtc/sdk/android/api/org/webrtc/VideoRenderer.java +++ b/webrtc/sdk/android/api/org/webrtc/VideoRenderer.java @@ -89,6 +89,38 @@ public class VideoRenderer { } } + /** + * Construct a frame of the given dimensions from VideoFrame.Buffer. + */ + public I420Frame(int width, int height, int rotationDegree, float[] samplingMatrix, + VideoFrame.Buffer buffer, long nativeFramePointer) { + this.width = width; + this.height = height; + this.rotationDegree = rotationDegree; + if (rotationDegree % 90 != 0) { + throw new IllegalArgumentException("Rotation degree not multiple of 90: " + rotationDegree); + } + this.samplingMatrix = samplingMatrix; + if (buffer instanceof VideoFrame.TextureBuffer) { + VideoFrame.TextureBuffer textureBuffer = (VideoFrame.TextureBuffer) buffer; + this.yuvFrame = false; + this.textureId = textureBuffer.getTextureId(); + + this.yuvStrides = null; + this.yuvPlanes = null; + } else { + VideoFrame.I420Buffer i420Buffer = buffer.toI420(); + this.yuvFrame = true; + this.yuvStrides = + new int[] {i420Buffer.getStrideY(), i420Buffer.getStrideU(), i420Buffer.getStrideV()}; + this.yuvPlanes = + new ByteBuffer[] {i420Buffer.getDataY(), i420Buffer.getDataU(), i420Buffer.getDataV()}; + + this.textureId = 0; + } + this.nativeFramePointer = nativeFramePointer; + } + public int rotatedWidth() { return (rotationDegree % 180 == 0) ? width : height; } diff --git a/webrtc/sdk/android/src/jni/classreferenceholder.cc b/webrtc/sdk/android/src/jni/classreferenceholder.cc index 31c25e42f5..01e1521b54 100644 --- a/webrtc/sdk/android/src/jni/classreferenceholder.cc +++ b/webrtc/sdk/android/src/jni/classreferenceholder.cc @@ -66,33 +66,33 @@ ClassReferenceHolder::ClassReferenceHolder(JNIEnv* jni) { LoadClass(jni, "org/webrtc/EglBase$Context"); LoadClass(jni, "org/webrtc/EglBase14$Context"); LoadClass(jni, "org/webrtc/IceCandidate"); + LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder"); + LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer"); + LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer"); + LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$VideoCodecType"); LoadClass(jni, "org/webrtc/MediaCodecVideoEncoder"); LoadClass(jni, "org/webrtc/MediaCodecVideoEncoder$OutputBufferInfo"); LoadClass(jni, "org/webrtc/MediaCodecVideoEncoder$VideoCodecType"); - LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder"); - LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer"); - LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer"); - LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$VideoCodecType"); LoadClass(jni, "org/webrtc/MediaSource$State"); LoadClass(jni, "org/webrtc/MediaStream"); - LoadClass(jni, "org/webrtc/MediaStreamTrack$State"); LoadClass(jni, "org/webrtc/MediaStreamTrack$MediaType"); + LoadClass(jni, "org/webrtc/MediaStreamTrack$State"); LoadClass(jni, "org/webrtc/NetworkMonitor"); LoadClass(jni, "org/webrtc/NetworkMonitorAutoDetect$ConnectionType"); LoadClass(jni, "org/webrtc/NetworkMonitorAutoDetect$IPAddress"); LoadClass(jni, "org/webrtc/NetworkMonitorAutoDetect$NetworkInformation"); - LoadClass(jni, "org/webrtc/PeerConnectionFactory"); LoadClass(jni, "org/webrtc/PeerConnection$BundlePolicy"); + LoadClass(jni, "org/webrtc/PeerConnection$CandidateNetworkPolicy"); LoadClass(jni, "org/webrtc/PeerConnection$ContinualGatheringPolicy"); - LoadClass(jni, "org/webrtc/PeerConnection$RtcpMuxPolicy"); LoadClass(jni, "org/webrtc/PeerConnection$IceConnectionState"); LoadClass(jni, "org/webrtc/PeerConnection$IceGatheringState"); LoadClass(jni, "org/webrtc/PeerConnection$IceTransportsType"); + LoadClass(jni, "org/webrtc/PeerConnection$KeyType"); + LoadClass(jni, "org/webrtc/PeerConnection$RtcpMuxPolicy"); + LoadClass(jni, "org/webrtc/PeerConnection$SignalingState"); LoadClass(jni, "org/webrtc/PeerConnection$TcpCandidatePolicy"); LoadClass(jni, "org/webrtc/PeerConnection$TlsCertPolicy"); - LoadClass(jni, "org/webrtc/PeerConnection$CandidateNetworkPolicy"); - LoadClass(jni, "org/webrtc/PeerConnection$KeyType"); - LoadClass(jni, "org/webrtc/PeerConnection$SignalingState"); + LoadClass(jni, "org/webrtc/PeerConnectionFactory"); LoadClass(jni, "org/webrtc/RTCStats"); LoadClass(jni, "org/webrtc/RTCStatsReport"); LoadClass(jni, "org/webrtc/RtpReceiver"); @@ -103,6 +103,9 @@ ClassReferenceHolder::ClassReferenceHolder(JNIEnv* jni) { LoadClass(jni, "org/webrtc/StatsReport$Value"); LoadClass(jni, "org/webrtc/SurfaceTextureHelper"); LoadClass(jni, "org/webrtc/VideoCapturer"); + LoadClass(jni, "org/webrtc/VideoFrame"); + LoadClass(jni, "org/webrtc/VideoFrame$Buffer"); + LoadClass(jni, "org/webrtc/VideoFrame$I420Buffer"); LoadClass(jni, "org/webrtc/VideoRenderer$I420Frame"); LoadClass(jni, "org/webrtc/VideoTrack"); LoadClass(jni, "org/webrtc/WrappedNativeI420Buffer"); diff --git a/webrtc/sdk/android/src/jni/native_handle_impl.cc b/webrtc/sdk/android/src/jni/native_handle_impl.cc index 0f93cbdc26..7d028c0676 100644 --- a/webrtc/sdk/android/src/jni/native_handle_impl.cc +++ b/webrtc/sdk/android/src/jni/native_handle_impl.cc @@ -17,7 +17,9 @@ #include "webrtc/base/keep_ref_until_done.h" #include "webrtc/base/logging.h" #include "webrtc/base/scoped_ref_ptr.h" +#include "webrtc/base/timeutils.h" #include "webrtc/common_video/include/video_frame_buffer.h" +#include "webrtc/sdk/android/src/jni/classreferenceholder.h" #include "webrtc/sdk/android/src/jni/jni_helpers.h" #include "webrtc/system_wrappers/include/aligned_malloc.h" @@ -32,7 +34,25 @@ Matrix::Matrix(JNIEnv* jni, jfloatArray a) { jni->ReleaseFloatArrayElements(a, ptr, 0); } -jfloatArray Matrix::ToJava(JNIEnv* jni) { +Matrix Matrix::fromAndroidGraphicsMatrix(JNIEnv* jni, jobject j_matrix) { + jfloatArray array_3x3 = jni->NewFloatArray(9); + jclass j_matrix_class = jni->FindClass("android/graphics/Matrix"); + jni->CallVoidMethod(j_matrix, + GetMethodID(jni, j_matrix_class, "getValues", "([F)V"), + array_3x3); + jfloat* array_3x3_ptr = jni->GetFloatArrayElements(array_3x3, nullptr); + Matrix matrix; + memset(matrix.elem_, 0, sizeof(matrix.elem_)); + for (int y = 0; y < 3; ++y) { + for (int x = 0; x < 3; ++x) { + matrix.elem_[y * 4 + x] = array_3x3_ptr[x + y * 3]; + } + } + matrix.elem_[3 + 3 * 3] = 1; // Bottom-right corner should be 1. + return matrix; +} + +jfloatArray Matrix::ToJava(JNIEnv* jni) const { jfloatArray matrix = jni->NewFloatArray(16); jni->SetFloatArrayRegion(matrix, 0, 16, elem_); return matrix; @@ -195,4 +215,123 @@ rtc::scoped_refptr AndroidTextureBuffer::ToI420() { return copy; } +AndroidVideoBuffer::AndroidVideoBuffer(JNIEnv* jni, + jmethodID j_retain_id, + jmethodID j_release_id, + int width, + int height, + const Matrix& matrix, + jobject j_video_frame_buffer) + : j_release_id_(j_release_id), + width_(width), + height_(height), + matrix_(matrix), + j_video_frame_buffer_(jni, j_video_frame_buffer) { + jni->CallVoidMethod(j_video_frame_buffer, j_retain_id); +} + +AndroidVideoBuffer::~AndroidVideoBuffer() { + JNIEnv* jni = AttachCurrentThreadIfNeeded(); + jni->CallVoidMethod(*j_video_frame_buffer_, j_release_id_); +} + +jobject AndroidVideoBuffer::video_frame_buffer() const { + return *j_video_frame_buffer_; +} + +webrtc::VideoFrameBuffer::Type AndroidVideoBuffer::type() const { + return Type::kNative; +} + +int AndroidVideoBuffer::width() const { + return width_; +} + +int AndroidVideoBuffer::height() const { + return height_; +} + +rtc::scoped_refptr AndroidVideoBuffer::ToI420() { + // TODO(magjed): Implement using Java ToI420. + return nullptr; +} + +jobject AndroidVideoBuffer::ToJavaI420Frame(JNIEnv* jni, + int width, + int height, + int rotation) { + jclass j_byte_buffer_class = jni->FindClass("java/nio/ByteBuffer"); + jclass j_i420_frame_class = + FindClass(jni, "org/webrtc/VideoRenderer$I420Frame"); + jmethodID j_i420_frame_ctor_id = + GetMethodID(jni, j_i420_frame_class, "", + "(III[FLorg/webrtc/VideoFrame$Buffer;J)V"); + // Java code just uses the native frame to hold a reference to the buffer so + // this is okay. + webrtc::VideoFrame* native_frame = new webrtc::VideoFrame( + this, 0 /* timestamp */, 0 /* render_time_ms */, + webrtc::VideoRotation::kVideoRotation_0 /* rotation */); + return jni->NewObject(j_i420_frame_class, j_i420_frame_ctor_id, width, height, + rotation, matrix_.ToJava(jni), *j_video_frame_buffer_, + jlongFromPointer(native_frame)); +} + +AndroidVideoBufferFactory::AndroidVideoBufferFactory(JNIEnv* jni) + : j_video_frame_class_(jni, FindClass(jni, "org/webrtc/VideoFrame")), + j_get_buffer_id_(GetMethodID(jni, + *j_video_frame_class_, + "getBuffer", + "()Lorg/webrtc/VideoFrame$Buffer;")), + j_get_width_id_( + GetMethodID(jni, *j_video_frame_class_, "getWidth", "()I")), + j_get_height_id_( + GetMethodID(jni, *j_video_frame_class_, "getHeight", "()I")), + j_get_rotation_id_( + GetMethodID(jni, *j_video_frame_class_, "getRotation", "()I")), + j_get_transform_matrix_id_(GetMethodID(jni, + *j_video_frame_class_, + "getTransformMatrix", + "()Landroid/graphics/Matrix;")), + j_get_timestamp_ns_id_( + GetMethodID(jni, *j_video_frame_class_, "getTimestampNs", "()J")), + j_video_frame_buffer_class_( + jni, + FindClass(jni, "org/webrtc/VideoFrame$Buffer")), + j_retain_id_( + GetMethodID(jni, *j_video_frame_buffer_class_, "retain", "()V")), + j_release_id_( + GetMethodID(jni, *j_video_frame_buffer_class_, "release", "()V")) {} + +webrtc::VideoFrame AndroidVideoBufferFactory::CreateFrame( + JNIEnv* jni, + jobject j_video_frame, + uint32_t timestamp_rtp) const { + jobject j_video_frame_buffer = + jni->CallObjectMethod(j_video_frame, j_get_buffer_id_); + int width = jni->CallIntMethod(j_video_frame, j_get_width_id_); + int height = jni->CallIntMethod(j_video_frame, j_get_height_id_); + int rotation = jni->CallIntMethod(j_video_frame, j_get_rotation_id_); + jobject j_matrix = + jni->CallObjectMethod(j_video_frame, j_get_transform_matrix_id_); + Matrix matrix = Matrix::fromAndroidGraphicsMatrix(jni, j_matrix); + uint32_t timestamp_ns = + jni->CallLongMethod(j_video_frame, j_get_timestamp_ns_id_); + rtc::scoped_refptr buffer = + CreateBuffer(width, height, matrix, j_video_frame_buffer); + return webrtc::VideoFrame(buffer, timestamp_rtp, + timestamp_ns / rtc::kNumNanosecsPerMillisec, + static_cast(rotation)); +} + +rtc::scoped_refptr AndroidVideoBufferFactory::CreateBuffer( + int width, + int height, + const Matrix& matrix, + jobject j_video_frame_buffer) const { + JNIEnv* jni = AttachCurrentThreadIfNeeded(); + return new rtc::RefCountedObject( + jni, j_retain_id_, j_release_id_, width, height, matrix, + j_video_frame_buffer); +} + } // namespace webrtc_jni diff --git a/webrtc/sdk/android/src/jni/native_handle_impl.h b/webrtc/sdk/android/src/jni/native_handle_impl.h index b12c3c4651..ea83f62627 100644 --- a/webrtc/sdk/android/src/jni/native_handle_impl.h +++ b/webrtc/sdk/android/src/jni/native_handle_impl.h @@ -13,9 +13,11 @@ #include +#include "webrtc/api/video/video_frame.h" #include "webrtc/api/video/video_frame_buffer.h" #include "webrtc/api/video/video_rotation.h" #include "webrtc/base/callback.h" +#include "webrtc/sdk/android/src/jni/jni_helpers.h" namespace webrtc_jni { @@ -25,7 +27,9 @@ class Matrix { public: Matrix(JNIEnv* jni, jfloatArray a); - jfloatArray ToJava(JNIEnv* jni); + static Matrix fromAndroidGraphicsMatrix(JNIEnv* jni, jobject j_matrix); + + jfloatArray ToJava(JNIEnv* jni) const; // Crop arguments are relative to original size. void Crop(float cropped_width, @@ -36,6 +40,8 @@ class Matrix { void Rotate(webrtc::VideoRotation rotation); private: + Matrix() {} + static void Multiply(const float a[16], const float b[16], float result[16]); float elem_[16]; }; @@ -52,7 +58,18 @@ struct NativeHandleImpl { Matrix sampling_matrix; }; -class AndroidTextureBuffer : public webrtc::VideoFrameBuffer { +// Base class to differentiate between the old texture frames and the new +// Java-based frames. +// TODO(sakal): Remove this and AndroidTextureBuffer once they are no longer +// needed. +class AndroidVideoFrameBuffer : public webrtc::VideoFrameBuffer { + public: + enum class AndroidType { kTextureBuffer, kJavaBuffer }; + + virtual AndroidType android_type() = 0; +}; + +class AndroidTextureBuffer : public AndroidVideoFrameBuffer { public: AndroidTextureBuffer(int width, int height, @@ -70,6 +87,8 @@ class AndroidTextureBuffer : public webrtc::VideoFrameBuffer { rtc::scoped_refptr ToI420() override; + AndroidType android_type() override { return AndroidType::kTextureBuffer; } + const int width_; const int height_; NativeHandleImpl native_handle_; @@ -82,6 +101,67 @@ class AndroidTextureBuffer : public webrtc::VideoFrameBuffer { rtc::Callback0 no_longer_used_cb_; }; +class AndroidVideoBuffer : public AndroidVideoFrameBuffer { + public: + AndroidVideoBuffer(JNIEnv* jni, + jmethodID j_retain_id, + jmethodID j_release_id, + int width, + int height, + const Matrix& matrix, + jobject j_video_frame_buffer); + ~AndroidVideoBuffer() override; + + jobject video_frame_buffer() const; + + // Returns an instance of VideoRenderer.I420Frame (deprecated) + jobject ToJavaI420Frame(JNIEnv* jni, int width, int height, int rotation); + + private: + Type type() const override; + int width() const override; + int height() const override; + + rtc::scoped_refptr ToI420() override; + + AndroidType android_type() override { return AndroidType::kJavaBuffer; } + + const jmethodID j_release_id_; + const int width_; + const int height_; + const Matrix matrix_; + // Holds a VideoFrame.Buffer. + ScopedGlobalRef j_video_frame_buffer_; +}; + +class AndroidVideoBufferFactory { + public: + explicit AndroidVideoBufferFactory(JNIEnv* jni); + + webrtc::VideoFrame CreateFrame(JNIEnv* jni, + jobject j_video_frame, + uint32_t timestamp_rtp) const; + + rtc::scoped_refptr CreateBuffer( + int width, + int height, + const Matrix& matrix, + jobject j_video_frame_buffer) const; + + private: + ScopedGlobalRef j_video_frame_class_; + jmethodID j_get_buffer_id_; + jmethodID j_get_width_id_; + jmethodID j_get_height_id_; + jmethodID j_get_rotation_id_; + jmethodID j_get_transform_matrix_id_; + jmethodID j_get_timestamp_ns_id_; + + ScopedGlobalRef j_video_frame_buffer_class_; + jmethodID j_retain_id_; + jmethodID j_release_id_; +}; + } // namespace webrtc_jni #endif // WEBRTC_SDK_ANDROID_SRC_JNI_NATIVE_HANDLE_IMPL_H_ diff --git a/webrtc/sdk/android/src/jni/peerconnection_jni.cc b/webrtc/sdk/android/src/jni/peerconnection_jni.cc index 5f33ba35ca..0bf66f4856 100644 --- a/webrtc/sdk/android/src/jni/peerconnection_jni.cc +++ b/webrtc/sdk/android/src/jni/peerconnection_jni.cc @@ -871,10 +871,29 @@ class JavaVideoRendererWrapper void OnFrame(const webrtc::VideoFrame& video_frame) override { ScopedLocalRefFrame local_ref_frame(jni()); - jobject j_frame = (video_frame.video_frame_buffer()->type() == - webrtc::VideoFrameBuffer::Type::kNative) - ? ToJavaTextureFrame(&video_frame) - : ToJavaI420Frame(&video_frame); + + jobject j_frame; + if (video_frame.video_frame_buffer()->type() == + webrtc::VideoFrameBuffer::Type::kNative) { + AndroidVideoFrameBuffer* android_buffer = + static_cast( + video_frame.video_frame_buffer().get()); + switch (android_buffer->android_type()) { + case AndroidVideoFrameBuffer::AndroidType::kTextureBuffer: + j_frame = ToJavaTextureFrame(&video_frame); + break; + case AndroidVideoFrameBuffer::AndroidType::kJavaBuffer: + j_frame = static_cast(android_buffer) + ->ToJavaI420Frame(jni(), video_frame.width(), + video_frame.height(), + video_frame.rotation()); + break; + default: + RTC_NOTREACHED(); + } + } else { + j_frame = ToJavaI420Frame(&video_frame); + } // |j_callbacks_| is responsible for releasing |j_frame| with // VideoRenderer.renderFrameDone(). jni()->CallVoidMethod(*j_callbacks_, j_render_frame_id_, j_frame);