From 0ba43b5a2045b0b6de4c8e374895462961330b85 Mon Sep 17 00:00:00 2001 From: sakal Date: Mon, 14 Aug 2017 05:17:49 -0700 Subject: [PATCH] Add support for adding VideoSinks to VideoTracks. VideoSinks receive the new kind of VideoFrames and will replace VideoRenderers. Converting from old texture frames to VideoFrames will involve conversion to I420 so it is not recommended to use VideoSinks before all sources produce VideoFrames. BUG=webrtc:7749, webrtc:7760 Review-Url: https://codereview.webrtc.org/3002553002 Cr-Commit-Position: refs/heads/master@{#19335} --- webrtc/sdk/android/BUILD.gn | 1 + .../sdk/android/api/org/webrtc/VideoSink.java | 23 ++++++ .../android/api/org/webrtc/VideoTrack.java | 44 ++++++++++- .../org/webrtc/WrappedNativeI420Buffer.java | 2 + .../android/src/jni/classreferenceholder.cc | 1 + .../sdk/android/src/jni/native_handle_impl.cc | 38 +++++++--- webrtc/sdk/android/src/jni/videotrack_jni.cc | 73 +++++++++++++++---- .../src/jni/wrapped_native_i420_buffer.cc | 4 +- .../src/jni/wrapped_native_i420_buffer.h | 1 + 9 files changed, 157 insertions(+), 30 deletions(-) create mode 100644 webrtc/sdk/android/api/org/webrtc/VideoSink.java diff --git a/webrtc/sdk/android/BUILD.gn b/webrtc/sdk/android/BUILD.gn index c8523fb96d..c13f426dbd 100644 --- a/webrtc/sdk/android/BUILD.gn +++ b/webrtc/sdk/android/BUILD.gn @@ -417,6 +417,7 @@ android_library("libjingle_peerconnection_java") { "api/org/webrtc/VideoFileRenderer.java", "api/org/webrtc/VideoFrame.java", "api/org/webrtc/VideoRenderer.java", + "api/org/webrtc/VideoSink.java", "api/org/webrtc/VideoSource.java", "api/org/webrtc/VideoTrack.java", "src/java/org/webrtc/AndroidVideoTrackSourceObserver.java", diff --git a/webrtc/sdk/android/api/org/webrtc/VideoSink.java b/webrtc/sdk/android/api/org/webrtc/VideoSink.java new file mode 100644 index 0000000000..e4162feb8b --- /dev/null +++ b/webrtc/sdk/android/api/org/webrtc/VideoSink.java @@ -0,0 +1,23 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +package org.webrtc; + +/** + * Java version of rtc::VideoSinkInterface. + */ +public interface VideoSink { + /** + * Implementations should call frame.retain() if they need to hold a reference to the frame after + * this function returns. Each call to retain() should be followed by a call to frame.release() + * when the reference is no longer needed. + */ + void onFrame(VideoFrame frame); +} diff --git a/webrtc/sdk/android/api/org/webrtc/VideoTrack.java b/webrtc/sdk/android/api/org/webrtc/VideoTrack.java index 8118940a03..18a826cb73 100644 --- a/webrtc/sdk/android/api/org/webrtc/VideoTrack.java +++ b/webrtc/sdk/android/api/org/webrtc/VideoTrack.java @@ -10,26 +10,55 @@ package org.webrtc; +import java.util.IdentityHashMap; import java.util.LinkedList; /** Java version of VideoTrackInterface. */ public class VideoTrack extends MediaStreamTrack { private final LinkedList renderers = new LinkedList(); + private final IdentityHashMap sinks = new IdentityHashMap(); public VideoTrack(long nativeTrack) { super(nativeTrack); } + /** + * Adds a VideoSink to the track. + * + * A track can have any number of VideoSinks. VideoSinks will replace + * renderers. However, converting old style texture frames will involve costly + * conversion to I420 so it is not recommended to upgrade before all your + * sources produce VideoFrames. + */ + public void addSink(VideoSink sink) { + final long nativeSink = nativeWrapSink(sink); + sinks.put(sink, nativeSink); + nativeAddSink(nativeTrack, nativeSink); + } + + /** + * Removes a VideoSink from the track. + * + * If the VideoSink was not attached to the track, this is a no-op. + */ + public void removeSink(VideoSink sink) { + final long nativeSink = sinks.remove(sink); + if (nativeSink != 0) { + nativeRemoveSink(nativeTrack, nativeSink); + nativeFreeSink(nativeSink); + } + } + public void addRenderer(VideoRenderer renderer) { renderers.add(renderer); - nativeAddRenderer(nativeTrack, renderer.nativeVideoRenderer); + nativeAddSink(nativeTrack, renderer.nativeVideoRenderer); } public void removeRenderer(VideoRenderer renderer) { if (!renderers.remove(renderer)) { return; } - nativeRemoveRenderer(nativeTrack, renderer.nativeVideoRenderer); + nativeRemoveSink(nativeTrack, renderer.nativeVideoRenderer); renderer.dispose(); } @@ -37,10 +66,17 @@ public class VideoTrack extends MediaStreamTrack { while (!renderers.isEmpty()) { removeRenderer(renderers.getFirst()); } + for (long nativeSink : sinks.values()) { + nativeRemoveSink(nativeTrack, nativeSink); + nativeFreeSink(nativeSink); + } + sinks.clear(); super.dispose(); } - private static native void nativeAddRenderer(long nativeTrack, long nativeRenderer); + private static native void nativeAddSink(long nativeTrack, long nativeSink); + private static native void nativeRemoveSink(long nativeTrack, long nativeSink); - private static native void nativeRemoveRenderer(long nativeTrack, long nativeRenderer); + private static native long nativeWrapSink(VideoSink sink); + private static native void nativeFreeSink(long nativeSink); } diff --git a/webrtc/sdk/android/src/java/org/webrtc/WrappedNativeI420Buffer.java b/webrtc/sdk/android/src/java/org/webrtc/WrappedNativeI420Buffer.java index 96cc9f1a02..8a3186934d 100644 --- a/webrtc/sdk/android/src/java/org/webrtc/WrappedNativeI420Buffer.java +++ b/webrtc/sdk/android/src/java/org/webrtc/WrappedNativeI420Buffer.java @@ -37,6 +37,8 @@ class WrappedNativeI420Buffer implements VideoFrame.I420Buffer { this.dataV = dataV; this.strideV = strideV; this.nativeBuffer = nativeBuffer; + + retain(); } @Override diff --git a/webrtc/sdk/android/src/jni/classreferenceholder.cc b/webrtc/sdk/android/src/jni/classreferenceholder.cc index dd5cfe74cf..5b309d08d3 100644 --- a/webrtc/sdk/android/src/jni/classreferenceholder.cc +++ b/webrtc/sdk/android/src/jni/classreferenceholder.cc @@ -113,6 +113,7 @@ ClassReferenceHolder::ClassReferenceHolder(JNIEnv* jni) { LoadClass(jni, "org/webrtc/VideoFrame$I420Buffer"); LoadClass(jni, "org/webrtc/VideoFrame$TextureBuffer"); LoadClass(jni, "org/webrtc/VideoRenderer$I420Frame"); + LoadClass(jni, "org/webrtc/VideoSink"); LoadClass(jni, "org/webrtc/VideoTrack"); LoadClass(jni, "org/webrtc/WrappedNativeI420Buffer"); } diff --git a/webrtc/sdk/android/src/jni/native_handle_impl.cc b/webrtc/sdk/android/src/jni/native_handle_impl.cc index cd78a27584..3be618570d 100644 --- a/webrtc/sdk/android/src/jni/native_handle_impl.cc +++ b/webrtc/sdk/android/src/jni/native_handle_impl.cc @@ -21,6 +21,7 @@ #include "webrtc/rtc_base/timeutils.h" #include "webrtc/sdk/android/src/jni/classreferenceholder.h" #include "webrtc/sdk/android/src/jni/jni_helpers.h" +#include "webrtc/sdk/android/src/jni/wrapped_native_i420_buffer.h" #include "webrtc/system_wrappers/include/aligned_malloc.h" namespace webrtc_jni { @@ -474,20 +475,37 @@ JavaVideoFrameFactory::JavaVideoFrameFactory(JNIEnv* jni) "(Lorg/webrtc/VideoFrame$Buffer;IJ)V"); } +static bool IsJavaVideoBuffer( + rtc::scoped_refptr buffer) { + if (buffer->type() != webrtc::VideoFrameBuffer::Type::kNative) { + return false; + } + AndroidVideoFrameBuffer* android_buffer = + static_cast(buffer.get()); + return android_buffer->android_type() == + AndroidVideoFrameBuffer::AndroidType::kJavaBuffer; +} + jobject JavaVideoFrameFactory::ToJavaFrame( JNIEnv* jni, const webrtc::VideoFrame& frame) const { - RTC_DCHECK(frame.video_frame_buffer()->type() == - webrtc::VideoFrameBuffer::Type::kNative); - AndroidVideoFrameBuffer* android_buffer = - static_cast(frame.video_frame_buffer().get()); - RTC_DCHECK(android_buffer->android_type() == - AndroidVideoFrameBuffer::AndroidType::kJavaBuffer); - AndroidVideoBuffer* android_video_buffer = - static_cast(android_buffer); - jobject buffer = android_video_buffer->video_frame_buffer(); + rtc::scoped_refptr buffer = + frame.video_frame_buffer(); + jobject j_buffer; + if (IsJavaVideoBuffer(buffer)) { + RTC_DCHECK(buffer->type() == webrtc::VideoFrameBuffer::Type::kNative); + AndroidVideoFrameBuffer* android_buffer = + static_cast(buffer.get()); + RTC_DCHECK(android_buffer->android_type() == + AndroidVideoFrameBuffer::AndroidType::kJavaBuffer); + AndroidVideoBuffer* android_video_buffer = + static_cast(android_buffer); + j_buffer = android_video_buffer->video_frame_buffer(); + } else { + j_buffer = WrapI420Buffer(jni, buffer->ToI420()); + } return jni->NewObject( - *j_video_frame_class_, j_video_frame_constructor_id_, buffer, + *j_video_frame_class_, j_video_frame_constructor_id_, j_buffer, static_cast(frame.rotation()), static_cast(frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec)); } diff --git a/webrtc/sdk/android/src/jni/videotrack_jni.cc b/webrtc/sdk/android/src/jni/videotrack_jni.cc index 96ab928500..9368acffbd 100644 --- a/webrtc/sdk/android/src/jni/videotrack_jni.cc +++ b/webrtc/sdk/android/src/jni/videotrack_jni.cc @@ -12,31 +12,78 @@ #include "webrtc/api/mediastreaminterface.h" #include "webrtc/rtc_base/logging.h" +#include "webrtc/sdk/android/src/jni/classreferenceholder.h" +#include "webrtc/sdk/android/src/jni/jni_helpers.h" +#include "webrtc/sdk/android/src/jni/native_handle_impl.h" namespace webrtc_jni { +namespace { + +class VideoSinkWrapper : public rtc::VideoSinkInterface { + public: + VideoSinkWrapper(JNIEnv* jni, jobject j_sink); + ~VideoSinkWrapper() override {} + + private: + void OnFrame(const webrtc::VideoFrame& frame) override; + + jmethodID j_on_frame_method_; + + const JavaVideoFrameFactory java_video_frame_factory_; + const ScopedGlobalRef j_sink_; +}; + +VideoSinkWrapper::VideoSinkWrapper(JNIEnv* jni, jobject j_sink) + : java_video_frame_factory_(jni), j_sink_(jni, j_sink) { + jclass j_video_sink_class = FindClass(jni, "org/webrtc/VideoSink"); + j_on_frame_method_ = jni->GetMethodID(j_video_sink_class, "onFrame", + "(Lorg/webrtc/VideoFrame;)V"); +} + +void VideoSinkWrapper::OnFrame(const webrtc::VideoFrame& frame) { + JNIEnv* jni = AttachCurrentThreadIfNeeded(); + ScopedLocalRefFrame local_ref_frame(jni); + jni->CallVoidMethod(*j_sink_, j_on_frame_method_, + java_video_frame_factory_.ToJavaFrame(jni, frame)); +} + +} // namespace + extern "C" JNIEXPORT void JNICALL -Java_org_webrtc_VideoTrack_nativeAddRenderer(JNIEnv* jni, - jclass, - jlong j_video_track_pointer, - jlong j_renderer_pointer) { - LOG(LS_INFO) << "VideoTrack::nativeAddRenderer"; - reinterpret_cast(j_video_track_pointer) +Java_org_webrtc_VideoTrack_nativeAddSink(JNIEnv* jni, + jclass, + jlong j_native_track, + jlong j_native_sink) { + reinterpret_cast(j_native_track) ->AddOrUpdateSink( reinterpret_cast*>( - j_renderer_pointer), + j_native_sink), rtc::VideoSinkWants()); } extern "C" JNIEXPORT void JNICALL -Java_org_webrtc_VideoTrack_nativeRemoveRenderer(JNIEnv* jni, - jclass, - jlong j_video_track_pointer, - jlong j_renderer_pointer) { - reinterpret_cast(j_video_track_pointer) +Java_org_webrtc_VideoTrack_nativeRemoveSink(JNIEnv* jni, + jclass, + jlong j_native_track, + jlong j_native_sink) { + reinterpret_cast(j_native_track) ->RemoveSink( reinterpret_cast*>( - j_renderer_pointer)); + j_native_sink)); +} + +extern "C" JNIEXPORT jlong JNICALL +Java_org_webrtc_VideoTrack_nativeWrapSink(JNIEnv* jni, jclass, jobject sink) { + return jlongFromPointer(new VideoSinkWrapper(jni, sink)); +} + +extern "C" JNIEXPORT void JNICALL +Java_org_webrtc_VideoTrack_nativeFreeSink(JNIEnv* jni, + jclass, + jlong j_native_sink) { + delete reinterpret_cast*>( + j_native_sink); } } // namespace webrtc_jni diff --git a/webrtc/sdk/android/src/jni/wrapped_native_i420_buffer.cc b/webrtc/sdk/android/src/jni/wrapped_native_i420_buffer.cc index c9b691e114..c5bbe5a34f 100644 --- a/webrtc/sdk/android/src/jni/wrapped_native_i420_buffer.cc +++ b/webrtc/sdk/android/src/jni/wrapped_native_i420_buffer.cc @@ -17,10 +17,8 @@ namespace webrtc_jni { // TODO(magjed): Write a test for this function. jobject WrapI420Buffer( + JNIEnv* jni, const rtc::scoped_refptr& i420_buffer) { - JNIEnv* jni = AttachCurrentThreadIfNeeded(); - ScopedLocalRefFrame local_ref_frame(jni); - jclass j_wrapped_native_i420_buffer_class = FindClass(jni, "org/webrtc/WrappedNativeI420Buffer"); jmethodID j_wrapped_native_i420_buffer_ctor_id = diff --git a/webrtc/sdk/android/src/jni/wrapped_native_i420_buffer.h b/webrtc/sdk/android/src/jni/wrapped_native_i420_buffer.h index e5ae137cd9..3eaedd6b05 100644 --- a/webrtc/sdk/android/src/jni/wrapped_native_i420_buffer.h +++ b/webrtc/sdk/android/src/jni/wrapped_native_i420_buffer.h @@ -20,6 +20,7 @@ namespace webrtc_jni { // This function wraps the C++ I420 buffer and returns a Java // VideoFrame.I420Buffer as a jobject. jobject WrapI420Buffer( + JNIEnv* jni, const rtc::scoped_refptr& i420_buffer); } // namespace webrtc_jni