/* * Copyright 2017 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "sdk/android/src/jni/videodecoderwrapper.h" #include "api/video/video_frame.h" #include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/utility/vp8_header_parser.h" #include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" #include "rtc_base/logging.h" #include "rtc_base/timeutils.h" #include "sdk/android/src/jni/classreferenceholder.h" namespace webrtc { namespace jni { VideoDecoderWrapper::VideoDecoderWrapper(JNIEnv* jni, jobject decoder) : android_video_buffer_factory_(jni), decoder_(jni, decoder), encoded_image_class_(jni, FindClass(jni, "org/webrtc/EncodedImage")), frame_type_class_(jni, FindClass(jni, "org/webrtc/EncodedImage$FrameType")), settings_class_(jni, FindClass(jni, "org/webrtc/VideoDecoder$Settings")), video_frame_class_(jni, FindClass(jni, "org/webrtc/VideoFrame")), video_codec_status_class_(jni, FindClass(jni, "org/webrtc/VideoCodecStatus")), integer_class_(jni, jni->FindClass("java/lang/Integer")) { encoded_image_constructor_ = jni->GetMethodID(*encoded_image_class_, "", "(Ljava/nio/ByteBuffer;IIJLorg/webrtc/" "EncodedImage$FrameType;IZLjava/lang/Integer;)V"); settings_constructor_ = jni->GetMethodID(*settings_class_, "", "(III)V"); empty_frame_field_ = jni->GetStaticFieldID( *frame_type_class_, "EmptyFrame", "Lorg/webrtc/EncodedImage$FrameType;"); video_frame_key_field_ = jni->GetStaticFieldID(*frame_type_class_, "VideoFrameKey", "Lorg/webrtc/EncodedImage$FrameType;"); video_frame_delta_field_ = jni->GetStaticFieldID(*frame_type_class_, "VideoFrameDelta", "Lorg/webrtc/EncodedImage$FrameType;"); video_frame_get_timestamp_ns_method_ = jni->GetMethodID(*video_frame_class_, "getTimestampNs", "()J"); jclass decoder_class = jni->GetObjectClass(decoder); init_decode_method_ = jni->GetMethodID(decoder_class, "initDecode", "(Lorg/webrtc/VideoDecoder$Settings;Lorg/webrtc/" "VideoDecoder$Callback;)Lorg/webrtc/VideoCodecStatus;"); release_method_ = jni->GetMethodID(decoder_class, "release", "()Lorg/webrtc/VideoCodecStatus;"); decode_method_ = jni->GetMethodID(decoder_class, "decode", "(Lorg/webrtc/EncodedImage;Lorg/webrtc/" "VideoDecoder$DecodeInfo;)Lorg/webrtc/" "VideoCodecStatus;"); get_prefers_late_decoding_method_ = jni->GetMethodID(decoder_class, "getPrefersLateDecoding", "()Z"); get_implementation_name_method_ = jni->GetMethodID( decoder_class, "getImplementationName", "()Ljava/lang/String;"); get_number_method_ = jni->GetMethodID(*video_codec_status_class_, "getNumber", "()I"); integer_constructor_ = jni->GetMethodID(*integer_class_, "", "(I)V"); int_value_method_ = jni->GetMethodID(*integer_class_, "intValue", "()I"); initialized_ = false; // QP parsing starts enabled and we disable it if the decoder provides frames. qp_parsing_enabled_ = true; implementation_name_ = GetImplementationName(jni); } int32_t VideoDecoderWrapper::InitDecode(const VideoCodec* codec_settings, int32_t number_of_cores) { JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); codec_settings_ = *codec_settings; number_of_cores_ = number_of_cores; return InitDecodeInternal(jni); } int32_t VideoDecoderWrapper::InitDecodeInternal(JNIEnv* jni) { jobject settings = jni->NewObject(*settings_class_, settings_constructor_, number_of_cores_, codec_settings_.width, codec_settings_.height); jclass callback_class = FindClass(jni, "org/webrtc/VideoDecoderWrapperCallback"); jmethodID callback_constructor = jni->GetMethodID(callback_class, "", "(J)V"); jobject callback = jni->NewObject(callback_class, callback_constructor, jlongFromPointer(this)); jobject ret = jni->CallObjectMethod(*decoder_, init_decode_method_, settings, callback); if (jni->CallIntMethod(ret, get_number_method_) == WEBRTC_VIDEO_CODEC_OK) { initialized_ = true; } // The decoder was reinitialized so re-enable the QP parsing in case it stops // providing QP values. qp_parsing_enabled_ = true; return HandleReturnCode(jni, ret); } int32_t VideoDecoderWrapper::Decode( const EncodedImage& input_image, bool missing_frames, const RTPFragmentationHeader* fragmentation, const CodecSpecificInfo* codec_specific_info, int64_t render_time_ms) { if (!initialized_) { // Most likely initializing the codec failed. return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; } JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); FrameExtraInfo frame_extra_info; frame_extra_info.capture_time_ns = input_image.capture_time_ms_ * rtc::kNumNanosecsPerMillisec; frame_extra_info.timestamp_rtp = input_image._timeStamp; frame_extra_info.qp = qp_parsing_enabled_ ? ParseQP(input_image) : rtc::Optional(); frame_extra_infos_.push_back(frame_extra_info); jobject jinput_image = ConvertEncodedImageToJavaEncodedImage(jni, input_image); jobject ret = jni->CallObjectMethod(*decoder_, decode_method_, jinput_image, nullptr); return HandleReturnCode(jni, ret); } int32_t VideoDecoderWrapper::RegisterDecodeCompleteCallback( DecodedImageCallback* callback) { callback_ = callback; return WEBRTC_VIDEO_CODEC_OK; } int32_t VideoDecoderWrapper::Release() { JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); jobject ret = jni->CallObjectMethod(*decoder_, release_method_); frame_extra_infos_.clear(); initialized_ = false; return HandleReturnCode(jni, ret); } bool VideoDecoderWrapper::PrefersLateDecoding() const { JNIEnv* jni = AttachCurrentThreadIfNeeded(); return jni->CallBooleanMethod(*decoder_, get_prefers_late_decoding_method_); } const char* VideoDecoderWrapper::ImplementationName() const { return implementation_name_.c_str(); } void VideoDecoderWrapper::OnDecodedFrame(JNIEnv* jni, jobject jframe, jobject jdecode_time_ms, jobject jqp) { const jlong capture_time_ns = jni->CallLongMethod(jframe, video_frame_get_timestamp_ns_method_); FrameExtraInfo frame_extra_info; do { if (frame_extra_infos_.empty()) { RTC_LOG(LS_WARNING) << "Java decoder produced an unexpected frame."; return; } frame_extra_info = frame_extra_infos_.front(); frame_extra_infos_.pop_front(); // If the decoder might drop frames so iterate through the queue until we // find a matching timestamp. } while (frame_extra_info.capture_time_ns != capture_time_ns); VideoFrame frame = android_video_buffer_factory_.CreateFrame( jni, jframe, frame_extra_info.timestamp_rtp); rtc::Optional decoding_time_ms; if (jdecode_time_ms != nullptr) { decoding_time_ms = rtc::Optional( jni->CallIntMethod(jdecode_time_ms, int_value_method_)); } rtc::Optional qp; if (jqp != nullptr) { qp = rtc::Optional(jni->CallIntMethod(jqp, int_value_method_)); // The decoder provides QP values itself, no need to parse the bitstream. qp_parsing_enabled_ = false; } else { qp = frame_extra_info.qp; // The decoder doesn't provide QP values, ensure bitstream parsing is // enabled. qp_parsing_enabled_ = true; } callback_->Decoded(frame, decoding_time_ms, qp); } jobject VideoDecoderWrapper::ConvertEncodedImageToJavaEncodedImage( JNIEnv* jni, const EncodedImage& image) { jobject buffer = jni->NewDirectByteBuffer(image._buffer, image._length); jfieldID frame_type_field; switch (image._frameType) { case kEmptyFrame: frame_type_field = empty_frame_field_; break; case kVideoFrameKey: frame_type_field = video_frame_key_field_; break; case kVideoFrameDelta: frame_type_field = video_frame_delta_field_; break; default: RTC_NOTREACHED(); return nullptr; } jobject frame_type = jni->GetStaticObjectField(*frame_type_class_, frame_type_field); jobject qp = nullptr; if (image.qp_ != -1) { qp = jni->NewObject(*integer_class_, integer_constructor_, image.qp_); } return jni->NewObject( *encoded_image_class_, encoded_image_constructor_, buffer, static_cast(image._encodedWidth), static_cast(image._encodedHeight), static_cast(image.capture_time_ms_ * rtc::kNumNanosecsPerMillisec), frame_type, static_cast(image.rotation_), image._completeFrame, qp); } int32_t VideoDecoderWrapper::HandleReturnCode(JNIEnv* jni, jobject code) { int32_t value = jni->CallIntMethod(code, get_number_method_); if (value < 0) { // Any errors are represented by negative values. // Reset the codec. if (Release() == WEBRTC_VIDEO_CODEC_OK) { InitDecodeInternal(jni); } RTC_LOG(LS_WARNING) << "Falling back to software decoder."; return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; } else { return value; } } rtc::Optional VideoDecoderWrapper::ParseQP( const EncodedImage& input_image) { if (input_image.qp_ != -1) { return rtc::Optional(input_image.qp_); } rtc::Optional qp; switch (codec_settings_.codecType) { case kVideoCodecVP8: { int qp_int; if (vp8::GetQp(input_image._buffer, input_image._length, &qp_int)) { qp = rtc::Optional(qp_int); } break; } case kVideoCodecVP9: { int qp_int; if (vp9::GetQp(input_image._buffer, input_image._length, &qp_int)) { qp = rtc::Optional(qp_int); } break; } case kVideoCodecH264: { h264_bitstream_parser_.ParseBitstream(input_image._buffer, input_image._length); int qp_int; if (h264_bitstream_parser_.GetLastSliceQp(&qp_int)) { qp = rtc::Optional(qp_int); } break; } default: break; // Default is to not provide QP. } return qp; } std::string VideoDecoderWrapper::GetImplementationName(JNIEnv* jni) const { jstring jname = reinterpret_cast( jni->CallObjectMethod(*decoder_, get_implementation_name_method_)); return JavaToStdString(jni, jname); } JNI_FUNCTION_DECLARATION(void, VideoDecoderWrapperCallback_nativeOnDecodedFrame, JNIEnv* jni, jclass, jlong jnative_decoder, jobject jframe, jobject jdecode_time_ms, jobject jqp) { VideoDecoderWrapper* native_decoder = reinterpret_cast(jnative_decoder); native_decoder->OnDecodedFrame(jni, jframe, jdecode_time_ms, jqp); } } // namespace jni } // namespace webrtc