/* * Copyright 2017 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "sdk/android/src/jni/videodecoderwrapper.h" #include "api/video/video_frame.h" #include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/utility/vp8_header_parser.h" #include "modules/video_coding/utility/vp9_uncompressed_header_parser.h" #include "rtc_base/logging.h" #include "sdk/android/generated_video_jni/jni/VideoDecoderWrapper_jni.h" #include "sdk/android/generated_video_jni/jni/VideoDecoder_jni.h" #include "sdk/android/src/jni/classreferenceholder.h" #include "sdk/android/src/jni/encodedimage.h" #include "sdk/android/src/jni/videocodecstatus.h" #include "sdk/android/src/jni/videoframe.h" namespace webrtc { namespace jni { namespace { // RTP timestamps are 90 kHz. const int64_t kNumRtpTicksPerMillisec = 90000 / rtc::kNumMillisecsPerSec; template inline rtc::Optional cast_optional(const rtc::Optional& value) { return value ? rtc::Optional(rtc::dchecked_cast(*value)) : rtc::nullopt; } } // namespace VideoDecoderWrapper::VideoDecoderWrapper(JNIEnv* jni, jobject decoder) : decoder_(jni, decoder) { initialized_ = false; // QP parsing starts enabled and we disable it if the decoder provides frames. qp_parsing_enabled_ = true; implementation_name_ = JavaToStdString( jni, Java_VideoDecoder_getImplementationName(jni, decoder)); } int32_t VideoDecoderWrapper::InitDecode(const VideoCodec* codec_settings, int32_t number_of_cores) { JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); codec_settings_ = *codec_settings; number_of_cores_ = number_of_cores; return InitDecodeInternal(jni); } int32_t VideoDecoderWrapper::InitDecodeInternal(JNIEnv* jni) { jobject settings = Java_VideoDecoderWrapper_createSettings( jni, number_of_cores_, codec_settings_.width, codec_settings_.height); jobject callback = Java_VideoDecoderWrapper_createDecoderCallback( jni, jlongFromPointer(this)); jobject ret = Java_VideoDecoder_initDecode(jni, *decoder_, settings, callback); if (JavaToNativeVideoCodecStatus(jni, ret) == WEBRTC_VIDEO_CODEC_OK) { initialized_ = true; } // The decoder was reinitialized so re-enable the QP parsing in case it stops // providing QP values. qp_parsing_enabled_ = true; return HandleReturnCode(jni, ret); } int32_t VideoDecoderWrapper::Decode( const EncodedImage& image_param, bool missing_frames, const RTPFragmentationHeader* fragmentation, const CodecSpecificInfo* codec_specific_info, int64_t render_time_ms) { if (!initialized_) { // Most likely initializing the codec failed. return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; } JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); // Make a mutable copy so we can modify the timestamp. EncodedImage input_image(image_param); // We use RTP timestamp for capture time because capture_time_ms_ is always 0. input_image.capture_time_ms_ = input_image._timeStamp / kNumRtpTicksPerMillisec; FrameExtraInfo frame_extra_info; frame_extra_info.timestamp_ns = input_image.capture_time_ms_ * rtc::kNumNanosecsPerMillisec; frame_extra_info.timestamp_rtp = input_image._timeStamp; frame_extra_info.timestamp_ntp = input_image.ntp_time_ms_; frame_extra_info.qp = qp_parsing_enabled_ ? ParseQP(input_image) : rtc::nullopt; frame_extra_infos_.push_back(frame_extra_info); jobject jinput_image = NativeToJavaEncodedImage(jni, input_image); jobject ret = Java_VideoDecoder_decode(jni, *decoder_, jinput_image, nullptr); return HandleReturnCode(jni, ret); } int32_t VideoDecoderWrapper::RegisterDecodeCompleteCallback( DecodedImageCallback* callback) { callback_ = callback; return WEBRTC_VIDEO_CODEC_OK; } int32_t VideoDecoderWrapper::Release() { JNIEnv* jni = AttachCurrentThreadIfNeeded(); ScopedLocalRefFrame local_ref_frame(jni); jobject ret = Java_VideoDecoder_release(jni, *decoder_); frame_extra_infos_.clear(); initialized_ = false; return HandleReturnCode(jni, ret); } bool VideoDecoderWrapper::PrefersLateDecoding() const { JNIEnv* jni = AttachCurrentThreadIfNeeded(); return Java_VideoDecoder_getPrefersLateDecoding(jni, *decoder_); } const char* VideoDecoderWrapper::ImplementationName() const { return implementation_name_.c_str(); } void VideoDecoderWrapper::OnDecodedFrame(JNIEnv* env, jobject j_caller, jobject j_frame, jobject j_decode_time_ms, jobject j_qp) { const uint64_t timestamp_ns = GetJavaVideoFrameTimestampNs(env, j_frame); FrameExtraInfo frame_extra_info; do { if (frame_extra_infos_.empty()) { RTC_LOG(LS_WARNING) << "Java decoder produced an unexpected frame: " << timestamp_ns; return; } frame_extra_info = frame_extra_infos_.front(); frame_extra_infos_.pop_front(); // If the decoder might drop frames so iterate through the queue until we // find a matching timestamp. } while (frame_extra_info.timestamp_ns != timestamp_ns); VideoFrame frame = JavaToNativeFrame(env, j_frame, frame_extra_info.timestamp_rtp); frame.set_ntp_time_ms(frame_extra_info.timestamp_ntp); rtc::Optional decoding_time_ms = JavaIntegerToOptionalInt(env, j_decode_time_ms); rtc::Optional decoder_qp = cast_optional(JavaIntegerToOptionalInt(env, j_qp)); // If the decoder provides QP values itself, no need to parse the bitstream. // Enable QP parsing if decoder does not provide QP values itself. qp_parsing_enabled_ = !decoder_qp.has_value(); callback_->Decoded(frame, decoding_time_ms, decoder_qp ? decoder_qp : frame_extra_info.qp); } int32_t VideoDecoderWrapper::HandleReturnCode(JNIEnv* jni, jobject code) { int32_t value = JavaToNativeVideoCodecStatus(jni, code); if (value < 0) { // Any errors are represented by negative values. // Reset the codec. if (Release() == WEBRTC_VIDEO_CODEC_OK) { InitDecodeInternal(jni); } RTC_LOG(LS_WARNING) << "Falling back to software decoder."; return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE; } else { return value; } } rtc::Optional VideoDecoderWrapper::ParseQP( const EncodedImage& input_image) { if (input_image.qp_ != -1) { return input_image.qp_; } rtc::Optional qp; switch (codec_settings_.codecType) { case kVideoCodecVP8: { int qp_int; if (vp8::GetQp(input_image._buffer, input_image._length, &qp_int)) { qp = qp_int; } break; } case kVideoCodecVP9: { int qp_int; if (vp9::GetQp(input_image._buffer, input_image._length, &qp_int)) { qp = qp_int; } break; } case kVideoCodecH264: { h264_bitstream_parser_.ParseBitstream(input_image._buffer, input_image._length); int qp_int; if (h264_bitstream_parser_.GetLastSliceQp(&qp_int)) { qp = qp_int; } break; } default: break; // Default is to not provide QP. } return qp; } } // namespace jni } // namespace webrtc