diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 02ef00d4ff..329427e3ca 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -30,7 +30,6 @@ rtc_static_library("video_coding") { "frame_buffer2.h", "frame_object.cc", "frame_object.h", - "generic_decoder.cc", "generic_decoder.h", "generic_encoder.cc", "generic_encoder.h", @@ -82,6 +81,15 @@ rtc_static_library("video_coding") { "video_sender.cc", ] + if (is_android) { + sources += [ + "generic_decoder_android.cc", + "generic_decoder_android.h", + ] + } else { + sources += [ "generic_decoder.cc" ] + } + # TODO(jschuh): Bug 1348: fix this warning. configs += [ "//build/config/compiler:no_size_t_to_int_warning" ] diff --git a/modules/video_coding/generic_decoder.cc b/modules/video_coding/generic_decoder.cc index a7d6a85b90..fe6899858d 100644 --- a/modules/video_coding/generic_decoder.cc +++ b/modules/video_coding/generic_decoder.cc @@ -28,11 +28,13 @@ VCMDecodedFrameCallback::VCMDecodedFrameCallback(VCMTiming* timing, _timing(timing), _timestampMap(kDecoderFrameMemoryLength), _lastReceivedPictureID(0) { + decoder_thread_.DetachFromThread(); ntp_offset_ = _clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds(); } VCMDecodedFrameCallback::~VCMDecodedFrameCallback() { + RTC_DCHECK(construction_thread_.CalledOnValidThread()); } void VCMDecodedFrameCallback::SetUserReceiveCallback( @@ -41,9 +43,15 @@ void VCMDecodedFrameCallback::SetUserReceiveCallback( RTC_DCHECK((!_receiveCallback && receiveCallback) || (_receiveCallback && !receiveCallback)); _receiveCallback = receiveCallback; + // When the callback is cleared, it signals to us that the decoder thread + // is no longer running. Another decoder thread might be started, so it's + // important to reset the thread checker first. + if (!receiveCallback) + decoder_thread_.DetachFromThread(); } VCMReceiveCallback* VCMDecodedFrameCallback::UserReceiveCallback() { + RTC_DCHECK_RUN_ON(&decoder_thread_); // Called on the decode thread via VCMCodecDataBase::GetDecoder. // The callback must always have been set before this happens. RTC_DCHECK(_receiveCallback); @@ -66,16 +74,14 @@ int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, rtc::Optional decode_time_ms, rtc::Optional qp) { + RTC_DCHECK_RUN_ON(&decoder_thread_); RTC_DCHECK(_receiveCallback) << "Callback must not be null at this point"; + TRACE_EVENT_INSTANT1("webrtc", "VCMDecodedFrameCallback::Decoded", "timestamp", decodedImage.timestamp()); // TODO(holmer): We should improve this so that we can handle multiple // callbacks from one call to Decode(). - VCMFrameInformation* frameInfo; - { - rtc::CritScope cs(&lock_); - frameInfo = _timestampMap.Pop(decodedImage.timestamp()); - } + VCMFrameInformation* frameInfo = _timestampMap.Pop(decodedImage.timestamp()); if (frameInfo == NULL) { RTC_LOG(LS_WARNING) << "Too many frames backed up in the decoder, dropping " @@ -149,36 +155,37 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, int32_t VCMDecodedFrameCallback::ReceivedDecodedReferenceFrame( const uint64_t pictureId) { + RTC_DCHECK_RUN_ON(&decoder_thread_); return _receiveCallback->ReceivedDecodedReferenceFrame(pictureId); } int32_t VCMDecodedFrameCallback::ReceivedDecodedFrame( const uint64_t pictureId) { + RTC_DCHECK_RUN_ON(&decoder_thread_); _lastReceivedPictureID = pictureId; return 0; } uint64_t VCMDecodedFrameCallback::LastReceivedPictureID() const { + RTC_DCHECK_RUN_ON(&decoder_thread_); return _lastReceivedPictureID; } void VCMDecodedFrameCallback::OnDecoderImplementationName( const char* implementation_name) { + RTC_DCHECK_RUN_ON(&decoder_thread_); _receiveCallback->OnDecoderImplementationName(implementation_name); } void VCMDecodedFrameCallback::Map(uint32_t timestamp, VCMFrameInformation* frameInfo) { - rtc::CritScope cs(&lock_); + RTC_DCHECK_RUN_ON(&decoder_thread_); _timestampMap.Add(timestamp, frameInfo); } int32_t VCMDecodedFrameCallback::Pop(uint32_t timestamp) { - rtc::CritScope cs(&lock_); - if (_timestampMap.Pop(timestamp) == NULL) { - return VCM_GENERAL_ERROR; - } - return VCM_OK; + RTC_DCHECK_RUN_ON(&decoder_thread_); + return _timestampMap.Pop(timestamp) == nullptr ? VCM_GENERAL_ERROR : VCM_OK; } VCMGenericDecoder::VCMGenericDecoder(std::unique_ptr decoder) @@ -204,6 +211,7 @@ VCMGenericDecoder::~VCMGenericDecoder() { int32_t VCMGenericDecoder::InitDecode(const VideoCodec* settings, int32_t numberOfCores) { + RTC_DCHECK_RUN_ON(&decoder_thread_); TRACE_EVENT0("webrtc", "VCMGenericDecoder::InitDecode"); _codecType = settings->codecType; @@ -211,50 +219,58 @@ int32_t VCMGenericDecoder::InitDecode(const VideoCodec* settings, } int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, int64_t nowMs) { - TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp", - frame.EncodedImage()._timeStamp); - _frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs; - _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs(); - _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation(); - _frameInfos[_nextFrameInfoIdx].timing = frame.video_timing(); - // Set correctly only for key frames. Thus, use latest key frame - // content type. If the corresponding key frame was lost, decode will fail - // and content type will be ignored. - if (frame.FrameType() == kVideoFrameKey) { - _frameInfos[_nextFrameInfoIdx].content_type = frame.contentType(); - _last_keyframe_content_type = frame.contentType(); - } else { - _frameInfos[_nextFrameInfoIdx].content_type = _last_keyframe_content_type; - } - _callback->Map(frame.TimeStamp(), &_frameInfos[_nextFrameInfoIdx]); + RTC_DCHECK_RUN_ON(&decoder_thread_); + TRACE_EVENT2("webrtc", "VCMGenericDecoder::Decode", "timestamp", + frame.EncodedImage()._timeStamp, "decoder", + decoder_->ImplementationName()); + _frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs; + _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs(); + _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation(); + _frameInfos[_nextFrameInfoIdx].timing = frame.video_timing(); + // Set correctly only for key frames. Thus, use latest key frame + // content type. If the corresponding key frame was lost, decode will fail + // and content type will be ignored. + if (frame.FrameType() == kVideoFrameKey) { + _frameInfos[_nextFrameInfoIdx].content_type = frame.contentType(); + _last_keyframe_content_type = frame.contentType(); + } else { + _frameInfos[_nextFrameInfoIdx].content_type = _last_keyframe_content_type; + } + _callback->Map(frame.TimeStamp(), &_frameInfos[_nextFrameInfoIdx]); - _nextFrameInfoIdx = (_nextFrameInfoIdx + 1) % kDecoderFrameMemoryLength; - const RTPFragmentationHeader dummy_header; - int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(), - &dummy_header, - frame.CodecSpecific(), frame.RenderTimeMs()); + _nextFrameInfoIdx = (_nextFrameInfoIdx + 1) % kDecoderFrameMemoryLength; + const RTPFragmentationHeader dummy_header; + int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(), + &dummy_header, frame.CodecSpecific(), + frame.RenderTimeMs()); - _callback->OnDecoderImplementationName(decoder_->ImplementationName()); + // TODO(tommi): Necessary every time? + // Maybe this should be the first thing the function does, and only the first + // time around? + _callback->OnDecoderImplementationName(decoder_->ImplementationName()); + + if (ret != WEBRTC_VIDEO_CODEC_OK) { if (ret < WEBRTC_VIDEO_CODEC_OK) { RTC_LOG(LS_WARNING) << "Failed to decode frame with timestamp " << frame.TimeStamp() << ", error code: " << ret; - _callback->Pop(frame.TimeStamp()); - return ret; - } else if (ret == WEBRTC_VIDEO_CODEC_NO_OUTPUT || - ret == WEBRTC_VIDEO_CODEC_REQUEST_SLI) { - // No output - _callback->Pop(frame.TimeStamp()); } - return ret; + // We pop the frame for all non-'OK', failure or success codes such as + // WEBRTC_VIDEO_CODEC_NO_OUTPUT and WEBRTC_VIDEO_CODEC_REQUEST_SLI. + _callback->Pop(frame.TimeStamp()); + } + + return ret; } int32_t VCMGenericDecoder::RegisterDecodeCompleteCallback( VCMDecodedFrameCallback* callback) { + RTC_DCHECK_RUN_ON(&decoder_thread_); _callback = callback; return decoder_->RegisterDecodeCompleteCallback(callback); } bool VCMGenericDecoder::PrefersLateDecoding() const { + RTC_DCHECK_RUN_ON(&decoder_thread_); return decoder_->PrefersLateDecoding(); } diff --git a/modules/video_coding/generic_decoder.h b/modules/video_coding/generic_decoder.h index 2d940d883a..210a63b50c 100644 --- a/modules/video_coding/generic_decoder.h +++ b/modules/video_coding/generic_decoder.h @@ -18,9 +18,11 @@ #include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/timestamp_map.h" #include "modules/video_coding/timing.h" -#include "rtc_base/criticalsection.h" #include "rtc_base/thread_checker.h" +#if defined(WEBRTC_ANDROID) +#include "modules/video_coding/generic_decoder_android.h" // NOLINT +#else namespace webrtc { class VCMReceiveCallback; @@ -40,6 +42,7 @@ class VCMDecodedFrameCallback : public DecodedImageCallback { public: VCMDecodedFrameCallback(VCMTiming* timing, Clock* clock); ~VCMDecodedFrameCallback() override; + void SetUserReceiveCallback(VCMReceiveCallback* receiveCallback); VCMReceiveCallback* UserReceiveCallback(); @@ -59,7 +62,7 @@ class VCMDecodedFrameCallback : public DecodedImageCallback { private: rtc::ThreadChecker construction_thread_; - // Protect |_timestampMap|. + rtc::ThreadChecker decoder_thread_; Clock* const _clock; // This callback must be set before the decoder thread starts running // and must only be unset when external threads (e.g decoder thread) @@ -67,10 +70,9 @@ class VCMDecodedFrameCallback : public DecodedImageCallback { // while there are more than one threads involved, it must be set // from the same thread, and therfore a lock is not required to access it. VCMReceiveCallback* _receiveCallback = nullptr; - VCMTiming* _timing; - rtc::CriticalSection lock_; - VCMTimestampMap _timestampMap RTC_GUARDED_BY(lock_); - uint64_t _lastReceivedPictureID; + VCMTiming* _timing RTC_GUARDED_BY(decoder_thread_); + VCMTimestampMap _timestampMap RTC_GUARDED_BY(decoder_thread_); + uint64_t _lastReceivedPictureID RTC_GUARDED_BY(decoder_thread_); int64_t ntp_offset_; }; @@ -97,22 +99,24 @@ class VCMGenericDecoder { */ int32_t RegisterDecodeCompleteCallback(VCMDecodedFrameCallback* callback); - bool External() const; bool PrefersLateDecoding() const; bool IsSameDecoder(VideoDecoder* decoder) const { return decoder_.get() == decoder; } private: - VCMDecodedFrameCallback* _callback; - VCMFrameInformation _frameInfos[kDecoderFrameMemoryLength]; - uint32_t _nextFrameInfoIdx; + rtc::ThreadChecker decoder_thread_; + VCMDecodedFrameCallback* _callback RTC_GUARDED_BY(decoder_thread_); + VCMFrameInformation _frameInfos[kDecoderFrameMemoryLength] RTC_GUARDED_BY( + decoder_thread_); + uint32_t _nextFrameInfoIdx RTC_GUARDED_BY(decoder_thread_); std::unique_ptr decoder_; - VideoCodecType _codecType; + VideoCodecType _codecType RTC_GUARDED_BY(decoder_thread_); const bool _isExternal; VideoContentType _last_keyframe_content_type; }; } // namespace webrtc +#endif // defined(WEBRTC_ANDROID) #endif // MODULES_VIDEO_CODING_GENERIC_DECODER_H_ diff --git a/modules/video_coding/generic_decoder_android.cc b/modules/video_coding/generic_decoder_android.cc new file mode 100644 index 0000000000..fb19d25a0d --- /dev/null +++ b/modules/video_coding/generic_decoder_android.cc @@ -0,0 +1,261 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/generic_decoder_android.h" + +#include + +#include "modules/video_coding/include/video_coding.h" +#include "modules/video_coding/internal_defines.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/timeutils.h" +#include "rtc_base/trace_event.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { + +VCMDecodedFrameCallback::VCMDecodedFrameCallback(VCMTiming* timing, + Clock* clock) + : _clock(clock), + _timing(timing), + _timestampMap(kDecoderFrameMemoryLength), + _lastReceivedPictureID(0) { + ntp_offset_ = + _clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds(); +} + +VCMDecodedFrameCallback::~VCMDecodedFrameCallback() { +} + +void VCMDecodedFrameCallback::SetUserReceiveCallback( + VCMReceiveCallback* receiveCallback) { + RTC_DCHECK(construction_thread_.CalledOnValidThread()); + RTC_DCHECK((!_receiveCallback && receiveCallback) || + (_receiveCallback && !receiveCallback)); + _receiveCallback = receiveCallback; +} + +VCMReceiveCallback* VCMDecodedFrameCallback::UserReceiveCallback() { + // Called on the decode thread via VCMCodecDataBase::GetDecoder. + // The callback must always have been set before this happens. + RTC_DCHECK(_receiveCallback); + return _receiveCallback; +} + +int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage) { + return Decoded(decodedImage, -1); +} + +int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, + int64_t decode_time_ms) { + Decoded(decodedImage, + decode_time_ms >= 0 ? rtc::Optional(decode_time_ms) + : rtc::nullopt, + rtc::nullopt); + return WEBRTC_VIDEO_CODEC_OK; +} + +void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, + rtc::Optional decode_time_ms, + rtc::Optional qp) { + RTC_DCHECK(_receiveCallback) << "Callback must not be null at this point"; + TRACE_EVENT_INSTANT1("webrtc", "VCMDecodedFrameCallback::Decoded", + "timestamp", decodedImage.timestamp()); + // TODO(holmer): We should improve this so that we can handle multiple + // callbacks from one call to Decode(). + VCMFrameInformation* frameInfo; + { + rtc::CritScope cs(&lock_); + frameInfo = _timestampMap.Pop(decodedImage.timestamp()); + } + + if (frameInfo == NULL) { + RTC_LOG(LS_WARNING) << "Too many frames backed up in the decoder, dropping " + "this one."; + return; + } + + const int64_t now_ms = _clock->TimeInMilliseconds(); + if (!decode_time_ms) { + decode_time_ms = now_ms - frameInfo->decodeStartTimeMs; + } + _timing->StopDecodeTimer(decodedImage.timestamp(), *decode_time_ms, now_ms, + frameInfo->renderTimeMs); + + // Report timing information. + if (frameInfo->timing.flags != TimingFrameFlags::kInvalid) { + int64_t capture_time_ms = decodedImage.ntp_time_ms() - ntp_offset_; + // Convert remote timestamps to local time from ntp timestamps. + frameInfo->timing.encode_start_ms -= ntp_offset_; + frameInfo->timing.encode_finish_ms -= ntp_offset_; + frameInfo->timing.packetization_finish_ms -= ntp_offset_; + frameInfo->timing.pacer_exit_ms -= ntp_offset_; + frameInfo->timing.network_timestamp_ms -= ntp_offset_; + frameInfo->timing.network2_timestamp_ms -= ntp_offset_; + + int64_t sender_delta_ms = 0; + if (decodedImage.ntp_time_ms() < 0) { + // Sender clock is not estimated yet. Make sure that sender times are all + // negative to indicate that. Yet they still should be relatively correct. + sender_delta_ms = + std::max({capture_time_ms, frameInfo->timing.encode_start_ms, + frameInfo->timing.encode_finish_ms, + frameInfo->timing.packetization_finish_ms, + frameInfo->timing.pacer_exit_ms, + frameInfo->timing.network_timestamp_ms, + frameInfo->timing.network2_timestamp_ms}) + + 1; + } + + TimingFrameInfo timing_frame_info; + + timing_frame_info.capture_time_ms = capture_time_ms - sender_delta_ms; + timing_frame_info.encode_start_ms = + frameInfo->timing.encode_start_ms - sender_delta_ms; + timing_frame_info.encode_finish_ms = + frameInfo->timing.encode_finish_ms - sender_delta_ms; + timing_frame_info.packetization_finish_ms = + frameInfo->timing.packetization_finish_ms - sender_delta_ms; + timing_frame_info.pacer_exit_ms = + frameInfo->timing.pacer_exit_ms - sender_delta_ms; + timing_frame_info.network_timestamp_ms = + frameInfo->timing.network_timestamp_ms - sender_delta_ms; + timing_frame_info.network2_timestamp_ms = + frameInfo->timing.network2_timestamp_ms - sender_delta_ms; + timing_frame_info.receive_start_ms = frameInfo->timing.receive_start_ms; + timing_frame_info.receive_finish_ms = frameInfo->timing.receive_finish_ms; + timing_frame_info.decode_start_ms = frameInfo->decodeStartTimeMs; + timing_frame_info.decode_finish_ms = now_ms; + timing_frame_info.render_time_ms = frameInfo->renderTimeMs; + timing_frame_info.rtp_timestamp = decodedImage.timestamp(); + timing_frame_info.flags = frameInfo->timing.flags; + + _timing->SetTimingFrameInfo(timing_frame_info); + } + + decodedImage.set_timestamp_us( + frameInfo->renderTimeMs * rtc::kNumMicrosecsPerMillisec); + decodedImage.set_rotation(frameInfo->rotation); + _receiveCallback->FrameToRender(decodedImage, qp, frameInfo->content_type); +} + +int32_t VCMDecodedFrameCallback::ReceivedDecodedReferenceFrame( + const uint64_t pictureId) { + return _receiveCallback->ReceivedDecodedReferenceFrame(pictureId); +} + +int32_t VCMDecodedFrameCallback::ReceivedDecodedFrame( + const uint64_t pictureId) { + _lastReceivedPictureID = pictureId; + return 0; +} + +uint64_t VCMDecodedFrameCallback::LastReceivedPictureID() const { + return _lastReceivedPictureID; +} + +void VCMDecodedFrameCallback::OnDecoderImplementationName( + const char* implementation_name) { + _receiveCallback->OnDecoderImplementationName(implementation_name); +} + +void VCMDecodedFrameCallback::Map(uint32_t timestamp, + VCMFrameInformation* frameInfo) { + rtc::CritScope cs(&lock_); + _timestampMap.Add(timestamp, frameInfo); +} + +int32_t VCMDecodedFrameCallback::Pop(uint32_t timestamp) { + rtc::CritScope cs(&lock_); + if (_timestampMap.Pop(timestamp) == NULL) { + return VCM_GENERAL_ERROR; + } + return VCM_OK; +} + +VCMGenericDecoder::VCMGenericDecoder(std::unique_ptr decoder) + : VCMGenericDecoder(decoder.release(), false /* isExternal */) {} + +VCMGenericDecoder::VCMGenericDecoder(VideoDecoder* decoder, bool isExternal) + : _callback(NULL), + _frameInfos(), + _nextFrameInfoIdx(0), + decoder_(decoder), + _codecType(kVideoCodecUnknown), + _isExternal(isExternal), + _last_keyframe_content_type(VideoContentType::UNSPECIFIED) { + RTC_DCHECK(decoder_); +} + +VCMGenericDecoder::~VCMGenericDecoder() { + decoder_->Release(); + if (_isExternal) + decoder_.release(); + RTC_DCHECK(_isExternal || decoder_); +} + +int32_t VCMGenericDecoder::InitDecode(const VideoCodec* settings, + int32_t numberOfCores) { + TRACE_EVENT0("webrtc", "VCMGenericDecoder::InitDecode"); + _codecType = settings->codecType; + + return decoder_->InitDecode(settings, numberOfCores); +} + +int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, int64_t nowMs) { + TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp", + frame.EncodedImage()._timeStamp); + _frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs; + _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs(); + _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation(); + _frameInfos[_nextFrameInfoIdx].timing = frame.video_timing(); + // Set correctly only for key frames. Thus, use latest key frame + // content type. If the corresponding key frame was lost, decode will fail + // and content type will be ignored. + if (frame.FrameType() == kVideoFrameKey) { + _frameInfos[_nextFrameInfoIdx].content_type = frame.contentType(); + _last_keyframe_content_type = frame.contentType(); + } else { + _frameInfos[_nextFrameInfoIdx].content_type = _last_keyframe_content_type; + } + _callback->Map(frame.TimeStamp(), &_frameInfos[_nextFrameInfoIdx]); + + _nextFrameInfoIdx = (_nextFrameInfoIdx + 1) % kDecoderFrameMemoryLength; + const RTPFragmentationHeader dummy_header; + int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(), + &dummy_header, + frame.CodecSpecific(), frame.RenderTimeMs()); + + _callback->OnDecoderImplementationName(decoder_->ImplementationName()); + if (ret < WEBRTC_VIDEO_CODEC_OK) { + RTC_LOG(LS_WARNING) << "Failed to decode frame with timestamp " + << frame.TimeStamp() << ", error code: " << ret; + _callback->Pop(frame.TimeStamp()); + return ret; + } else if (ret == WEBRTC_VIDEO_CODEC_NO_OUTPUT || + ret == WEBRTC_VIDEO_CODEC_REQUEST_SLI) { + // No output + _callback->Pop(frame.TimeStamp()); + } + return ret; +} + +int32_t VCMGenericDecoder::RegisterDecodeCompleteCallback( + VCMDecodedFrameCallback* callback) { + _callback = callback; + return decoder_->RegisterDecodeCompleteCallback(callback); +} + +bool VCMGenericDecoder::PrefersLateDecoding() const { + return decoder_->PrefersLateDecoding(); +} + +} // namespace webrtc diff --git a/modules/video_coding/generic_decoder_android.h b/modules/video_coding/generic_decoder_android.h new file mode 100644 index 0000000000..3ef0957967 --- /dev/null +++ b/modules/video_coding/generic_decoder_android.h @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_GENERIC_DECODER_ANDROID_H_ +#define MODULES_VIDEO_CODING_GENERIC_DECODER_ANDROID_H_ + +#include + +#include "modules/include/module_common_types.h" +#include "modules/video_coding/encoded_frame.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/timestamp_map.h" +#include "modules/video_coding/timing.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/thread_checker.h" + +namespace webrtc { + +class VCMReceiveCallback; + +enum { kDecoderFrameMemoryLength = 10 }; + +struct VCMFrameInformation { + int64_t renderTimeMs; + int64_t decodeStartTimeMs; + void* userData; + VideoRotation rotation; + VideoContentType content_type; + EncodedImage::Timing timing; +}; + +// TODO(bugs.webrtc.org/7361): Delete this class and file once we've updated +// the decoder on Android to deliver frames on the decoder thread/taskqueue. +// Likely FrameBuffer will need to be updated first. +// Use generic_decoder.* instead. +class VCMDecodedFrameCallback : public DecodedImageCallback { + public: + VCMDecodedFrameCallback(VCMTiming* timing, Clock* clock); + ~VCMDecodedFrameCallback() override; + void SetUserReceiveCallback(VCMReceiveCallback* receiveCallback); + VCMReceiveCallback* UserReceiveCallback(); + + int32_t Decoded(VideoFrame& decodedImage) override; + int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) override; + void Decoded(VideoFrame& decodedImage, + rtc::Optional decode_time_ms, + rtc::Optional qp) override; + int32_t ReceivedDecodedReferenceFrame(const uint64_t pictureId) override; + int32_t ReceivedDecodedFrame(const uint64_t pictureId) override; + + uint64_t LastReceivedPictureID() const; + void OnDecoderImplementationName(const char* implementation_name); + + void Map(uint32_t timestamp, VCMFrameInformation* frameInfo); + int32_t Pop(uint32_t timestamp); + + private: + rtc::ThreadChecker construction_thread_; + // Protect |_timestampMap|. + Clock* const _clock; + // This callback must be set before the decoder thread starts running + // and must only be unset when external threads (e.g decoder thread) + // have been stopped. Due to that, the variable should regarded as const + // while there are more than one threads involved, it must be set + // from the same thread, and therfore a lock is not required to access it. + VCMReceiveCallback* _receiveCallback = nullptr; + VCMTiming* _timing; + rtc::CriticalSection lock_; + VCMTimestampMap _timestampMap RTC_GUARDED_BY(lock_); + uint64_t _lastReceivedPictureID; + int64_t ntp_offset_; +}; + +class VCMGenericDecoder { + public: + explicit VCMGenericDecoder(std::unique_ptr decoder); + explicit VCMGenericDecoder(VideoDecoder* decoder, bool isExternal = false); + ~VCMGenericDecoder(); + + /** + * Initialize the decoder with the information from the VideoCodec + */ + int32_t InitDecode(const VideoCodec* settings, int32_t numberOfCores); + + /** + * Decode to a raw I420 frame, + * + * inputVideoBuffer reference to encoded video frame + */ + int32_t Decode(const VCMEncodedFrame& inputFrame, int64_t nowMs); + + /** + * Set decode callback. Deregistering while decoding is illegal. + */ + int32_t RegisterDecodeCompleteCallback(VCMDecodedFrameCallback* callback); + + bool External() const; + bool PrefersLateDecoding() const; + bool IsSameDecoder(VideoDecoder* decoder) const { + return decoder_.get() == decoder; + } + + private: + VCMDecodedFrameCallback* _callback; + VCMFrameInformation _frameInfos[kDecoderFrameMemoryLength]; + uint32_t _nextFrameInfoIdx; + std::unique_ptr decoder_; + VideoCodecType _codecType; + const bool _isExternal; + VideoContentType _last_keyframe_content_type; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_GENERIC_DECODER_ANDROID_H_