zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Deliver video frames on Android, on the decode thread.

Browse Source 
VideoCoding
* Adding a method for polling for frames on Android only until the capture implementation takes care of this (longer term plan).

CodecDatabase
* Add an accessor for the current decoder
* Use std::unique_ptr<> for ownership.
* Remove "Release()" and "ReleaseDecoder()". Instead just delete.
* Remove |friend| relationship between CodecDatabase and VCMGenericDecoder.

VCMDecodedFrameCallback
* DCHECKs for thread correctness.
* Remove |lock_| now that a threading model has been established and verified.

VCMGenericDecoder
* All methods now have thread checks.
* Variable access associated with thread checkers.

VideoReceiver
* Added two notification methods, DecoderThreadStarting() and DecoderThreadStopped()
  * Allows us to establish a period when the decoder thread is not running and it is safe to modify variables such as callbacks, that are only read when the decoder thread is running.
  * Allows us to DCHECK thread guarantees.
  * Allows synchronizing callbacks from the module process thread and have them only active while the decoder thread is running.
  * The above, allows us to establish two modes for the thread, single-threaded-mutable and multi-threaded-const.
  * Using that knowledge, we can remove |receive_crit_| as well as locking for a number of member variables.

MediaCodecVideoDecoder
* Removed frame polling code from this class, since this is now done from the root thread function in VideoReceiveStream.

VideoReceiveStream
* On Android: Polls for decoded frames every 10ms (same interval as previously in MediaCodecVideoDecoder)
* [Un]Registers the |video_receiver_| with the module thread only around the time the decoder thread is started/stopped.
* Notifies the receiver of start/stop events of the decoder thread.
* Changed the decoder thread to use the new PlatformThread callback type.

BUG=webrtc:7361, 695438

Review-Url: https://codereview.webrtc.org/2764573002
Cr-Commit-Position: refs/heads/master@{#17527}
This commit is contained in:
tommi
2017-04-04 03:53:02 -07:00
committed by Commit bot
parent aa7d935cd5
commit e3aa88bbd5
12 changed files with 511 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files

194
webrtc/sdk/android/src/jni/androidmediadecoder_jni.cc
View File

@ -36,6 +36,18 @@
#include "webrtc/sdk/android/src/jni/surfacetexturehelper_jni.h"
#include "webrtc/system_wrappers/include/logcat_trace_context.h"
// Logging macros.
#define TAG_DECODER "MediaCodecVideoDecoder"
#ifdef TRACK_BUFFER_TIMING
#define ALOGV(...)
__android_log_print(ANDROID_LOG_VERBOSE, TAG_DECODER, __VA_ARGS__)
#else
#define ALOGV(...)
#endif
#define ALOGD LOG_TAG(rtc::LS_INFO, TAG_DECODER)
#define ALOGW LOG_TAG(rtc::LS_WARNING, TAG_DECODER)
#define ALOGE LOG_TAG(rtc::LS_ERROR, TAG_DECODER)
using rtc::Bind;
using rtc::Thread;
using rtc::ThreadManager;
@ -53,22 +65,7 @@ using webrtc::kVideoCodecVP9;
namespace webrtc_jni {
// Logging macros.
#define TAG_DECODER "MediaCodecVideoDecoder"
#ifdef TRACK_BUFFER_TIMING
#define ALOGV(...)
__android_log_print(ANDROID_LOG_VERBOSE, TAG_DECODER, __VA_ARGS__)
#else
#define ALOGV(...)
#endif
#define ALOGD LOG_TAG(rtc::LS_INFO, TAG_DECODER)
#define ALOGW LOG_TAG(rtc::LS_WARNING, TAG_DECODER)
#define ALOGE LOG_TAG(rtc::LS_ERROR, TAG_DECODER)
enum { kMaxWarningLogFrames = 2 };
class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
public rtc::MessageHandler {
class MediaCodecVideoDecoder : public webrtc::VideoDecoder {
public:
explicit MediaCodecVideoDecoder(
JNIEnv* jni, VideoCodecType codecType, jobject render_egl_context);
@ -83,6 +80,8 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
const CodecSpecificInfo* codecSpecificInfo = NULL,
int64_t renderTimeMs = -1) override;
void PollDecodedFrames() override;
int32_t RegisterDecodeCompleteCallback(DecodedImageCallback* callback)
override;
@ -90,22 +89,41 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
bool PrefersLateDecoding() const override { return true; }
// rtc::MessageHandler implementation.
void OnMessage(rtc::Message* msg) override;
const char* ImplementationName() const override;
private:
// CHECK-fail if not running on |codec_thread_|.
void CheckOnCodecThread();
struct DecodedFrame {
DecodedFrame(VideoFrame frame,
int decode_time_ms,
int64_t timestamp,
int64_t ntp_timestamp,
rtc::Optional<uint8_t> qp)
: frame(std::move(frame)),
decode_time_ms(decode_time_ms),
qp(std::move(qp)) {
frame.set_timestamp(timestamp);
frame.set_ntp_time_ms(ntp_timestamp);
}
VideoFrame frame;
int decode_time_ms;
rtc::Optional<uint8_t> qp;
};
// Returns true if running on |codec_thread_|. Used for DCHECKing.
bool IsOnCodecThread();
int32_t InitDecodeOnCodecThread();
int32_t ResetDecodeOnCodecThread();
int32_t ReleaseOnCodecThread();
int32_t DecodeOnCodecThread(const EncodedImage& inputImage);
int32_t DecodeOnCodecThread(const EncodedImage& inputImage,
std::vector<DecodedFrame>* frames);
void PollDecodedFramesOnCodecThread(std::vector<DecodedFrame>* frames);
// Deliver any outputs pending in the MediaCodec to our |callback_| and return
// true on success.
bool DeliverPendingOutputs(JNIEnv* jni, int dequeue_timeout_us);
bool DeliverPendingOutputs(JNIEnv* jni,
int dequeue_timeout_us,
std::vector<DecodedFrame>* frames);
int32_t ProcessHWErrorOnCodecThread();
void EnableFrameLogOnWarning();
void ResetVariables();
@ -179,6 +197,9 @@ class MediaCodecVideoDecoder : public webrtc::VideoDecoder,
// Global references; must be deleted in Release().
std::vector<jobject> input_buffers_;
// Added to on the codec thread, frames are delivered on the decoder thread.
std::vector<DecodedFrame> decoded_frames_;
};
MediaCodecVideoDecoder::MediaCodecVideoDecoder(
@ -200,7 +221,7 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder(
"<init>",
"()V"))) {
codec_thread_->SetName("MediaCodecVideoDecoder", NULL);
RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoDecoder";
RTC_CHECK(codec_thread_->Start());
j_init_decode_method_ = GetMethodID(
jni, *j_media_codec_video_decoder_class_, "initDecode",
@ -295,7 +316,7 @@ int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst,
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
// Factory should guard against other codecs being used with us.
RTC_CHECK(inst->codecType == codecType_)
RTC_DCHECK(inst->codecType == codecType_)
<< "Unsupported codec " << inst->codecType << " for " << codecType_;
if (sw_fallback_required_) {
@ -316,7 +337,7 @@ int32_t MediaCodecVideoDecoder::InitDecode(const VideoCodec* inst,
}
void MediaCodecVideoDecoder::ResetVariables() {
CheckOnCodecThread();
RTC_DCHECK(IsOnCodecThread());
key_frame_required_ = true;
frames_received_ = 0;
@ -331,7 +352,7 @@ void MediaCodecVideoDecoder::ResetVariables() {
}
int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
CheckOnCodecThread();
RTC_DCHECK(IsOnCodecThread());
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
ALOGD << "InitDecodeOnCodecThread Type: " << (int)codecType_ << ". "
@ -405,13 +426,11 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
}
}
codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t MediaCodecVideoDecoder::ResetDecodeOnCodecThread() {
CheckOnCodecThread();
RTC_DCHECK(IsOnCodecThread());
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
ALOGD << "ResetDecodeOnCodecThread Type: " << (int)codecType_ << ". "
@ -420,7 +439,6 @@ int32_t MediaCodecVideoDecoder::ResetDecodeOnCodecThread() {
". Frames decoded: " << frames_decoded_;
inited_ = false;
rtc::MessageQueueManager::Clear(this);
ResetVariables();
jni->CallVoidMethod(
@ -436,8 +454,6 @@ int32_t MediaCodecVideoDecoder::ResetDecodeOnCodecThread() {
}
inited_ = true;
codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
return WEBRTC_VIDEO_CODEC_OK;
}
@ -448,10 +464,10 @@ int32_t MediaCodecVideoDecoder::Release() {
}
int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() {
RTC_DCHECK(IsOnCodecThread());
if (!inited_) {
return WEBRTC_VIDEO_CODEC_OK;
}
CheckOnCodecThread();
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ALOGD << "DecoderReleaseOnCodecThread: Frames received: " <<
frames_received_ << ". Frames decoded: " << frames_decoded_;
@ -463,7 +479,6 @@ int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() {
jni->CallVoidMethod(*j_media_codec_video_decoder_, j_release_method_);
surface_texture_helper_ = nullptr;
inited_ = false;
rtc::MessageQueueManager::Clear(this);
if (CheckException(jni)) {
ALOGE << "Decoder release exception";
return WEBRTC_VIDEO_CODEC_ERROR;
@ -472,19 +487,19 @@ int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() {
return WEBRTC_VIDEO_CODEC_OK;
}
void MediaCodecVideoDecoder::CheckOnCodecThread() {
RTC_CHECK(codec_thread_.get() == ThreadManager::Instance()->CurrentThread())
<< "Running on wrong thread!";
bool MediaCodecVideoDecoder::IsOnCodecThread() {
return codec_thread_.get() == ThreadManager::Instance()->CurrentThread();
}
void MediaCodecVideoDecoder::EnableFrameLogOnWarning() {
// Log next 2 output frames.
static const int kMaxWarningLogFrames = 2;
frames_decoded_logged_ = std::max(
frames_decoded_logged_, frames_decoded_ + kMaxWarningLogFrames);
}
int32_t MediaCodecVideoDecoder::ProcessHWErrorOnCodecThread() {
CheckOnCodecThread();
RTC_DCHECK(IsOnCodecThread());
int ret_val = ReleaseOnCodecThread();
if (ret_val < 0) {
ALOGE << "ProcessHWError: Release failure";
@ -515,22 +530,17 @@ int32_t MediaCodecVideoDecoder::Decode(
const RTPFragmentationHeader* fragmentation,
const CodecSpecificInfo* codecSpecificInfo,
int64_t renderTimeMs) {
RTC_DCHECK(callback_);
RTC_DCHECK(inited_);
if (sw_fallback_required_) {
ALOGE << "Decode() - fallback to SW codec";
return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
}
if (callback_ == NULL) {
ALOGE << "Decode() - callback_ is NULL";
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (inputImage._buffer == NULL && inputImage._length > 0) {
ALOGE << "Decode() - inputImage is incorrect";
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (!inited_) {
ALOGE << "Decode() - decoder is not initialized";
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
// Check if encoded frame dimension has changed.
if ((inputImage._encodedWidth * inputImage._encodedHeight > 0) &&
@ -575,14 +585,32 @@ int32_t MediaCodecVideoDecoder::Decode(
return WEBRTC_VIDEO_CODEC_ERROR;
}
return codec_thread_->Invoke<int32_t>(
std::vector<DecodedFrame> frames;
int32_t ret = codec_thread_->Invoke<int32_t>(
RTC_FROM_HERE, Bind(&MediaCodecVideoDecoder::DecodeOnCodecThread, this,
inputImage, &frames));
for (auto& f : frames)
callback_->Decoded(f.frame, rtc::Optional<int32_t>(f.decode_time_ms), f.qp);
return ret;
}
void MediaCodecVideoDecoder::PollDecodedFrames() {
RTC_DCHECK(callback_);
std::vector<DecodedFrame> frames;
codec_thread_->Invoke<void>(
RTC_FROM_HERE,
Bind(&MediaCodecVideoDecoder::DecodeOnCodecThread, this, inputImage));
Bind(&MediaCodecVideoDecoder::PollDecodedFramesOnCodecThread, this,
&frames));
for (auto& f : frames)
callback_->Decoded(f.frame, rtc::Optional<int32_t>(f.decode_time_ms), f.qp);
}
int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
const EncodedImage& inputImage) {
CheckOnCodecThread();
const EncodedImage& inputImage,
std::vector<DecodedFrame>* frames) {
RTC_DCHECK(IsOnCodecThread());
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
@ -598,7 +626,7 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
const int64 drain_start = rtc::TimeMillis();
while ((frames_received_ > frames_decoded_ + max_pending_frames_) &&
(rtc::TimeMillis() - drain_start) < kMediaCodecTimeoutMs) {
if (!DeliverPendingOutputs(jni, kMediaCodecPollMs)) {
if (!DeliverPendingOutputs(jni, kMediaCodecPollMs, frames)) {
ALOGE << "DeliverPendingOutputs error. Frames received: " <<
frames_received_ << ". Frames decoded: " << frames_decoded_;
return ProcessHWErrorOnCodecThread();
@ -618,7 +646,7 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
". Retry DeliverPendingOutputs.";
EnableFrameLogOnWarning();
// Try to drain the decoder.
if (!DeliverPendingOutputs(jni, kMediaCodecPollMs)) {
if (!DeliverPendingOutputs(jni, kMediaCodecPollMs, frames)) {
ALOGE << "DeliverPendingOutputs error. Frames received: " <<
frames_received_ << ". Frames decoded: " << frames_decoded_;
return ProcessHWErrorOnCodecThread();
@ -636,7 +664,7 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
jobject j_input_buffer = input_buffers_[j_input_buffer_index];
uint8_t* buffer =
reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer));
RTC_CHECK(buffer) << "Indirect buffer??";
RTC_DCHECK(buffer) << "Indirect buffer??";
int64_t buffer_capacity = jni->GetDirectBufferCapacity(j_input_buffer);
if (CheckException(jni) || buffer_capacity < inputImage._length) {
ALOGE << "Input frame size "<< inputImage._length <<
@ -689,7 +717,7 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
}
// Try to drain the decoder
if (!DeliverPendingOutputs(jni, 0)) {
if (!DeliverPendingOutputs(jni, 0, frames)) {
ALOGE << "DeliverPendingOutputs error";
return ProcessHWErrorOnCodecThread();
}
@ -697,9 +725,26 @@ int32_t MediaCodecVideoDecoder::DecodeOnCodecThread(
return WEBRTC_VIDEO_CODEC_OK;
}
void MediaCodecVideoDecoder::PollDecodedFramesOnCodecThread(
std::vector<DecodedFrame>* frames) {
RTC_DCHECK(IsOnCodecThread());
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
if (!DeliverPendingOutputs(jni, 0, frames)) {
ALOGE << "PollDecodedFramesOnCodecThread: DeliverPendingOutputs error";
ProcessHWErrorOnCodecThread();
}
}
bool MediaCodecVideoDecoder::DeliverPendingOutputs(
JNIEnv* jni, int dequeue_timeout_ms) {
CheckOnCodecThread();
JNIEnv* jni,
int dequeue_timeout_ms,
std::vector<DecodedFrame>* frames) {
RTC_DCHECK(IsOnCodecThread());
RTC_DCHECK(frames);
if (frames_received_ <= frames_decoded_) {
// No need to query for output buffers - decoder is drained.
return true;
@ -808,7 +853,7 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs(
rtc::scoped_refptr<webrtc::I420Buffer> i420_buffer =
decoded_frame_pool_.CreateBuffer(width, height);
if (color_format == COLOR_FormatYUV420Planar) {
RTC_CHECK_EQ(0, stride % 2);
RTC_DCHECK_EQ(0, stride % 2);
const int uv_stride = stride / 2;
const uint8_t* y_ptr = payload;
const uint8_t* u_ptr = y_ptr + stride * slice_height;
@ -834,7 +879,7 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs(
i420_buffer->MutableDataV(), i420_buffer->StrideV(),
chroma_width, chroma_height);
if (slice_height % 2 == 1) {
RTC_CHECK_EQ(height, slice_height);
RTC_DCHECK_EQ(height, slice_height);
// Duplicate the last chroma rows.
uint8_t* u_last_row_ptr = i420_buffer->MutableDataU() +
chroma_height * i420_buffer->StrideU();
@ -909,9 +954,16 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs(
rtc::Optional<uint8_t> qp = pending_frame_qps_.front();
pending_frame_qps_.pop_front();
callback_->Decoded(decoded_frame, rtc::Optional<int32_t>(decode_time_ms),
qp);
decoded_frames_.push_back(DecodedFrame(std::move(decoded_frame),
decode_time_ms, output_timestamps_ms,
output_ntp_timestamps_ms, qp));
}
frames->reserve(frames->size() + decoded_frames_.size());
std::move(decoded_frames_.begin(), decoded_frames_.end(),
std::back_inserter(*frames));
decoded_frames_.clear();
return true;
}
@ -921,26 +973,6 @@ int32_t MediaCodecVideoDecoder::RegisterDecodeCompleteCallback(
return WEBRTC_VIDEO_CODEC_OK;
}
void MediaCodecVideoDecoder::OnMessage(rtc::Message* msg) {
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedLocalRefFrame local_ref_frame(jni);
if (!inited_) {
return;
}
// We only ever send one message to |this| directly (not through a Bind()'d
// functor), so expect no ID/data.
RTC_CHECK(!msg->message_id) << "Unexpected message!";
RTC_CHECK(!msg->pdata) << "Unexpected message!";
CheckOnCodecThread();
if (!DeliverPendingOutputs(jni, 0)) {
ALOGE << "OnMessage: DeliverPendingOutputs error";
ProcessHWErrorOnCodecThread();
return;
}
codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
}
MediaCodecVideoDecoderFactory::MediaCodecVideoDecoderFactory()
: egl_context_(nullptr) {
ALOGD << "MediaCodecVideoDecoderFactory ctor";