zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Deliver video frames on Android, on the decode thread.

Browse Source 
VideoCoding
* Adding a method for polling for frames on Android only until the capture implementation takes care of this (longer term plan).

CodecDatabase
* Add an accessor for the current decoder
* Use std::unique_ptr<> for ownership.
* Remove "Release()" and "ReleaseDecoder()". Instead just delete.
* Remove |friend| relationship between CodecDatabase and VCMGenericDecoder.

VCMDecodedFrameCallback
* DCHECKs for thread correctness.
* Remove |lock_| now that a threading model has been established and verified.

VCMGenericDecoder
* All methods now have thread checks.
* Variable access associated with thread checkers.

VideoReceiver
* Added two notification methods, DecoderThreadStarting() and DecoderThreadStopped()
  * Allows us to establish a period when the decoder thread is not running and it is safe to modify variables such as callbacks, that are only read when the decoder thread is running.
  * Allows us to DCHECK thread guarantees.
  * Allows synchronizing callbacks from the module process thread and have them only active while the decoder thread is running.
  * The above, allows us to establish two modes for the thread, single-threaded-mutable and multi-threaded-const.
  * Using that knowledge, we can remove |receive_crit_| as well as locking for a number of member variables.

MediaCodecVideoDecoder
* Removed frame polling code from this class, since this is now done from the root thread function in VideoReceiveStream.

VideoReceiveStream
* On Android: Polls for decoded frames every 10ms (same interval as previously in MediaCodecVideoDecoder)
* [Un]Registers the |video_receiver_| with the module thread only around the time the decoder thread is started/stopped.
* Notifies the receiver of start/stop events of the decoder thread.
* Changed the decoder thread to use the new PlatformThread callback type.

BUG=webrtc:7361, 695438

Review-Url: https://codereview.webrtc.org/2764573002
Cr-Commit-Position: refs/heads/master@{#17527}
This commit is contained in:
tommi
2017-04-04 03:53:02 -07:00
committed by Commit bot
parent aa7d935cd5
commit e3aa88bbd5
12 changed files with 511 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
webrtc/modules/video_coding/codec_database.cc
View File

@ -71,6 +71,31 @@ VideoCodecH264 VideoEncoder::GetDefaultH264Settings() {
return h264_settings;
}
// Create an internal Decoder given a codec type
static std::unique_ptr<VCMGenericDecoder> CreateDecoder(VideoCodecType type) {
switch (type) {
case kVideoCodecVP8:
return std::unique_ptr<VCMGenericDecoder>(
new VCMGenericDecoder(VP8Decoder::Create()));
case kVideoCodecVP9:
return std::unique_ptr<VCMGenericDecoder>(
new VCMGenericDecoder(VP9Decoder::Create()));
case kVideoCodecI420:
return std::unique_ptr<VCMGenericDecoder>(
new VCMGenericDecoder(new I420Decoder()));
case kVideoCodecH264:
if (H264Decoder::IsSupported()) {
return std::unique_ptr<VCMGenericDecoder>(
new VCMGenericDecoder(H264Decoder::Create()));
}
break;
default:
break;
}
LOG(LS_WARNING) << "No internal decoder of this type exists.";
return std::unique_ptr<VCMGenericDecoder>();
}
VCMDecoderMapItem::VCMDecoderMapItem(VideoCodec* settings,
int number_of_cores,
bool require_key_frame)
@ -98,13 +123,12 @@ VCMCodecDataBase::VCMCodecDataBase(
external_encoder_(nullptr),
internal_source_(false),
encoded_frame_callback_(encoded_frame_callback),
ptr_decoder_(nullptr),
dec_map_(),
dec_external_map_() {}
VCMCodecDataBase::~VCMCodecDataBase() {
DeleteEncoder();
ReleaseDecoder(ptr_decoder_);
ptr_decoder_.reset();
for (auto& kv : dec_map_)
delete kv.second;
for (auto& kv : dec_external_map_)
@ -391,11 +415,10 @@ bool VCMCodecDataBase::DeregisterExternalDecoder(uint8_t payload_type) {
// We can't use payload_type to check if the decoder is currently in use,
// because payload type may be out of date (e.g. before we decode the first
// frame after RegisterReceiveCodec)
if (ptr_decoder_ != nullptr &&
ptr_decoder_->_decoder == (*it).second->external_decoder_instance) {
if (ptr_decoder_ &&
ptr_decoder_->IsSameDecoder((*it).second->external_decoder_instance)) {
// Release it if it was registered and in use.
ReleaseDecoder(ptr_decoder_);
ptr_decoder_ = nullptr;
ptr_decoder_.reset();
}
DeregisterReceiveCodec(payload_type);
delete it->second;
@ -455,12 +478,11 @@ VCMGenericDecoder* VCMCodecDataBase::GetDecoder(
RTC_DCHECK(decoded_frame_callback->UserReceiveCallback());
uint8_t payload_type = frame.PayloadType();
if (payload_type == receive_codec_.plType || payload_type == 0) {
return ptr_decoder_;
return ptr_decoder_.get();
}
// Check for exisitng decoder, if exists - delete.
if (ptr_decoder_) {
ReleaseDecoder(ptr_decoder_);
ptr_decoder_ = nullptr;
ptr_decoder_.reset();
memset(&receive_codec_, 0, sizeof(VideoCodec));
}
ptr_decoder_ = CreateAndInitDecoder(frame, &receive_codec_);
@ -471,36 +493,26 @@ VCMGenericDecoder* VCMCodecDataBase::GetDecoder(
callback->OnIncomingPayloadType(receive_codec_.plType);
if (ptr_decoder_->RegisterDecodeCompleteCallback(decoded_frame_callback) <
0) {
ReleaseDecoder(ptr_decoder_);
ptr_decoder_ = nullptr;
ptr_decoder_.reset();
memset(&receive_codec_, 0, sizeof(VideoCodec));
return nullptr;
}
return ptr_decoder_;
return ptr_decoder_.get();
}
void VCMCodecDataBase::ReleaseDecoder(VCMGenericDecoder* decoder) const {
if (decoder) {
RTC_DCHECK(decoder->_decoder);
decoder->Release();
if (!decoder->External()) {
delete decoder->_decoder;
}
delete decoder;
}
VCMGenericDecoder* VCMCodecDataBase::GetCurrentDecoder() {
return ptr_decoder_.get();
}
bool VCMCodecDataBase::PrefersLateDecoding() const {
if (!ptr_decoder_)
return true;
return ptr_decoder_->PrefersLateDecoding();
return ptr_decoder_ ? ptr_decoder_->PrefersLateDecoding() : true;
}
bool VCMCodecDataBase::MatchesCurrentResolution(int width, int height) const {
return send_codec_.width == width && send_codec_.height == height;
}
VCMGenericDecoder* VCMCodecDataBase::CreateAndInitDecoder(
std::unique_ptr<VCMGenericDecoder> VCMCodecDataBase::CreateAndInitDecoder(
const VCMEncodedFrame& frame,
VideoCodec* new_codec) const {
uint8_t payload_type = frame.PayloadType();
@ -513,13 +525,13 @@ VCMGenericDecoder* VCMCodecDataBase::CreateAndInitDecoder(
<< static_cast<int>(payload_type);
return nullptr;
}
VCMGenericDecoder* ptr_decoder = nullptr;
std::unique_ptr<VCMGenericDecoder> ptr_decoder;
const VCMExtDecoderMapItem* external_dec_item =
FindExternalDecoderItem(payload_type);
if (external_dec_item) {
// External codec.
ptr_decoder = new VCMGenericDecoder(
external_dec_item->external_decoder_instance, true);
ptr_decoder.reset(new VCMGenericDecoder(
external_dec_item->external_decoder_instance, true));
} else {
// Create decoder.
ptr_decoder = CreateDecoder(decoder_item->settings->codecType);
@ -538,7 +550,6 @@ VCMGenericDecoder* VCMCodecDataBase::CreateAndInitDecoder(
}
if (ptr_decoder->InitDecode(decoder_item->settings.get(),
decoder_item->number_of_cores) < 0) {
ReleaseDecoder(ptr_decoder);
return nullptr;
}
memcpy(new_codec, decoder_item->settings.get(), sizeof(VideoCodec));
@ -552,26 +563,6 @@ void VCMCodecDataBase::DeleteEncoder() {
ptr_encoder_.reset();
}
VCMGenericDecoder* VCMCodecDataBase::CreateDecoder(VideoCodecType type) const {
switch (type) {
case kVideoCodecVP8:
return new VCMGenericDecoder(VP8Decoder::Create());
case kVideoCodecVP9:
return new VCMGenericDecoder(VP9Decoder::Create());
case kVideoCodecI420:
return new VCMGenericDecoder(new I420Decoder());
case kVideoCodecH264:
if (H264Decoder::IsSupported()) {
return new VCMGenericDecoder(H264Decoder::Create());
}
break;
default:
break;
}
LOG(LS_WARNING) << "No internal decoder of this type exists.";
return nullptr;
}
const VCMDecoderMapItem* VCMCodecDataBase::FindDecoderItem(
uint8_t payload_type) const {
DecoderMap::const_iterator it = dec_map_.find(payload_type);

16
webrtc/modules/video_coding/codec_database.h
View File

@ -107,9 +107,9 @@ class VCMCodecDataBase {
const VCMEncodedFrame& frame,
VCMDecodedFrameCallback* decoded_frame_callback);
// Deletes the memory of the decoder instance |decoder|. Used to delete
// deep copies returned by CreateDecoderCopy().
void ReleaseDecoder(VCMGenericDecoder* decoder) const;
// Returns the current decoder (i.e. the same value as was last returned from
// GetDecoder();
VCMGenericDecoder* GetCurrentDecoder();
// Returns true if the currently active decoder prefer to decode frames late.
// That means that frames must be decoded near the render times stamp.
@ -121,8 +121,9 @@ class VCMCodecDataBase {
typedef std::map<uint8_t, VCMDecoderMapItem*> DecoderMap;
typedef std::map<uint8_t, VCMExtDecoderMapItem*> ExternalDecoderMap;
VCMGenericDecoder* CreateAndInitDecoder(const VCMEncodedFrame& frame,
VideoCodec* new_codec) const;
std::unique_ptr<VCMGenericDecoder> CreateAndInitDecoder(
const VCMEncodedFrame& frame,
VideoCodec* new_codec) const;
// Determines whether a new codec has to be created or not.
// Checks every setting apart from maxFramerate and startBitrate.
@ -130,9 +131,6 @@ class VCMCodecDataBase {
void DeleteEncoder();
// Create an internal Decoder given a codec type
VCMGenericDecoder* CreateDecoder(VideoCodecType type) const;
const VCMDecoderMapItem* FindDecoderItem(uint8_t payload_type) const;
const VCMExtDecoderMapItem* FindExternalDecoderItem(
@ -149,7 +147,7 @@ class VCMCodecDataBase {
bool internal_source_;
VCMEncodedFrameCallback* const encoded_frame_callback_;
std::unique_ptr<VCMGenericEncoder> ptr_encoder_;
VCMGenericDecoder* ptr_decoder_;
std::unique_ptr<VCMGenericDecoder> ptr_decoder_;
DecoderMap dec_map_;
ExternalDecoderMap dec_external_map_;
}; // VCMCodecDataBase

113
webrtc/modules/video_coding/generic_decoder.cc
View File

@ -8,11 +8,12 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/modules/video_coding/generic_decoder.h"
#include "webrtc/base/checks.h"
#include "webrtc/base/logging.h"
#include "webrtc/base/trace_event.h"
#include "webrtc/modules/video_coding/include/video_coding.h"
#include "webrtc/modules/video_coding/generic_decoder.h"
#include "webrtc/modules/video_coding/internal_defines.h"
#include "webrtc/system_wrappers/include/clock.h"
@ -23,9 +24,12 @@ VCMDecodedFrameCallback::VCMDecodedFrameCallback(VCMTiming* timing,
: _clock(clock),
_timing(timing),
_timestampMap(kDecoderFrameMemoryLength),
_lastReceivedPictureID(0) {}
_lastReceivedPictureID(0) {
decoder_thread_.DetachFromThread();
}
VCMDecodedFrameCallback::~VCMDecodedFrameCallback() {
RTC_DCHECK(construction_thread_.CalledOnValidThread());
}
void VCMDecodedFrameCallback::SetUserReceiveCallback(
@ -37,6 +41,7 @@ void VCMDecodedFrameCallback::SetUserReceiveCallback(
}
VCMReceiveCallback* VCMDecodedFrameCallback::UserReceiveCallback() {
RTC_DCHECK_RUN_ON(&decoder_thread_);
// Called on the decode thread via VCMCodecDataBase::GetDecoder.
// The callback must always have been set before this happens.
RTC_DCHECK(_receiveCallback);
@ -59,16 +64,14 @@ int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
rtc::Optional<int32_t> decode_time_ms,
rtc::Optional<uint8_t> qp) {
RTC_DCHECK_RUN_ON(&decoder_thread_);
RTC_DCHECK(_receiveCallback) << "Callback must not be null at this point";
TRACE_EVENT_INSTANT1("webrtc", "VCMDecodedFrameCallback::Decoded",
"timestamp", decodedImage.timestamp());
// TODO(holmer): We should improve this so that we can handle multiple
// callbacks from one call to Decode().
VCMFrameInformation* frameInfo;
{
rtc::CritScope cs(&lock_);
frameInfo = _timestampMap.Pop(decodedImage.timestamp());
}
VCMFrameInformation* frameInfo = _timestampMap.Pop(decodedImage.timestamp());
if (frameInfo == NULL) {
LOG(LS_WARNING) << "Too many frames backed up in the decoder, dropping "
@ -92,101 +95,115 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
int32_t VCMDecodedFrameCallback::ReceivedDecodedReferenceFrame(
const uint64_t pictureId) {
RTC_DCHECK_RUN_ON(&decoder_thread_);
return _receiveCallback->ReceivedDecodedReferenceFrame(pictureId);
}
int32_t VCMDecodedFrameCallback::ReceivedDecodedFrame(
const uint64_t pictureId) {
RTC_DCHECK_RUN_ON(&decoder_thread_);
_lastReceivedPictureID = pictureId;
return 0;
}
uint64_t VCMDecodedFrameCallback::LastReceivedPictureID() const {
RTC_DCHECK_RUN_ON(&decoder_thread_);
return _lastReceivedPictureID;
}
void VCMDecodedFrameCallback::OnDecoderImplementationName(
const char* implementation_name) {
RTC_DCHECK_RUN_ON(&decoder_thread_);
_receiveCallback->OnDecoderImplementationName(implementation_name);
}
void VCMDecodedFrameCallback::Map(uint32_t timestamp,
VCMFrameInformation* frameInfo) {
rtc::CritScope cs(&lock_);
RTC_DCHECK_RUN_ON(&decoder_thread_);
_timestampMap.Add(timestamp, frameInfo);
}
int32_t VCMDecodedFrameCallback::Pop(uint32_t timestamp) {
rtc::CritScope cs(&lock_);
if (_timestampMap.Pop(timestamp) == NULL) {
return VCM_GENERAL_ERROR;
}
return VCM_OK;
RTC_DCHECK_RUN_ON(&decoder_thread_);
return _timestampMap.Pop(timestamp) == nullptr ? VCM_GENERAL_ERROR : VCM_OK;
}
VCMGenericDecoder::VCMGenericDecoder(VideoDecoder* decoder, bool isExternal)
: _callback(NULL),
_frameInfos(),
_nextFrameInfoIdx(0),
_decoder(decoder),
decoder_(decoder),
_codecType(kVideoCodecUnknown),
_isExternal(isExternal),
_keyFrameDecoded(false) {}
_isExternal(isExternal) {}
VCMGenericDecoder::~VCMGenericDecoder() {}
VCMGenericDecoder::~VCMGenericDecoder() {
decoder_->Release();
if (_isExternal)
decoder_.release();
RTC_DCHECK(_isExternal || !decoder_);
}
int32_t VCMGenericDecoder::InitDecode(const VideoCodec* settings,
int32_t numberOfCores) {
RTC_DCHECK_RUN_ON(&decoder_thread_);
TRACE_EVENT0("webrtc", "VCMGenericDecoder::InitDecode");
_codecType = settings->codecType;
return _decoder->InitDecode(settings, numberOfCores);
return decoder_->InitDecode(settings, numberOfCores);
}
int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, int64_t nowMs) {
TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp",
frame.EncodedImage()._timeStamp);
_frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs;
_frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs();
_frameInfos[_nextFrameInfoIdx].rotation = frame.rotation();
_callback->Map(frame.TimeStamp(), &_frameInfos[_nextFrameInfoIdx]);
RTC_DCHECK_RUN_ON(&decoder_thread_);
TRACE_EVENT2("webrtc", "VCMGenericDecoder::Decode", "timestamp",
frame.EncodedImage()._timeStamp, "decoder",
decoder_->ImplementationName());
_frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs;
_frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs();
_frameInfos[_nextFrameInfoIdx].rotation = frame.rotation();
_callback->Map(frame.TimeStamp(), &_frameInfos[_nextFrameInfoIdx]);
_nextFrameInfoIdx = (_nextFrameInfoIdx + 1) % kDecoderFrameMemoryLength;
const RTPFragmentationHeader dummy_header;
int32_t ret = _decoder->Decode(frame.EncodedImage(), frame.MissingFrame(),
&dummy_header,
frame.CodecSpecific(), frame.RenderTimeMs());
_nextFrameInfoIdx = (_nextFrameInfoIdx + 1) % kDecoderFrameMemoryLength;
const RTPFragmentationHeader dummy_header;
int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(),
&dummy_header, frame.CodecSpecific(),
frame.RenderTimeMs());
_callback->OnDecoderImplementationName(_decoder->ImplementationName());
// TODO(tommi): Necessary every time?
// Maybe this should be the first thing the function does, and only the first
// time around?
_callback->OnDecoderImplementationName(decoder_->ImplementationName());
if (ret != WEBRTC_VIDEO_CODEC_OK) {
if (ret < WEBRTC_VIDEO_CODEC_OK) {
LOG(LS_WARNING) << "Failed to decode frame with timestamp "
<< frame.TimeStamp() << ", error code: " << ret;
_callback->Pop(frame.TimeStamp());
return ret;
} else if (ret == WEBRTC_VIDEO_CODEC_NO_OUTPUT ||
ret == WEBRTC_VIDEO_CODEC_REQUEST_SLI) {
// No output
_callback->Pop(frame.TimeStamp());
LOG(LS_WARNING) << "Failed to decode frame with timestamp "
<< frame.TimeStamp() << ", error code: " << ret;
}
return ret;
}
// We pop the frame for all non-'OK', failure or success codes such as
// WEBRTC_VIDEO_CODEC_NO_OUTPUT and WEBRTC_VIDEO_CODEC_REQUEST_SLI.
_callback->Pop(frame.TimeStamp());
}
int32_t VCMGenericDecoder::Release() {
return _decoder->Release();
return ret;
}
int32_t VCMGenericDecoder::RegisterDecodeCompleteCallback(
VCMDecodedFrameCallback* callback) {
RTC_DCHECK_RUN_ON(&decoder_thread_);
_callback = callback;
return _decoder->RegisterDecodeCompleteCallback(callback);
}
bool VCMGenericDecoder::External() const {
return _isExternal;
return decoder_->RegisterDecodeCompleteCallback(callback);
}
bool VCMGenericDecoder::PrefersLateDecoding() const {
return _decoder->PrefersLateDecoding();
RTC_DCHECK_RUN_ON(&decoder_thread_);
return decoder_->PrefersLateDecoding();
}
#if defined(WEBRTC_ANDROID)
void VCMGenericDecoder::PollDecodedFrames() {
RTC_DCHECK_RUN_ON(&decoder_thread_);
decoder_->PollDecodedFrames();
}
#endif
} // namespace webrtc

45
webrtc/modules/video_coding/generic_decoder.h
View File

@ -11,7 +11,8 @@
#ifndef WEBRTC_MODULES_VIDEO_CODING_GENERIC_DECODER_H_
#define WEBRTC_MODULES_VIDEO_CODING_GENERIC_DECODER_H_
#include "webrtc/base/criticalsection.h"
#include <memory>
#include "webrtc/base/thread_checker.h"
#include "webrtc/modules/include/module_common_types.h"
#include "webrtc/modules/video_coding/encoded_frame.h"
@ -36,6 +37,7 @@ class VCMDecodedFrameCallback : public DecodedImageCallback {
public:
VCMDecodedFrameCallback(VCMTiming* timing, Clock* clock);
~VCMDecodedFrameCallback() override;
void SetUserReceiveCallback(VCMReceiveCallback* receiveCallback);
VCMReceiveCallback* UserReceiveCallback();
@ -55,7 +57,7 @@ class VCMDecodedFrameCallback : public DecodedImageCallback {
private:
rtc::ThreadChecker construction_thread_;
// Protect |_timestampMap|.
rtc::ThreadChecker decoder_thread_;
Clock* const _clock;
// This callback must be set before the decoder thread starts running
// and must only be unset when external threads (e.g decoder thread)
@ -63,15 +65,12 @@ class VCMDecodedFrameCallback : public DecodedImageCallback {
// while there are more than one threads involved, it must be set
// from the same thread, and therfore a lock is not required to access it.
VCMReceiveCallback* _receiveCallback = nullptr;
VCMTiming* _timing;
rtc::CriticalSection lock_;
VCMTimestampMap _timestampMap GUARDED_BY(lock_);
uint64_t _lastReceivedPictureID;
VCMTiming* _timing ACCESS_ON(decoder_thread_);
VCMTimestampMap _timestampMap ACCESS_ON(decoder_thread_);
uint64_t _lastReceivedPictureID ACCESS_ON(decoder_thread_);
};
class VCMGenericDecoder {
friend class VCMCodecDataBase;
public:
explicit VCMGenericDecoder(VideoDecoder* decoder, bool isExternal = false);
~VCMGenericDecoder();
@ -88,27 +87,31 @@ class VCMGenericDecoder {
*/
int32_t Decode(const VCMEncodedFrame& inputFrame, int64_t nowMs);
/**
* Free the decoder memory
*/
int32_t Release();
/**
* Set decode callback. Deregistering while decoding is illegal.
*/
int32_t RegisterDecodeCompleteCallback(VCMDecodedFrameCallback* callback);
bool External() const;
bool PrefersLateDecoding() const;
#if defined(WEBRTC_ANDROID)
// See https://bugs.chromium.org/p/webrtc/issues/detail?id=7361
void PollDecodedFrames();
#endif
bool IsSameDecoder(VideoDecoder* decoder) const {
return decoder_.get() == decoder;
}
private:
VCMDecodedFrameCallback* _callback;
VCMFrameInformation _frameInfos[kDecoderFrameMemoryLength];
uint32_t _nextFrameInfoIdx;
VideoDecoder* const _decoder;
VideoCodecType _codecType;
bool _isExternal;
bool _keyFrameDecoded;
rtc::ThreadChecker decoder_thread_;
VCMDecodedFrameCallback* _callback ACCESS_ON(decoder_thread_);
VCMFrameInformation _frameInfos[kDecoderFrameMemoryLength] ACCESS_ON(
decoder_thread_);
uint32_t _nextFrameInfoIdx ACCESS_ON(decoder_thread_);
std::unique_ptr<VideoDecoder> decoder_;
VideoCodecType _codecType ACCESS_ON(decoder_thread_);
const bool _isExternal;
};
} // namespace webrtc

66
webrtc/modules/video_coding/video_coding_impl.h
View File

@ -35,6 +35,7 @@
namespace webrtc {
class ProcessThread;
class VideoBitrateAllocator;
class VideoBitrateAllocationObserver;
@ -150,7 +151,7 @@ class VideoReceiver : public Module {
VCMTiming* timing,
NackSender* nack_sender = nullptr,
KeyFrameRequestSender* keyframe_request_sender = nullptr);
~VideoReceiver();
~VideoReceiver() override;
int32_t RegisterReceiveCodec(const VideoCodec* receiveCodec,
int32_t numberOfCores,
@ -168,8 +169,10 @@ class VideoReceiver : public Module {
int32_t Decode(const webrtc::VCMEncodedFrame* frame);
// Called on the decoder thread when thread is exiting.
void DecodingStopped();
#if defined(WEBRTC_ANDROID)
// See https://bugs.chromium.org/p/webrtc/issues/detail?id=7361
void PollDecodedFrames();
#endif
int32_t IncomingPacket(const uint8_t* incomingPayload,
size_t payloadLength,
@ -195,39 +198,66 @@ class VideoReceiver : public Module {
int64_t TimeUntilNextProcess() override;
void Process() override;
void ProcessThreadAttached(ProcessThread* process_thread) override;
void TriggerDecoderShutdown();
void DecoderThreadStarting();
void DecoderThreadStopped();
protected:
int32_t Decode(const webrtc::VCMEncodedFrame& frame)
EXCLUSIVE_LOCKS_REQUIRED(receive_crit_);
int32_t Decode(const webrtc::VCMEncodedFrame& frame);
int32_t RequestKeyFrame();
private:
// Used for DCHECKing thread correctness.
// In build where DCHECKs are enabled, will return false before
// DecoderThreadStarting is called, then true until DecoderThreadStopped
// is called.
// In builds where DCHECKs aren't enabled, it will return true.
bool IsDecoderThreadRunning();
rtc::ThreadChecker construction_thread_;
rtc::ThreadChecker decoder_thread_;
rtc::ThreadChecker module_thread_;
Clock* const clock_;
rtc::CriticalSection process_crit_;
rtc::CriticalSection receive_crit_;
VCMTiming* _timing;
VCMReceiver _receiver;
VCMDecodedFrameCallback _decodedFrameCallback;
VCMFrameTypeCallback* _frameTypeCallback GUARDED_BY(process_crit_);
VCMReceiveStatisticsCallback* _receiveStatsCallback GUARDED_BY(process_crit_);
VCMPacketRequestCallback* _packetRequestCallback GUARDED_BY(process_crit_);
VCMFrameBuffer _frameFromFile;
// These callbacks are set on the construction thread before being attached
// to the module thread or decoding started, so a lock is not required.
VCMFrameTypeCallback* _frameTypeCallback;
VCMReceiveStatisticsCallback* _receiveStatsCallback;
VCMPacketRequestCallback* _packetRequestCallback;
// Used on both the module and decoder thread.
bool _scheduleKeyRequest GUARDED_BY(process_crit_);
bool drop_frames_until_keyframe_ GUARDED_BY(process_crit_);
size_t max_nack_list_size_ GUARDED_BY(process_crit_);
VCMCodecDataBase _codecDataBase GUARDED_BY(receive_crit_);
EncodedImageCallback* pre_decode_image_callback_;
// Modified on the construction thread while not attached to the process
// thread. Once attached to the process thread, its value is only read
// so a lock is not required.
size_t max_nack_list_size_;
VCMProcessTimer _receiveStatsTimer;
VCMProcessTimer _retransmissionTimer;
VCMProcessTimer _keyRequestTimer;
QpParser qp_parser_;
ThreadUnsafeOneTimeEvent first_frame_received_;
// Callbacks are set before the decoder thread starts.
// Once the decoder thread has been started, usage of |_codecDataBase| moves
// over to the decoder thread.
VCMCodecDataBase _codecDataBase;
EncodedImageCallback* const pre_decode_image_callback_;
VCMProcessTimer _receiveStatsTimer ACCESS_ON(module_thread_);
VCMProcessTimer _retransmissionTimer ACCESS_ON(module_thread_);
VCMProcessTimer _keyRequestTimer ACCESS_ON(module_thread_);
QpParser qp_parser_ ACCESS_ON(decoder_thread_);
ThreadUnsafeOneTimeEvent first_frame_received_ ACCESS_ON(decoder_thread_);
// Modified on the construction thread. Can be read without a lock and assumed
// to be non-null on the module and decoder threads.
ProcessThread* process_thread_ = nullptr;
bool is_attached_to_process_thread_ ACCESS_ON(construction_thread_) = false;
#if RTC_DCHECK_IS_ON
bool decoder_thread_is_running_ = false;
#endif
};
} // namespace vcm

217
webrtc/modules/video_coding/video_receiver.cc
View File

@ -9,12 +9,14 @@
*/
#include "webrtc/base/checks.h"
#include "webrtc/base/location.h"
#include "webrtc/base/logging.h"
#include "webrtc/base/trace_event.h"
#include "webrtc/common_types.h"
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
#include "webrtc/modules/video_coding/include/video_codec_interface.h"
#include "webrtc/modules/utility/include/process_thread.h"
#include "webrtc/modules/video_coding/encoded_frame.h"
#include "webrtc/modules/video_coding/include/video_codec_interface.h"
#include "webrtc/modules/video_coding/jitter_buffer.h"
#include "webrtc/modules/video_coding/packet.h"
#include "webrtc/modules/video_coding/video_coding_impl.h"
@ -40,7 +42,6 @@ VideoReceiver::VideoReceiver(Clock* clock,
_frameTypeCallback(nullptr),
_receiveStatsCallback(nullptr),
_packetRequestCallback(nullptr),
_frameFromFile(),
_scheduleKeyRequest(false),
drop_frames_until_keyframe_(false),
max_nack_list_size_(0),
@ -48,18 +49,23 @@ VideoReceiver::VideoReceiver(Clock* clock,
pre_decode_image_callback_(pre_decode_image_callback),
_receiveStatsTimer(1000, clock_),
_retransmissionTimer(10, clock_),
_keyRequestTimer(500, clock_) {}
_keyRequestTimer(500, clock_) {
decoder_thread_.DetachFromThread();
module_thread_.DetachFromThread();
}
VideoReceiver::~VideoReceiver() {}
VideoReceiver::~VideoReceiver() {
RTC_DCHECK_RUN_ON(&construction_thread_);
}
void VideoReceiver::Process() {
RTC_DCHECK_RUN_ON(&module_thread_);
// Receive-side statistics
// TODO(philipel): Remove this if block when we know what to do with
// ReceiveStatisticsProxy::QualitySample.
if (_receiveStatsTimer.TimeUntilProcess() == 0) {
_receiveStatsTimer.Processed();
rtc::CritScope cs(&process_crit_);
if (_receiveStatsCallback != nullptr) {
_receiveStatsCallback->OnReceiveRatesUpdated(0, 0);
}
@ -68,10 +74,10 @@ void VideoReceiver::Process() {
// Key frame requests
if (_keyRequestTimer.TimeUntilProcess() == 0) {
_keyRequestTimer.Processed();
bool request_key_frame = false;
{
bool request_key_frame = _frameTypeCallback != nullptr;
if (request_key_frame) {
rtc::CritScope cs(&process_crit_);
request_key_frame = _scheduleKeyRequest && _frameTypeCallback != nullptr;
request_key_frame = _scheduleKeyRequest;
}
if (request_key_frame)
RequestKeyFrame();
@ -82,13 +88,8 @@ void VideoReceiver::Process() {
// disabled when NACK is off.
if (_retransmissionTimer.TimeUntilProcess() == 0) {
_retransmissionTimer.Processed();
bool callback_registered = false;
uint16_t length;
{
rtc::CritScope cs(&process_crit_);
length = max_nack_list_size_;
callback_registered = _packetRequestCallback != nullptr;
}
bool callback_registered = _packetRequestCallback != nullptr;
uint16_t length = max_nack_list_size_;
if (callback_registered && length > 0) {
// Collect sequence numbers from the default receiver.
bool request_key_frame = false;
@ -98,7 +99,6 @@ void VideoReceiver::Process() {
ret = RequestKeyFrame();
}
if (ret == VCM_OK && !nackList.empty()) {
rtc::CritScope cs(&process_crit_);
if (_packetRequestCallback != nullptr) {
_packetRequestCallback->ResendPackets(&nackList[0], nackList.size());
}
@ -107,7 +107,18 @@ void VideoReceiver::Process() {
}
}
void VideoReceiver::ProcessThreadAttached(ProcessThread* process_thread) {
RTC_DCHECK_RUN_ON(&construction_thread_);
if (process_thread) {
is_attached_to_process_thread_ = true;
process_thread_ = process_thread;
} else {
is_attached_to_process_thread_ = false;
}
}
int64_t VideoReceiver::TimeUntilNextProcess() {
RTC_DCHECK_RUN_ON(&module_thread_);
int64_t timeUntilNextProcess = _receiveStatsTimer.TimeUntilProcess();
if (_receiver.NackMode() != kNoNack) {
// We need a Process call more often if we are relying on
@ -122,7 +133,7 @@ int64_t VideoReceiver::TimeUntilNextProcess() {
}
int32_t VideoReceiver::SetReceiveChannelParameters(int64_t rtt) {
rtc::CritScope cs(&receive_crit_);
RTC_DCHECK_RUN_ON(&module_thread_);
_receiver.UpdateRtt(rtt);
return 0;
}
@ -142,7 +153,6 @@ int32_t VideoReceiver::SetVideoProtection(VCMVideoProtection videoProtection,
}
case kProtectionNackFEC: {
rtc::CritScope cs(&receive_crit_);
RTC_DCHECK(enable);
_receiver.SetNackMode(kNack,
media_optimization::kLowRttNackMs,
@ -165,20 +175,22 @@ int32_t VideoReceiver::SetVideoProtection(VCMVideoProtection videoProtection,
// ready for rendering.
int32_t VideoReceiver::RegisterReceiveCallback(
VCMReceiveCallback* receiveCallback) {
RTC_DCHECK(construction_thread_.CalledOnValidThread());
// TODO(tommi): Callback may be null, but only after the decoder thread has
// been stopped. Use the signal we now get that tells us when the decoder
// thread isn't running, to DCHECK that the method is never called while it
// is. Once we're confident, we can remove the lock.
rtc::CritScope cs(&receive_crit_);
RTC_DCHECK_RUN_ON(&construction_thread_);
RTC_DCHECK(!IsDecoderThreadRunning());
// This value is set before the decoder thread starts and unset after
// the decoder thread has been stopped.
_decodedFrameCallback.SetUserReceiveCallback(receiveCallback);
return VCM_OK;
}
int32_t VideoReceiver::RegisterReceiveStatisticsCallback(
VCMReceiveStatisticsCallback* receiveStats) {
RTC_DCHECK(construction_thread_.CalledOnValidThread());
rtc::CritScope cs(&process_crit_);
RTC_DCHECK_RUN_ON(&construction_thread_);
RTC_DCHECK(!IsDecoderThreadRunning() && !is_attached_to_process_thread_);
// |_receiver| is used on both the decoder and module threads.
// However, since we make sure that we never do anything on the module thread
// when the decoder thread is not running, we don't need a lock for the
// |_receiver| or |_receiveStatsCallback| here.
_receiver.RegisterStatsCallback(receiveStats);
_receiveStatsCallback = receiveStats;
return VCM_OK;
@ -187,10 +199,8 @@ int32_t VideoReceiver::RegisterReceiveStatisticsCallback(
// Register an externally defined decoder object.
void VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder,
uint8_t payloadType) {
RTC_DCHECK(construction_thread_.CalledOnValidThread());
// TODO(tommi): This method must be called when the decoder thread is not
// running. Do we need a lock in that case?
rtc::CritScope cs(&receive_crit_);
RTC_DCHECK_RUN_ON(&construction_thread_);
RTC_DCHECK(!IsDecoderThreadRunning());
if (externalDecoder == nullptr) {
RTC_CHECK(_codecDataBase.DeregisterExternalDecoder(payloadType));
return;
@ -201,53 +211,87 @@ void VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder,
// Register a frame type request callback.
int32_t VideoReceiver::RegisterFrameTypeCallback(
VCMFrameTypeCallback* frameTypeCallback) {
rtc::CritScope cs(&process_crit_);
RTC_DCHECK_RUN_ON(&construction_thread_);
RTC_DCHECK(!IsDecoderThreadRunning() && !is_attached_to_process_thread_);
// This callback is used on the module thread, but since we don't get
// callbacks on the module thread while the decoder thread isn't running
// (and this function must not be called when the decoder is running),
// we don't need a lock here.
_frameTypeCallback = frameTypeCallback;
return VCM_OK;
}
int32_t VideoReceiver::RegisterPacketRequestCallback(
VCMPacketRequestCallback* callback) {
rtc::CritScope cs(&process_crit_);
RTC_DCHECK_RUN_ON(&construction_thread_);
RTC_DCHECK(!IsDecoderThreadRunning() && !is_attached_to_process_thread_);
// This callback is used on the module thread, but since we don't get
// callbacks on the module thread while the decoder thread isn't running
// (and this function must not be called when the decoder is running),
// we don't need a lock here.
_packetRequestCallback = callback;
return VCM_OK;
}
void VideoReceiver::TriggerDecoderShutdown() {
RTC_DCHECK(construction_thread_.CalledOnValidThread());
RTC_DCHECK_RUN_ON(&construction_thread_);
RTC_DCHECK(IsDecoderThreadRunning());
_receiver.TriggerDecoderShutdown();
}
void VideoReceiver::DecoderThreadStarting() {
RTC_DCHECK_RUN_ON(&construction_thread_);
RTC_DCHECK(!IsDecoderThreadRunning());
if (process_thread_ && !is_attached_to_process_thread_) {
process_thread_->RegisterModule(this, RTC_FROM_HERE);
}
#if RTC_DCHECK_IS_ON
decoder_thread_is_running_ = true;
#endif
}
void VideoReceiver::DecoderThreadStopped() {
RTC_DCHECK_RUN_ON(&construction_thread_);
RTC_DCHECK(IsDecoderThreadRunning());
if (process_thread_ && is_attached_to_process_thread_) {
process_thread_->DeRegisterModule(this);
}
#if RTC_DCHECK_IS_ON
decoder_thread_is_running_ = false;
decoder_thread_.DetachFromThread();
#endif
}
// Decode next frame, blocking.
// Should be called as often as possible to get the most out of the decoder.
int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) {
bool prefer_late_decoding = false;
{
// TODO(tommi): Chances are that this lock isn't required.
rtc::CritScope cs(&receive_crit_);
prefer_late_decoding = _codecDataBase.PrefersLateDecoding();
}
VCMEncodedFrame* frame =
_receiver.FrameForDecoding(maxWaitTimeMs, prefer_late_decoding);
RTC_DCHECK_RUN_ON(&decoder_thread_);
VCMEncodedFrame* frame = _receiver.FrameForDecoding(
maxWaitTimeMs, _codecDataBase.PrefersLateDecoding());
if (!frame)
return VCM_FRAME_NOT_READY;
bool drop_frame = false;
{
rtc::CritScope cs(&process_crit_);
if (drop_frames_until_keyframe_) {
// Still getting delta frames, schedule another keyframe request as if
// decode failed.
if (frame->FrameType() != kVideoFrameKey) {
drop_frame = true;
_scheduleKeyRequest = true;
_receiver.ReleaseFrame(frame);
return VCM_FRAME_NOT_READY;
} else {
drop_frames_until_keyframe_ = false;
}
drop_frames_until_keyframe_ = false;
}
}
if (drop_frame) {
_receiver.ReleaseFrame(frame);
return VCM_FRAME_NOT_READY;
}
if (pre_decode_image_callback_) {
EncodedImage encoded_image(frame->EncodedImage());
int qp = -1;
@ -258,7 +302,6 @@ int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) {
frame->CodecSpecific(), nullptr);
}
rtc::CritScope cs(&receive_crit_);
// If this frame was too late, we should adjust the delay accordingly
_timing->UpdateCurrentDelay(frame->RenderTimeMs(),
clock_->TimeInMilliseconds());
@ -278,7 +321,7 @@ int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) {
// TODO(philipel): Clean up among the Decode functions as we replace
// VCMEncodedFrame with FrameObject.
int32_t VideoReceiver::Decode(const webrtc::VCMEncodedFrame* frame) {
rtc::CritScope lock(&receive_crit_);
RTC_DCHECK_RUN_ON(&decoder_thread_);
if (pre_decode_image_callback_) {
EncodedImage encoded_image(frame->EncodedImage());
int qp = -1;
@ -291,19 +334,20 @@ int32_t VideoReceiver::Decode(const webrtc::VCMEncodedFrame* frame) {
return Decode(*frame);
}
void VideoReceiver::DecodingStopped() {
// No further calls to Decode() will be made after this point.
// TODO(tommi): Make use of this to clarify and check threading model.
}
int32_t VideoReceiver::RequestKeyFrame() {
RTC_DCHECK_RUN_ON(&module_thread_);
// Since we deregister from the module thread when the decoder thread isn't
// running, we should get no calls here if decoding isn't being done.
RTC_DCHECK(IsDecoderThreadRunning());
TRACE_EVENT0("webrtc", "RequestKeyFrame");
rtc::CritScope cs(&process_crit_);
if (_frameTypeCallback != nullptr) {
const int32_t ret = _frameTypeCallback->RequestKeyFrame();
if (ret < 0) {
return ret;
}
rtc::CritScope cs(&process_crit_);
_scheduleKeyRequest = false;
} else {
return VCM_MISSING_CALLBACK;
@ -313,6 +357,7 @@ int32_t VideoReceiver::RequestKeyFrame() {
// Must be called from inside the receive side critical section.
int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) {
RTC_DCHECK_RUN_ON(&decoder_thread_);
TRACE_EVENT0("webrtc", "VideoReceiver::Decode");
// Change decoder if payload type has changed
VCMGenericDecoder* decoder =
@ -324,31 +369,41 @@ int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) {
int32_t ret = decoder->Decode(frame, clock_->TimeInMilliseconds());
// Check for failed decoding, run frame type request callback if needed.
bool request_key_frame = false;
if (ret < 0) {
request_key_frame = true;
}
bool request_key_frame = (ret < 0);
if (!frame.Complete() || frame.MissingFrame()) {
request_key_frame = true;
ret = VCM_OK;
}
if (request_key_frame) {
rtc::CritScope cs(&process_crit_);
_scheduleKeyRequest = true;
if (!_scheduleKeyRequest) {
_scheduleKeyRequest = true;
// TODO(tommi): Consider if we could instead post a task to the module
// thread and call RequestKeyFrame directly. Here we call WakeUp so that
// TimeUntilNextProcess() gets called straight away.
process_thread_->WakeUp(this);
}
}
return ret;
}
#if defined(WEBRTC_ANDROID)
void VideoReceiver::PollDecodedFrames() {
RTC_DCHECK_RUN_ON(&decoder_thread_);
auto* current_decoder = _codecDataBase.GetCurrentDecoder();
if (current_decoder)
current_decoder->PollDecodedFrames();
}
#endif
// Register possible receive codecs, can be called multiple times
int32_t VideoReceiver::RegisterReceiveCodec(const VideoCodec* receiveCodec,
int32_t numberOfCores,
bool requireKeyFrame) {
RTC_DCHECK(construction_thread_.CalledOnValidThread());
// TODO(tommi): This method must only be called when the decoder thread
// is not running. Do we need a lock? If not, it looks like we might not need
// a lock at all for |_codecDataBase|.
rtc::CritScope cs(&receive_crit_);
RTC_DCHECK_RUN_ON(&construction_thread_);
RTC_DCHECK(!IsDecoderThreadRunning());
if (receiveCodec == nullptr) {
return VCM_PARAMETER_ERROR;
}
@ -363,6 +418,7 @@ int32_t VideoReceiver::RegisterReceiveCodec(const VideoCodec* receiveCodec,
int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload,
size_t payloadLength,
const WebRtcRTPHeader& rtpInfo) {
RTC_DCHECK_RUN_ON(&module_thread_);
if (rtpInfo.frameType == kVideoFrameKey) {
TRACE_EVENT1("webrtc", "VCM::PacketKeyFrame", "seqnum",
rtpInfo.header.sequenceNumber);
@ -394,6 +450,7 @@ int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload,
// to sync with audio. Not included in VideoCodingModule::Delay()
// Defaults to 0 ms.
int32_t VideoReceiver::SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) {
RTC_DCHECK_RUN_ON(&module_thread_);
_timing->set_min_playout_delay(minPlayoutDelayMs);
return VCM_OK;
}
@ -401,22 +458,24 @@ int32_t VideoReceiver::SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) {
// The estimated delay caused by rendering, defaults to
// kDefaultRenderDelayMs = 10 ms
int32_t VideoReceiver::SetRenderDelay(uint32_t timeMS) {
RTC_DCHECK_RUN_ON(&construction_thread_);
RTC_DCHECK(!IsDecoderThreadRunning());
_timing->set_render_delay(timeMS);
return VCM_OK;
}
// Current video delay
int32_t VideoReceiver::Delay() const {
RTC_DCHECK_RUN_ON(&module_thread_);
return _timing->TargetVideoDelay();
}
// Only used by VCMRobustnessTest.
int VideoReceiver::SetReceiverRobustnessMode(
VideoCodingModule::ReceiverRobustness robustnessMode,
VCMDecodeErrorMode decode_error_mode) {
RTC_DCHECK(construction_thread_.CalledOnValidThread());
// TODO(tommi): This method must only be called when the decoder thread
// is not running and we don't need to hold this lock.
rtc::CritScope cs(&receive_crit_);
RTC_DCHECK_RUN_ON(&construction_thread_);
RTC_DCHECK(!IsDecoderThreadRunning());
switch (robustnessMode) {
case VideoCodingModule::kNone:
_receiver.SetNackMode(kNoNack, -1, -1);
@ -434,24 +493,40 @@ int VideoReceiver::SetReceiverRobustnessMode(
}
void VideoReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) {
rtc::CritScope cs(&receive_crit_);
RTC_DCHECK_RUN_ON(&construction_thread_);
RTC_DCHECK(!IsDecoderThreadRunning());
_receiver.SetDecodeErrorMode(decode_error_mode);
}
void VideoReceiver::SetNackSettings(size_t max_nack_list_size,
int max_packet_age_to_nack,
int max_incomplete_time_ms) {
if (max_nack_list_size != 0) {
rtc::CritScope cs(&process_crit_);
RTC_DCHECK_RUN_ON(&construction_thread_);
RTC_DCHECK(!IsDecoderThreadRunning());
if (max_nack_list_size != 0)
max_nack_list_size_ = max_nack_list_size;
}
_receiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack,
max_incomplete_time_ms);
}
int VideoReceiver::SetMinReceiverDelay(int desired_delay_ms) {
RTC_DCHECK_RUN_ON(&construction_thread_);
RTC_DCHECK(!IsDecoderThreadRunning());
// TODO(tommi): Is the method only used by tests? Maybe could be offered
// via a test only subclass?
// Info from Stefan: If it is indeed only used by tests I think it's just that
// it hasn't been cleaned up when the calling code was cleaned up.
return _receiver.SetMinReceiverDelay(desired_delay_ms);
}
bool VideoReceiver::IsDecoderThreadRunning() {
#if RTC_DCHECK_IS_ON
return decoder_thread_is_running_;
#else
return true;
#endif
}
} // namespace vcm
} // namespace webrtc