zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix VideoToolbox backgrounding issues.

Browse Source 
When the iOS application is not in the foreground, the hardware encoder and
decoder become invalidated. There doesn't seem to be a way to query their state
so we don't know they're invalid until we get an error code after an
encode/decode request. To solve the issue, we just don't encode/decode when the
app is not active, and reinitialize the encoder/decoder when the app is active
again.

Also fixes a leak in the decoder.

BUG=webrtc:4081

Review URL: https://codereview.webrtc.org/1732953003

Cr-Commit-Position: refs/heads/master@{#11916}
This commit is contained in:
tkchin
2016-03-08 10:51:54 -08:00
committed by Commit bot
parent 3816bfd87b
commit 5ed5ed953d
11 changed files with 237 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
webrtc/modules/video_coding/codecs/h264/h264.gypi
View File

@ -62,6 +62,10 @@
{
'target_name': 'webrtc_h264_video_toolbox',
'type': 'static_library',
'includes': [ '../../../../build/objc_common.gypi' ],
'dependencies': [
'<(webrtc_root)/base/base.gyp:rtc_base_objc',
],
'link_settings': {
'xcode_settings': {
'OTHER_LDFLAGS': [

57
webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_decoder.cc
View File

@ -18,6 +18,9 @@
#include "libyuv/convert.h"
#include "webrtc/base/checks.h"
#include "webrtc/base/logging.h"
#if defined(WEBRTC_IOS)
#include "webrtc/base/objc/RTCUIApplication.h"
#endif
#include "webrtc/common_video/include/video_frame_buffer.h"
#include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h"
#include "webrtc/video_frame.h"
@ -128,6 +131,40 @@ int H264VideoToolboxDecoder::Decode(
int64_t render_time_ms) {
RTC_DCHECK(input_image._buffer);
#if defined(WEBRTC_IOS)
if (!RTCIsUIApplicationActive()) {
// Ignore all decode requests when app isn't active. In this state, the
// hardware decoder has been invalidated by the OS.
// Reset video format so that we won't process frames until the next
// keyframe.
SetVideoFormat(nullptr);
return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
}
#endif
CMVideoFormatDescriptionRef input_format = nullptr;
if (H264AnnexBBufferHasVideoFormatDescription(input_image._buffer,
input_image._length)) {
input_format = CreateVideoFormatDescription(input_image._buffer,
input_image._length);
if (input_format) {
// Check if the video format has changed, and reinitialize decoder if
// needed.
if (!CMFormatDescriptionEqual(input_format, video_format_)) {
SetVideoFormat(input_format);
ResetDecompressionSession();
}
CFRelease(input_format);
}
}
if (!video_format_) {
// We received a frame but we don't have format information so we can't
// decode it.
// This can happen after backgrounding. We need to wait for the next
// sps/pps before we can resume so we request a keyframe by returning an
// error.
LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
CMSampleBufferRef sample_buffer = nullptr;
if (!H264AnnexBBufferToCMSampleBuffer(input_image._buffer,
input_image._length, video_format_,
@ -135,13 +172,6 @@ int H264VideoToolboxDecoder::Decode(
return WEBRTC_VIDEO_CODEC_ERROR;
}
RTC_DCHECK(sample_buffer);
// Check if the video format has changed, and reinitialize decoder if needed.
CMVideoFormatDescriptionRef description =
CMSampleBufferGetFormatDescription(sample_buffer);
if (!CMFormatDescriptionEqual(description, video_format_)) {
SetVideoFormat(description);
ResetDecompressionSession();
}
VTDecodeFrameFlags decode_flags =
kVTDecodeFrame_EnableAsynchronousDecompression;
std::unique_ptr<internal::FrameDecodeParams> frame_decode_params;
@ -150,6 +180,18 @@ int H264VideoToolboxDecoder::Decode(
OSStatus status = VTDecompressionSessionDecodeFrame(
decompression_session_, sample_buffer, decode_flags,
frame_decode_params.release(), nullptr);
#if defined(WEBRTC_IOS)
// Re-initialize the decoder if we have an invalid session while the app is
// active and retry the decode request.
if (status == kVTInvalidSessionErr &&
ResetDecompressionSession() == WEBRTC_VIDEO_CODEC_OK) {
frame_decode_params.reset(
new internal::FrameDecodeParams(callback_, input_image._timeStamp));
status = VTDecompressionSessionDecodeFrame(
decompression_session_, sample_buffer, decode_flags,
frame_decode_params.release(), nullptr);
}
#endif
CFRelease(sample_buffer);
if (status != noErr) {
LOG(LS_ERROR) << "Failed to decode frame with code: " << status;
@ -244,6 +286,7 @@ void H264VideoToolboxDecoder::ConfigureDecompressionSession() {
void H264VideoToolboxDecoder::DestroyDecompressionSession() {
if (decompression_session_) {
VTDecompressionSessionInvalidate(decompression_session_);
CFRelease(decompression_session_);
decompression_session_ = nullptr;
}
}

32
webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc
View File

@ -20,6 +20,9 @@
#include "libyuv/convert_from.h"
#include "webrtc/base/checks.h"
#include "webrtc/base/logging.h"
#if defined(WEBRTC_IOS)
#include "webrtc/base/objc/RTCUIApplication.h"
#endif
#include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h"
#include "webrtc/system_wrappers/include/clock.h"
@ -238,10 +241,31 @@ int H264VideoToolboxEncoder::Encode(
if (!callback_ || !compression_session_) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
#if defined(WEBRTC_IOS)
if (!RTCIsUIApplicationActive()) {
// Ignore all encode requests when app isn't active. In this state, the
// hardware encoder has been invalidated by the OS.
return WEBRTC_VIDEO_CODEC_OK;
}
#endif
// Get a pixel buffer from the pool and copy frame data over.
CVPixelBufferPoolRef pixel_buffer_pool =
VTCompressionSessionGetPixelBufferPool(compression_session_);
#if defined(WEBRTC_IOS)
if (!pixel_buffer_pool) {
// Kind of a hack. On backgrounding, the compression session seems to get
// invalidated, which causes this pool call to fail when the application
// is foregrounded and frames are being sent for encoding again.
// Resetting the session when this happens fixes the issue.
ResetCompressionSession();
pixel_buffer_pool =
VTCompressionSessionGetPixelBufferPool(compression_session_);
}
#endif
if (!pixel_buffer_pool) {
LOG(LS_ERROR) << "Failed to get pixel buffer pool.";
return WEBRTC_VIDEO_CODEC_ERROR;
}
CVPixelBufferRef pixel_buffer = nullptr;
CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool,
&pixel_buffer);
@ -285,7 +309,7 @@ int H264VideoToolboxEncoder::Encode(
// Update the bitrate if needed.
SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
VTCompressionSessionEncodeFrame(
OSStatus status = VTCompressionSessionEncodeFrame(
compression_session_, pixel_buffer, presentation_time_stamp,
kCMTimeInvalid, frame_properties, encode_params.release(), nullptr);
if (frame_properties) {
@ -294,6 +318,10 @@ int H264VideoToolboxEncoder::Encode(
if (pixel_buffer) {
CVBufferRelease(pixel_buffer);
}
if (status != noErr) {
LOG(LS_ERROR) << "Failed to encode frame with code: " << status;
return WEBRTC_VIDEO_CODEC_ERROR;
}
return WEBRTC_VIDEO_CODEC_OK;
}

92
webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.cc
View File

@ -161,55 +161,34 @@ bool H264AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer,
CMSampleBufferRef* out_sample_buffer) {
RTC_DCHECK(annexb_buffer);
RTC_DCHECK(out_sample_buffer);
RTC_DCHECK(video_format);
*out_sample_buffer = nullptr;
// The buffer we receive via RTP has 00 00 00 01 start code artifically
// embedded by the RTP depacketizer. Extract NALU information.
// TODO(tkchin): handle potential case where sps and pps are delivered
// separately.
uint8_t first_nalu_type = annexb_buffer[4] & 0x1f;
bool is_first_nalu_type_sps = first_nalu_type == 0x7;
AnnexBBufferReader reader(annexb_buffer, annexb_buffer_size);
CMVideoFormatDescriptionRef description = nullptr;
OSStatus status = noErr;
if (is_first_nalu_type_sps) {
// Parse the SPS and PPS into a CMVideoFormatDescription.
const uint8_t* param_set_ptrs[2] = {};
size_t param_set_sizes[2] = {};
if (!reader.ReadNalu(&param_set_ptrs[0], &param_set_sizes[0])) {
if (H264AnnexBBufferHasVideoFormatDescription(annexb_buffer,
annexb_buffer_size)) {
// Advance past the SPS and PPS.
const uint8_t* data = nullptr;
size_t data_len = 0;
if (!reader.ReadNalu(&data, &data_len)) {
LOG(LS_ERROR) << "Failed to read SPS";
return false;
}
if (!reader.ReadNalu(&param_set_ptrs[1], &param_set_sizes[1])) {
if (!reader.ReadNalu(&data, &data_len)) {
LOG(LS_ERROR) << "Failed to read PPS";
return false;
}
status = CMVideoFormatDescriptionCreateFromH264ParameterSets(
kCFAllocatorDefault, 2, param_set_ptrs, param_set_sizes, 4,
&description);
if (status != noErr) {
LOG(LS_ERROR) << "Failed to create video format description.";
return false;
}
} else {
RTC_DCHECK(video_format);
description = video_format;
// We don't need to retain, but it makes logic easier since we are creating
// in the other block.
CFRetain(description);
}
// Allocate memory as a block buffer.
// TODO(tkchin): figure out how to use a pool.
CMBlockBufferRef block_buffer = nullptr;
status = CMBlockBufferCreateWithMemoryBlock(
OSStatus status = CMBlockBufferCreateWithMemoryBlock(
nullptr, nullptr, reader.BytesRemaining(), nullptr, nullptr, 0,
reader.BytesRemaining(), kCMBlockBufferAssureMemoryNowFlag,
&block_buffer);
if (status != kCMBlockBufferNoErr) {
LOG(LS_ERROR) << "Failed to create block buffer.";
CFRelease(description);
return false;
}
@ -221,7 +200,6 @@ bool H264AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer,
if (status != noErr) {
LOG(LS_ERROR) << "Failed to flatten non-contiguous block buffer: "
<< status;
CFRelease(description);
CFRelease(block_buffer);
return false;
}
@ -237,7 +215,6 @@ bool H264AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer,
&block_buffer_size, &data_ptr);
if (status != kCMBlockBufferNoErr) {
LOG(LS_ERROR) << "Failed to get block buffer data pointer.";
CFRelease(description);
CFRelease(contiguous_buffer);
return false;
}
@ -256,19 +233,62 @@ bool H264AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer,
// Create sample buffer.
status = CMSampleBufferCreate(nullptr, contiguous_buffer, true, nullptr,
nullptr, description, 1, 0, nullptr, 0, nullptr,
out_sample_buffer);
nullptr, video_format, 1, 0, nullptr, 0,
nullptr, out_sample_buffer);
if (status != noErr) {
LOG(LS_ERROR) << "Failed to create sample buffer.";
CFRelease(description);
CFRelease(contiguous_buffer);
return false;
}
CFRelease(description);
CFRelease(contiguous_buffer);
return true;
}
bool H264AnnexBBufferHasVideoFormatDescription(const uint8_t* annexb_buffer,
size_t annexb_buffer_size) {
RTC_DCHECK(annexb_buffer);
RTC_DCHECK_GT(annexb_buffer_size, 4u);
// The buffer we receive via RTP has 00 00 00 01 start code artifically
// embedded by the RTP depacketizer. Extract NALU information.
// TODO(tkchin): handle potential case where sps and pps are delivered
// separately.
uint8_t first_nalu_type = annexb_buffer[4] & 0x1f;
bool is_first_nalu_type_sps = first_nalu_type == 0x7;
return is_first_nalu_type_sps;
}
CMVideoFormatDescriptionRef CreateVideoFormatDescription(
const uint8_t* annexb_buffer,
size_t annexb_buffer_size) {
if (!H264AnnexBBufferHasVideoFormatDescription(annexb_buffer,
annexb_buffer_size)) {
return nullptr;
}
AnnexBBufferReader reader(annexb_buffer, annexb_buffer_size);
CMVideoFormatDescriptionRef description = nullptr;
OSStatus status = noErr;
// Parse the SPS and PPS into a CMVideoFormatDescription.
const uint8_t* param_set_ptrs[2] = {};
size_t param_set_sizes[2] = {};
if (!reader.ReadNalu(&param_set_ptrs[0], &param_set_sizes[0])) {
LOG(LS_ERROR) << "Failed to read SPS";
return nullptr;
}
if (!reader.ReadNalu(&param_set_ptrs[1], &param_set_sizes[1])) {
LOG(LS_ERROR) << "Failed to read PPS";
return nullptr;
}
status = CMVideoFormatDescriptionCreateFromH264ParameterSets(
kCFAllocatorDefault, 2, param_set_ptrs, param_set_sizes, 4,
&description);
if (status != noErr) {
LOG(LS_ERROR) << "Failed to create video format description.";
return nullptr;
}
return description;
}
AnnexBBufferReader::AnnexBBufferReader(const uint8_t* annexb_buffer,
size_t length)
: start_(annexb_buffer), offset_(0), next_offset_(0), length_(length) {

12
webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h
View File

@ -44,6 +44,18 @@ bool H264AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer,
CMVideoFormatDescriptionRef video_format,
CMSampleBufferRef* out_sample_buffer);
// Returns true if the type of the first NALU in the supplied Annex B buffer is
// the SPS type.
bool H264AnnexBBufferHasVideoFormatDescription(const uint8_t* annexb_buffer,
size_t annexb_buffer_size);
// Returns a video format description created from the sps/pps information in
// the Annex B buffer. If there is no such information, nullptr is returned.
// The caller is responsible for releasing the description.
CMVideoFormatDescriptionRef CreateVideoFormatDescription(
const uint8_t* annexb_buffer,
size_t annexb_buffer_size);
// Helper class for reading NALUs from an RTP Annex B buffer.
class AnnexBBufferReader final {
public:

35
webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu_unittest.cc
View File

@ -16,11 +16,44 @@
#include "webrtc/base/arraysize.h"
#include "webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_nalu.h"
#if defined(WEBRTC_VIDEO_TOOLBOX_SUPPORTED)
namespace webrtc {
static const uint8_t NALU_TEST_DATA_0[] = {0xAA, 0xBB, 0xCC};
static const uint8_t NALU_TEST_DATA_1[] = {0xDE, 0xAD, 0xBE, 0xEF};
TEST(H264VideoToolboxNaluTest, TestHasVideoFormatDescription) {
const uint8_t sps_buffer[] = {0x00, 0x00, 0x00, 0x01, 0x27};
EXPECT_TRUE(H264AnnexBBufferHasVideoFormatDescription(sps_buffer,
arraysize(sps_buffer)));
const uint8_t other_buffer[] = {0x00, 0x00, 0x00, 0x01, 0x28};
EXPECT_FALSE(H264AnnexBBufferHasVideoFormatDescription(
other_buffer, arraysize(other_buffer)));
}
TEST(H264VideoToolboxNaluTest, TestCreateVideoFormatDescription) {
const uint8_t sps_pps_buffer[] = {
// SPS nalu.
0x00, 0x00, 0x00, 0x01,
0x27, 0x42, 0x00, 0x1E, 0xAB, 0x40, 0xF0, 0x28, 0xD3, 0x70, 0x20, 0x20,
0x20, 0x20,
// PPS nalu.
0x00, 0x00, 0x00, 0x01,
0x28, 0xCE, 0x3C, 0x30
};
CMVideoFormatDescriptionRef description =
CreateVideoFormatDescription(sps_pps_buffer, arraysize(sps_pps_buffer));
EXPECT_TRUE(description);
if (description) {
CFRelease(description);
description = nullptr;
}
const uint8_t other_buffer[] = {0x00, 0x00, 0x00, 0x01, 0x28};
EXPECT_FALSE(CreateVideoFormatDescription(other_buffer,
arraysize(other_buffer)));
}
TEST(AnnexBBufferReaderTest, TestReadEmptyInput) {
const uint8_t annex_b_test_data[] = {0x00};
AnnexBBufferReader reader(annex_b_test_data, 0);
@ -151,3 +184,5 @@ TEST(AvccBufferWriterTest, TestOverflow) {
}
} // namespace webrtc
#endif // WEBRTC_VIDEO_TOOLBOX_SUPPORTED