zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update multiplex encoder to support having augmenting data attached to the video

Browse Source 
Multiplex encoder is now supporting attaching user-defined data to the video
frame. This data will be sent with the video frame and thus is guaranteed to
be synchronized. This is useful in cases where the data and video frame need
to by synchronized such as sending information about 3D objects or camera
tracking information with the video stream

Multiplex Encoder with data is implemented in a modular way. A new
VideoFrameBuffer type is created with the encoder. AugmentedVideoFrameBuffer
holds the video frame and the data. MultiplexVideoEncoder encodes both
the frame and data.

Change-Id: I23263f70d111f6f1783c070edec70bd11ebb9868
Bug: webrtc:9632
Reviewed-on: https://webrtc-review.googlesource.com/92642
Commit-Queue: Tarek Hefny <tarekh@google.com>
Reviewed-by: Niklas Enbom <niklas.enbom@webrtc.org>
Reviewed-by: Emircan Uysaler <emircan@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24297}
This commit is contained in:
Tarek Hefny
2018-08-15 10:19:32 -07:00
committed by Commit Bot
parent ed125fb2f3
commit 77c8e65b88
12 changed files with 402 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
modules/video_coding/codecs/multiplex/multiplex_decoder_adapter.cc
View File

@ -15,6 +15,7 @@
#include "common_video/include/video_frame.h"
#include "common_video/include/video_frame_buffer.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "modules/video_coding/codecs/multiplex/include/augmented_video_frame_buffer.h"
#include "rtc_base/keep_ref_until_done.h"
#include "rtc_base/logging.h"
@ -79,10 +80,23 @@ struct MultiplexDecoderAdapter::DecodedImageData {
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(DecodedImageData);
};
struct MultiplexDecoderAdapter::AugmentingData {
AugmentingData(std::unique_ptr<uint8_t[]> augmenting_data, uint16_t data_size)
: data_(std::move(augmenting_data)), size_(data_size) {}
std::unique_ptr<uint8_t[]> data_;
const uint16_t size_;
private:
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(AugmentingData);
};
MultiplexDecoderAdapter::MultiplexDecoderAdapter(
VideoDecoderFactory* factory,
const SdpVideoFormat& associated_format)
: factory_(factory), associated_format_(associated_format) {}
const SdpVideoFormat& associated_format,
bool supports_augmenting_data)
: factory_(factory),
associated_format_(associated_format),
supports_augmenting_data_(supports_augmenting_data) {}
MultiplexDecoderAdapter::~MultiplexDecoderAdapter() {
Release();
@ -113,8 +127,16 @@ int32_t MultiplexDecoderAdapter::Decode(
bool missing_frames,
const CodecSpecificInfo* codec_specific_info,
int64_t render_time_ms) {
const MultiplexImage& image =
MultiplexEncodedImagePacker::Unpack(input_image);
MultiplexImage image = MultiplexEncodedImagePacker::Unpack(input_image);
if (supports_augmenting_data_) {
RTC_DCHECK(decoded_augmenting_data_.find(input_image._timeStamp) ==
decoded_augmenting_data_.end());
decoded_augmenting_data_.emplace(
std::piecewise_construct, std::forward_as_tuple(input_image._timeStamp),
std::forward_as_tuple(std::move(image.augmenting_data),
image.augmenting_data_size));
}
if (image.component_count == 1) {
RTC_DCHECK(decoded_data_.find(input_image._timeStamp) ==
@ -157,21 +179,36 @@ void MultiplexDecoderAdapter::Decoded(AlphaCodecStream stream_idx,
absl::optional<uint8_t> qp) {
const auto& other_decoded_data_it =
decoded_data_.find(decoded_image->timestamp());
const auto& augmenting_data_it =
decoded_augmenting_data_.find(decoded_image->timestamp());
if (other_decoded_data_it != decoded_data_.end()) {
uint16_t augmenting_data_size =
augmenting_data_it == decoded_augmenting_data_.end()
? 0
: augmenting_data_it->second.size_;
std::unique_ptr<uint8_t[]> augmenting_data =
augmenting_data_size == 0 ? NULL
: std::move(augmenting_data_it->second.data_);
auto& other_image_data = other_decoded_data_it->second;
if (stream_idx == kYUVStream) {
RTC_DCHECK_EQ(kAXXStream, other_image_data.stream_idx_);
MergeAlphaImages(decoded_image, decode_time_ms, qp,
&other_image_data.decoded_image_,
other_image_data.decode_time_ms_, other_image_data.qp_);
other_image_data.decode_time_ms_, other_image_data.qp_,
std::move(augmenting_data), augmenting_data_size);
} else {
RTC_DCHECK_EQ(kYUVStream, other_image_data.stream_idx_);
RTC_DCHECK_EQ(kAXXStream, stream_idx);
MergeAlphaImages(&other_image_data.decoded_image_,
other_image_data.decode_time_ms_, other_image_data.qp_,
decoded_image, decode_time_ms, qp);
decoded_image, decode_time_ms, qp,
std::move(augmenting_data), augmenting_data_size);
}
decoded_data_.erase(decoded_data_.begin(), other_decoded_data_it);
if (supports_augmenting_data_) {
decoded_augmenting_data_.erase(decoded_augmenting_data_.begin(),
augmenting_data_it);
}
return;
}
RTC_DCHECK(decoded_data_.find(decoded_image->timestamp()) ==
@ -188,24 +225,31 @@ void MultiplexDecoderAdapter::MergeAlphaImages(
const absl::optional<uint8_t>& qp,
VideoFrame* alpha_decoded_image,
const absl::optional<int32_t>& alpha_decode_time_ms,
const absl::optional<uint8_t>& alpha_qp) {
const absl::optional<uint8_t>& alpha_qp,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_length) {
rtc::scoped_refptr<VideoFrameBuffer> merged_buffer;
if (!alpha_decoded_image->timestamp()) {
decoded_complete_callback_->Decoded(*decoded_image, decode_time_ms, qp);
return;
merged_buffer = decoded_image->video_frame_buffer();
} else {
rtc::scoped_refptr<webrtc::I420BufferInterface> yuv_buffer =
decoded_image->video_frame_buffer()->ToI420();
rtc::scoped_refptr<webrtc::I420BufferInterface> alpha_buffer =
alpha_decoded_image->video_frame_buffer()->ToI420();
RTC_DCHECK_EQ(yuv_buffer->width(), alpha_buffer->width());
RTC_DCHECK_EQ(yuv_buffer->height(), alpha_buffer->height());
merged_buffer = WrapI420ABuffer(
yuv_buffer->width(), yuv_buffer->height(), yuv_buffer->DataY(),
yuv_buffer->StrideY(), yuv_buffer->DataU(), yuv_buffer->StrideU(),
yuv_buffer->DataV(), yuv_buffer->StrideV(), alpha_buffer->DataY(),
alpha_buffer->StrideY(),
rtc::Bind(&KeepBufferRefs, yuv_buffer, alpha_buffer));
}
if (supports_augmenting_data_) {
merged_buffer = rtc::scoped_refptr<webrtc::AugmentedVideoFrameBuffer>(
new rtc::RefCountedObject<AugmentedVideoFrameBuffer>(
merged_buffer, std::move(augmenting_data), augmenting_data_length));
}
rtc::scoped_refptr<webrtc::I420BufferInterface> yuv_buffer =
decoded_image->video_frame_buffer()->ToI420();
rtc::scoped_refptr<webrtc::I420BufferInterface> alpha_buffer =
alpha_decoded_image->video_frame_buffer()->ToI420();
RTC_DCHECK_EQ(yuv_buffer->width(), alpha_buffer->width());
RTC_DCHECK_EQ(yuv_buffer->height(), alpha_buffer->height());
rtc::scoped_refptr<I420ABufferInterface> merged_buffer = WrapI420ABuffer(
yuv_buffer->width(), yuv_buffer->height(), yuv_buffer->DataY(),
yuv_buffer->StrideY(), yuv_buffer->DataU(), yuv_buffer->StrideU(),
yuv_buffer->DataV(), yuv_buffer->StrideV(), alpha_buffer->DataY(),
alpha_buffer->StrideY(),
rtc::Bind(&KeepBufferRefs, yuv_buffer, alpha_buffer));
VideoFrame merged_image(merged_buffer, decoded_image->timestamp(),
0 /* render_time_ms */, decoded_image->rotation());