zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update multiplex encoder to support having augmenting data attached to the video

Browse Source 
Multiplex encoder is now supporting attaching user-defined data to the video
frame. This data will be sent with the video frame and thus is guaranteed to
be synchronized. This is useful in cases where the data and video frame need
to by synchronized such as sending information about 3D objects or camera
tracking information with the video stream

Multiplex Encoder with data is implemented in a modular way. A new
VideoFrameBuffer type is created with the encoder. AugmentedVideoFrameBuffer
holds the video frame and the data. MultiplexVideoEncoder encodes both
the frame and data.

Change-Id: I23263f70d111f6f1783c070edec70bd11ebb9868
Bug: webrtc:9632
Reviewed-on: https://webrtc-review.googlesource.com/92642
Commit-Queue: Tarek Hefny <tarekh@google.com>
Reviewed-by: Niklas Enbom <niklas.enbom@webrtc.org>
Reviewed-by: Emircan Uysaler <emircan@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24297}
This commit is contained in:
Tarek Hefny
2018-08-15 10:19:32 -07:00
committed by Commit Bot
parent ed125fb2f3
commit 77c8e65b88
12 changed files with 402 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
modules/video_coding/codecs/multiplex/test/multiplex_adapter_unittest.cc
View File

@ -15,6 +15,7 @@
#include "common_video/include/video_frame_buffer.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "media/base/mediaconstants.h"
#include "modules/video_coding/codecs/multiplex/include/augmented_video_frame_buffer.h"
#include "modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h"
#include "modules/video_coding/codecs/multiplex/include/multiplex_encoded_image_packer.h"
#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
@ -32,21 +33,26 @@ constexpr const char* kMultiplexAssociatedCodecName = cricket::kVp9CodecName;
const VideoCodecType kMultiplexAssociatedCodecType =
PayloadStringToCodecType(kMultiplexAssociatedCodecName);
class TestMultiplexAdapter : public VideoCodecUnitTest {
class TestMultiplexAdapter
: public VideoCodecUnitTest,
public testing::WithParamInterface<bool /* supports_augmenting_data */> {
public:
TestMultiplexAdapter()
: decoder_factory_(new webrtc::MockVideoDecoderFactory),
encoder_factory_(new webrtc::MockVideoEncoderFactory) {}
encoder_factory_(new webrtc::MockVideoEncoderFactory),
supports_augmenting_data_(GetParam()) {}
protected:
std::unique_ptr<VideoDecoder> CreateDecoder() override {
return absl::make_unique<MultiplexDecoderAdapter>(
decoder_factory_.get(), SdpVideoFormat(kMultiplexAssociatedCodecName));
decoder_factory_.get(), SdpVideoFormat(kMultiplexAssociatedCodecName),
supports_augmenting_data_);
}
std::unique_ptr<VideoEncoder> CreateEncoder() override {
return absl::make_unique<MultiplexEncoderAdapter>(
encoder_factory_.get(), SdpVideoFormat(kMultiplexAssociatedCodecName));
encoder_factory_.get(), SdpVideoFormat(kMultiplexAssociatedCodecName),
supports_augmenting_data_);
}
void ModifyCodecSettings(VideoCodec* codec_settings) override {
@ -56,6 +62,23 @@ class TestMultiplexAdapter : public VideoCodecUnitTest {
codec_settings->codecType = webrtc::kVideoCodecMultiplex;
}
std::unique_ptr<VideoFrame> CreateDataAugmentedInputFrame(
VideoFrame* video_frame) {
rtc::scoped_refptr<VideoFrameBuffer> video_buffer =
video_frame->video_frame_buffer();
std::unique_ptr<uint8_t[]> data =
std::unique_ptr<uint8_t[]>(new uint8_t[16]);
for (int i = 0; i < 16; i++) {
data[i] = i;
}
rtc::scoped_refptr<AugmentedVideoFrameBuffer> augmented_video_frame_buffer =
new rtc::RefCountedObject<AugmentedVideoFrameBuffer>(
video_buffer, std::move(data), 16);
return absl::WrapUnique<VideoFrame>(
new VideoFrame(augmented_video_frame_buffer, video_frame->timestamp(),
video_frame->render_time_ms(), video_frame->rotation()));
}
std::unique_ptr<VideoFrame> CreateI420AInputFrame() {
VideoFrame* input_frame = NextInputFrame();
rtc::scoped_refptr<webrtc::I420BufferInterface> yuv_buffer =
@ -70,14 +93,50 @@ class TestMultiplexAdapter : public VideoCodecUnitTest {
345 /* render_time_ms */, kVideoRotation_0));
}
std::unique_ptr<VideoFrame> ExtractAXXFrame(const VideoFrame& yuva_frame) {
const I420ABufferInterface* yuva_buffer =
yuva_frame.video_frame_buffer()->GetI420A();
std::unique_ptr<VideoFrame> CreateInputFrame(bool contains_alpha) {
std::unique_ptr<VideoFrame> video_frame;
if (contains_alpha) {
video_frame = CreateI420AInputFrame();
} else {
VideoFrame* next_frame = NextInputFrame();
video_frame = absl::WrapUnique<VideoFrame>(new VideoFrame(
next_frame->video_frame_buffer(), next_frame->timestamp(),
next_frame->render_time_ms(), next_frame->rotation()));
}
if (supports_augmenting_data_) {
video_frame = CreateDataAugmentedInputFrame(video_frame.get());
}
return video_frame;
}
void CheckData(rtc::scoped_refptr<VideoFrameBuffer> video_frame_buffer) {
if (!supports_augmenting_data_) {
return;
}
AugmentedVideoFrameBuffer* augmented_buffer =
static_cast<AugmentedVideoFrameBuffer*>(video_frame_buffer.get());
EXPECT_EQ(augmented_buffer->GetAugmentingDataSize(), 16);
uint8_t* data = augmented_buffer->GetAugmentingData();
for (int i = 0; i < 16; i++) {
EXPECT_EQ(data[i], i);
}
}
std::unique_ptr<VideoFrame> ExtractAXXFrame(const VideoFrame& video_frame) {
rtc::scoped_refptr<VideoFrameBuffer> video_frame_buffer =
video_frame.video_frame_buffer();
if (supports_augmenting_data_) {
AugmentedVideoFrameBuffer* augmentedBuffer =
static_cast<AugmentedVideoFrameBuffer*>(video_frame_buffer.get());
video_frame_buffer = augmentedBuffer->GetVideoFrameBuffer();
}
const I420ABufferInterface* yuva_buffer = video_frame_buffer->GetI420A();
rtc::scoped_refptr<I420BufferInterface> axx_buffer = WrapI420Buffer(
yuva_buffer->width(), yuva_buffer->height(), yuva_buffer->DataA(),
yuva_buffer->StrideA(), yuva_buffer->DataU(), yuva_buffer->StrideU(),
yuva_buffer->DataV(), yuva_buffer->StrideV(),
rtc::KeepRefUntilDone(yuva_frame.video_frame_buffer()));
rtc::KeepRefUntilDone(video_frame_buffer));
return absl::WrapUnique<VideoFrame>(
new VideoFrame(axx_buffer, 123 /* RTP timestamp */,
345 /* render_time_ms */, kVideoRotation_0));
@ -106,21 +165,22 @@ class TestMultiplexAdapter : public VideoCodecUnitTest {
const std::unique_ptr<webrtc::MockVideoDecoderFactory> decoder_factory_;
const std::unique_ptr<webrtc::MockVideoEncoderFactory> encoder_factory_;
const bool supports_augmenting_data_;
};
// TODO(emircan): Currently VideoCodecUnitTest tests do a complete setup
// step that goes beyond constructing |decoder_|. Simplify these tests to do
// less.
TEST_F(TestMultiplexAdapter, ConstructAndDestructDecoder) {
TEST_P(TestMultiplexAdapter, ConstructAndDestructDecoder) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder_->Release());
}
TEST_F(TestMultiplexAdapter, ConstructAndDestructEncoder) {
TEST_P(TestMultiplexAdapter, ConstructAndDestructEncoder) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
}
TEST_F(TestMultiplexAdapter, EncodeDecodeI420Frame) {
VideoFrame* input_frame = NextInputFrame();
TEST_P(TestMultiplexAdapter, EncodeDecodeI420Frame) {
std::unique_ptr<VideoFrame> input_frame = CreateInputFrame(false);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->Encode(*input_frame, nullptr, nullptr));
EncodedImage encoded_frame;
@ -134,11 +194,12 @@ TEST_F(TestMultiplexAdapter, EncodeDecodeI420Frame) {
absl::optional<uint8_t> decoded_qp;
ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
ASSERT_TRUE(decoded_frame);
EXPECT_GT(I420PSNR(input_frame, decoded_frame.get()), 36);
EXPECT_GT(I420PSNR(input_frame.get(), decoded_frame.get()), 36);
CheckData(decoded_frame->video_frame_buffer());
}
TEST_F(TestMultiplexAdapter, EncodeDecodeI420AFrame) {
std::unique_ptr<VideoFrame> yuva_frame = CreateI420AInputFrame();
TEST_P(TestMultiplexAdapter, EncodeDecodeI420AFrame) {
std::unique_ptr<VideoFrame> yuva_frame = CreateInputFrame(true);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->Encode(*yuva_frame, nullptr, nullptr));
EncodedImage encoded_frame;
@ -159,10 +220,12 @@ TEST_F(TestMultiplexAdapter, EncodeDecodeI420AFrame) {
std::unique_ptr<VideoFrame> output_axx_frame =
ExtractAXXFrame(*decoded_frame);
EXPECT_GT(I420PSNR(input_axx_frame.get(), output_axx_frame.get()), 47);
CheckData(decoded_frame->video_frame_buffer());
}
TEST_F(TestMultiplexAdapter, CheckSingleFrameEncodedBitstream) {
VideoFrame* input_frame = NextInputFrame();
TEST_P(TestMultiplexAdapter, CheckSingleFrameEncodedBitstream) {
std::unique_ptr<VideoFrame> input_frame = CreateInputFrame(false);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->Encode(*input_frame, nullptr, nullptr));
EncodedImage encoded_frame;
@ -181,8 +244,8 @@ TEST_F(TestMultiplexAdapter, CheckSingleFrameEncodedBitstream) {
EXPECT_EQ(kVideoFrameKey, component.encoded_image._frameType);
}
TEST_F(TestMultiplexAdapter, CheckDoubleFramesEncodedBitstream) {
std::unique_ptr<VideoFrame> yuva_frame = CreateI420AInputFrame();
TEST_P(TestMultiplexAdapter, CheckDoubleFramesEncodedBitstream) {
std::unique_ptr<VideoFrame> yuva_frame = CreateInputFrame(true);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->Encode(*yuva_frame, nullptr, nullptr));
EncodedImage encoded_frame;
@ -206,8 +269,8 @@ TEST_F(TestMultiplexAdapter, CheckDoubleFramesEncodedBitstream) {
}
}
TEST_F(TestMultiplexAdapter, ImageIndexIncreases) {
std::unique_ptr<VideoFrame> yuva_frame = CreateI420AInputFrame();
TEST_P(TestMultiplexAdapter, ImageIndexIncreases) {
std::unique_ptr<VideoFrame> yuva_frame = CreateInputFrame(true);
const size_t expected_num_encoded_frames = 3;
for (size_t i = 0; i < expected_num_encoded_frames; ++i) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
@ -222,4 +285,8 @@ TEST_F(TestMultiplexAdapter, ImageIndexIncreases) {
}
}
INSTANTIATE_TEST_CASE_P(TestMultiplexAdapter,
TestMultiplexAdapter,
::testing::Bool());
} // namespace webrtc