zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update multiplex encoder to support having augmenting data attached to the video

Browse Source 
Multiplex encoder is now supporting attaching user-defined data to the video
frame. This data will be sent with the video frame and thus is guaranteed to
be synchronized. This is useful in cases where the data and video frame need
to by synchronized such as sending information about 3D objects or camera
tracking information with the video stream

Multiplex Encoder with data is implemented in a modular way. A new
VideoFrameBuffer type is created with the encoder. AugmentedVideoFrameBuffer
holds the video frame and the data. MultiplexVideoEncoder encodes both
the frame and data.

Change-Id: I23263f70d111f6f1783c070edec70bd11ebb9868
Bug: webrtc:9632
Reviewed-on: https://webrtc-review.googlesource.com/92642
Commit-Queue: Tarek Hefny <tarekh@google.com>
Reviewed-by: Niklas Enbom <niklas.enbom@webrtc.org>
Reviewed-by: Emircan Uysaler <emircan@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24297}
This commit is contained in:
Tarek Hefny
2018-08-15 10:19:32 -07:00
committed by Commit Bot
parent ed125fb2f3
commit 77c8e65b88
12 changed files with 402 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
modules/video_coding/codecs/multiplex/include/augmented_video_frame_buffer.h Normal file
View File

@ -0,0 +1,54 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_AUGMENTED_VIDEO_FRAME_BUFFER_H_
#define MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_AUGMENTED_VIDEO_FRAME_BUFFER_H_
#include <memory>
#include "api/video/video_frame_buffer.h"
namespace webrtc {
class AugmentedVideoFrameBuffer : public VideoFrameBuffer {
public:
AugmentedVideoFrameBuffer(
const rtc::scoped_refptr<VideoFrameBuffer>& video_frame_buffer,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_size);
// Retrieves the underlying VideoFrameBuffer without the augmented data
rtc::scoped_refptr<VideoFrameBuffer> GetVideoFrameBuffer() const;
// Gets a pointer to the augmenting data and moves ownership to the caller
uint8_t* GetAugmentingData() const;
// Get the size of the augmenting data
uint16_t GetAugmentingDataSize() const;
// Returns the type of the underlying VideoFrameBuffer
Type type() const final;
// Returns the width of the underlying VideoFrameBuffer
int width() const final;
// Returns the height of the underlying VideoFrameBuffer
int height() const final;
// Get the I140 Buffer from the underlying frame buffer
rtc::scoped_refptr<I420BufferInterface> ToI420() final;
private:
uint16_t augmenting_data_size_;
std::unique_ptr<uint8_t[]> augmenting_data_;
rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_;
};
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_AUGMENTED_VIDEO_FRAME_BUFFER_H_

12
modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h
View File

@ -26,7 +26,8 @@ class MultiplexDecoderAdapter : public VideoDecoder {
public:
// |factory| is not owned and expected to outlive this class' lifetime.
MultiplexDecoderAdapter(VideoDecoderFactory* factory,
const SdpVideoFormat& associated_format);
const SdpVideoFormat& associated_format,
bool supports_augmenting_data = false);
virtual ~MultiplexDecoderAdapter();
// Implements VideoDecoder
@ -52,12 +53,17 @@ class MultiplexDecoderAdapter : public VideoDecoder {
// Holds the decoded image output of a frame.
struct DecodedImageData;
// Holds the augmenting data of an image
struct AugmentingData;
void MergeAlphaImages(VideoFrame* decoded_image,
const absl::optional<int32_t>& decode_time_ms,
const absl::optional<uint8_t>& qp,
VideoFrame* multiplex_decoded_image,
const absl::optional<int32_t>& multiplex_decode_time_ms,
const absl::optional<uint8_t>& multiplex_qp);
const absl::optional<uint8_t>& multiplex_qp,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_length);
VideoDecoderFactory* const factory_;
const SdpVideoFormat associated_format_;
@ -67,6 +73,8 @@ class MultiplexDecoderAdapter : public VideoDecoder {
// Holds YUV or AXX decode output of a frame that is identified by timestamp.
std::map<uint32_t /* timestamp */, DecodedImageData> decoded_data_;
std::map<uint32_t /* timestamp */, AugmentingData> decoded_augmenting_data_;
const bool supports_augmenting_data_;
};
} // namespace webrtc

17
modules/video_coding/codecs/multiplex/include/multiplex_encoded_image_packer.h
View File

@ -11,6 +11,7 @@
#ifndef MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_MULTIPLEX_ENCODED_IMAGE_PACKER_H_
#define MODULES_VIDEO_CODING_CODECS_MULTIPLEX_INCLUDE_MULTIPLEX_ENCODED_IMAGE_PACKER_H_
#include <memory>
#include <vector>
#include "common_types.h" // NOLINT(build/include)
@ -33,9 +34,16 @@ struct MultiplexImageHeader {
// The location of the first MultiplexImageComponentHeader in the bitstream,
// in terms of byte from the beginning of the bitstream.
uint32_t first_component_header_offset;
// The location of the augmenting data in the bitstream, in terms of bytes
// from the beginning of the bitstream
uint32_t augmenting_data_offset;
// The size of the augmenting data in the bitstream it terms of byte
uint16_t augmenting_data_size;
};
const int kMultiplexImageHeaderSize =
sizeof(uint8_t) + sizeof(uint16_t) + sizeof(uint32_t);
sizeof(uint8_t) + 2 * sizeof(uint16_t) + 2 * sizeof(uint32_t);
// Struct describing the individual image component's content.
struct MultiplexImageComponentHeader {
@ -81,9 +89,14 @@ struct MultiplexImageComponent {
struct MultiplexImage {
uint16_t image_index;
uint8_t component_count;
uint16_t augmenting_data_size;
std::unique_ptr<uint8_t[]> augmenting_data;
std::vector<MultiplexImageComponent> image_components;
MultiplexImage(uint16_t picture_index, uint8_t component_count);
MultiplexImage(uint16_t picture_index,
uint8_t component_count,
std::unique_ptr<uint8_t[]> augmenting_data,
uint16_t augmenting_data_size);
};
// A utility class providing conversion between two representations of a

6
modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h
View File

@ -34,7 +34,8 @@ class MultiplexEncoderAdapter : public VideoEncoder {
public:
// |factory| is not owned and expected to outlive this class' lifetime.
MultiplexEncoderAdapter(VideoEncoderFactory* factory,
const SdpVideoFormat& associated_format);
const SdpVideoFormat& associated_format,
bool supports_augmenting_data = false);
virtual ~MultiplexEncoderAdapter();
// Implements VideoEncoder
@ -77,6 +78,9 @@ class MultiplexEncoderAdapter : public VideoEncoder {
EncodedImage combined_image_;
rtc::CriticalSection crit_;
const bool supports_augmented_data_;
int augmenting_data_size_ = 0;
};
} // namespace webrtc