zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add/rewrite H264 VUI video signal type description.

Browse Source 
The rewriter updates video signal parameters in VUI such that they
match to given webrtc::ColorSpace.

Bug: webrtc:10723
Change-Id: I8d0593e3cb727bfee7eb00e3f9ff0b41b93b78bf
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/140881
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#28306}
This commit is contained in:
Sergey Silkin
2019-06-12 14:27:34 +02:00
committed by Commit Bot
parent 449888ef99
commit 0c0c9693b6
5 changed files with 427 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

196
common_video/h264/sps_vui_rewriter.cc
View File

@ -15,6 +15,7 @@
#include <cstdint>
#include <vector>
#include "api/video/color_space.h"
#include "common_video/h264/h264_common.h"
#include "common_video/h264/sps_parser.h"
#include "rtc_base/bit_buffer.h"
@ -73,14 +74,22 @@ enum SpsValidEvent {
bool CopyAndRewriteVui(const SpsParser::SpsState& sps,
rtc::BitBuffer* source,
rtc::BitBufferWriter* destination,
const webrtc::ColorSpace* color_space,
SpsVuiRewriter::ParseResult* out_vui_rewritten);
bool CopyHrdParameters(rtc::BitBuffer* source,
rtc::BitBufferWriter* destination);
bool AddBitstreamRestriction(rtc::BitBufferWriter* destination,
uint32_t max_num_ref_frames);
bool IsDefaultColorSpace(const ColorSpace& color_space);
bool AddVideoSignalTypeInfo(rtc::BitBufferWriter* destination,
const ColorSpace& color_space);
bool CopyOrRewriteVideoSignalTypeInfo(
rtc::BitBuffer* source,
rtc::BitBufferWriter* destination,
const ColorSpace* color_space,
SpsVuiRewriter::ParseResult* out_vui_rewritten);
bool CopyRemainingBits(rtc::BitBuffer* source,
rtc::BitBufferWriter* destination);
} // namespace
void SpsVuiRewriter::UpdateStats(ParseResult result, Direction direction) {
@ -116,6 +125,7 @@ SpsVuiRewriter::ParseResult SpsVuiRewriter::ParseAndRewriteSps(
const uint8_t* buffer,
size_t length,
absl::optional<SpsParser::SpsState>* sps,
const webrtc::ColorSpace* color_space,
rtc::Buffer* destination) {
// Create temporary RBSP decoded buffer of the payload (exlcuding the
// leading nalu type header byte (the SpsParser uses only the payload).
@ -151,7 +161,7 @@ SpsVuiRewriter::ParseResult SpsVuiRewriter::ParseAndRewriteSps(
sps_writer.Seek(byte_offset, bit_offset);
ParseResult vui_updated;
if (!CopyAndRewriteVui(*sps_state, &source_buffer, &sps_writer,
if (!CopyAndRewriteVui(*sps_state, &source_buffer, &sps_writer, color_space,
&vui_updated)) {
RTC_LOG(LS_ERROR) << "Failed to parse/copy SPS VUI.";
return ParseResult::kFailure;
@ -190,9 +200,11 @@ SpsVuiRewriter::ParseResult SpsVuiRewriter::ParseAndRewriteSps(
const uint8_t* buffer,
size_t length,
absl::optional<SpsParser::SpsState>* sps,
const webrtc::ColorSpace* color_space,
rtc::Buffer* destination,
Direction direction) {
ParseResult result = ParseAndRewriteSps(buffer, length, sps, destination);
ParseResult result =
ParseAndRewriteSps(buffer, length, sps, color_space, destination);
UpdateStats(result, direction);
return result;
}
@ -202,6 +214,7 @@ void SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps(
size_t num_nalus,
const size_t* nalu_offsets,
const size_t* nalu_lengths,
const webrtc::ColorSpace* color_space,
rtc::CopyOnWriteBuffer* output_buffer,
size_t* output_nalu_offsets,
size_t* output_nalu_lengths) {
@ -244,7 +257,7 @@ void SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps(
ParseResult result = ParseAndRewriteSps(
nalu_ptr + H264::kNaluTypeSize, nalu_length - H264::kNaluTypeSize,
&sps, &output_nalu, Direction::kOutgoing);
&sps, color_space, &output_nalu, Direction::kOutgoing);
if (result == ParseResult::kVuiRewritten) {
updated_sps = true;
output_nalu_offsets[i] = output_buffer->size();
@ -265,14 +278,16 @@ void SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps(
}
namespace {
bool CopyAndRewriteVui(const SpsParser::SpsState& sps,
rtc::BitBuffer* source,
rtc::BitBufferWriter* destination,
const webrtc::ColorSpace* color_space,
SpsVuiRewriter::ParseResult* out_vui_rewritten) {
uint32_t golomb_tmp;
uint32_t bits_tmp;
*out_vui_rewritten = SpsVuiRewriter::ParseResult::kVuiOk;
//
// vui_parameters_present_flag: u(1)
//
@ -283,12 +298,27 @@ bool CopyAndRewriteVui(const SpsParser::SpsState& sps,
// (2) rewrite frame reordering values so no reordering is allowed.
if (!sps.vui_params_present) {
// Write a simple VUI with the parameters we want and 0 for all other flags.
// There are 8 flags to be off before the bitstream restriction flag.
RETURN_FALSE_ON_FAIL(destination->WriteBits(0, 8));
// aspect_ratio_info_present_flag, overscan_info_present_flag. Both u(1).
RETURN_FALSE_ON_FAIL(destination->WriteBits(0, 2));
uint32_t video_signal_type_present_flag =
(color_space && !IsDefaultColorSpace(*color_space)) ? 1 : 0;
RETURN_FALSE_ON_FAIL(
destination->WriteBits(video_signal_type_present_flag, 1));
if (video_signal_type_present_flag) {
RETURN_FALSE_ON_FAIL(AddVideoSignalTypeInfo(destination, *color_space));
}
// chroma_loc_info_present_flag, timing_info_present_flag,
// nal_hrd_parameters_present_flag, vcl_hrd_parameters_present_flag,
// pic_struct_present_flag, All u(1)
RETURN_FALSE_ON_FAIL(destination->WriteBits(0, 5));
// bitstream_restriction_flag: u(1)
RETURN_FALSE_ON_FAIL(destination->WriteBits(1, 1));
RETURN_FALSE_ON_FAIL(
AddBitstreamRestriction(destination, sps.max_num_ref_frames));
*out_vui_rewritten = SpsVuiRewriter::ParseResult::kVuiRewritten;
} else {
// Parse out the full VUI.
// aspect_ratio_info_present_flag: u(1)
@ -307,19 +337,10 @@ bool CopyAndRewriteVui(const SpsParser::SpsState& sps,
// overscan_appropriate_flag: u(1)
COPY_BITS(source, destination, bits_tmp, 1);
}
// video_signal_type_present_flag: u(1)
COPY_BITS(source, destination, bits_tmp, 1);
if (bits_tmp == 1) {
// video_format + video_full_range_flag: u(3) + u(1)
COPY_BITS(source, destination, bits_tmp, 4);
// colour_description_present_flag: u(1)
COPY_BITS(source, destination, bits_tmp, 1);
if (bits_tmp == 1) {
// colour_primaries, transfer_characteristics, matrix_coefficients:
// u(8) each.
COPY_BITS(source, destination, bits_tmp, 24);
}
}
CopyOrRewriteVideoSignalTypeInfo(source, destination, color_space,
out_vui_rewritten);
// chroma_loc_info_present_flag: u(1)
COPY_BITS(source, destination, bits_tmp, 1);
if (bits_tmp == 1) {
@ -364,6 +385,7 @@ bool CopyAndRewriteVui(const SpsParser::SpsState& sps,
// We're adding one from scratch.
RETURN_FALSE_ON_FAIL(
AddBitstreamRestriction(destination, sps.max_num_ref_frames));
*out_vui_rewritten = SpsVuiRewriter::ParseResult::kVuiRewritten;
} else {
// We're replacing.
// motion_vectors_over_pic_boundaries_flag: u(1)
@ -387,18 +409,15 @@ bool CopyAndRewriteVui(const SpsParser::SpsState& sps,
source->ReadExponentialGolomb(&max_num_reorder_frames));
RETURN_FALSE_ON_FAIL(
source->ReadExponentialGolomb(&max_dec_frame_buffering));
if (max_num_reorder_frames == 0 &&
max_dec_frame_buffering <= sps.max_num_ref_frames) {
RTC_LOG(LS_INFO) << "VUI bitstream already contains an optimal VUI.";
*out_vui_rewritten = SpsVuiRewriter::ParseResult::kVuiOk;
return true;
}
RETURN_FALSE_ON_FAIL(destination->WriteExponentialGolomb(0));
RETURN_FALSE_ON_FAIL(
destination->WriteExponentialGolomb(sps.max_num_ref_frames));
if (max_num_reorder_frames != 0 ||
max_dec_frame_buffering > sps.max_num_ref_frames) {
*out_vui_rewritten = SpsVuiRewriter::ParseResult::kVuiRewritten;
}
}
}
*out_vui_rewritten = SpsVuiRewriter::ParseResult::kVuiRewritten;
return true;
}
@ -461,6 +480,129 @@ bool AddBitstreamRestriction(rtc::BitBufferWriter* destination,
return true;
}
bool IsDefaultColorSpace(const ColorSpace& color_space) {
return color_space.range() != ColorSpace::RangeID::kFull &&
color_space.primaries() == ColorSpace::PrimaryID::kUnspecified &&
color_space.transfer() == ColorSpace::TransferID::kUnspecified &&
color_space.matrix() == ColorSpace::MatrixID::kUnspecified;
}
bool AddVideoSignalTypeInfo(rtc::BitBufferWriter* destination,
const ColorSpace& color_space) {
// video_format: u(3).
RETURN_FALSE_ON_FAIL(destination->WriteBits(5, 3)); // 5 = Unspecified
// video_full_range_flag: u(1)
RETURN_FALSE_ON_FAIL(destination->WriteBits(
color_space.range() == ColorSpace::RangeID::kFull ? 1 : 0, 1));
// colour_description_present_flag: u(1)
RETURN_FALSE_ON_FAIL(destination->WriteBits(1, 1));
// colour_primaries: u(8)
RETURN_FALSE_ON_FAIL(
destination->WriteUInt8(static_cast<uint8_t>(color_space.primaries())));
// transfer_characteristics: u(8)
RETURN_FALSE_ON_FAIL(
destination->WriteUInt8(static_cast<uint8_t>(color_space.transfer())));
// matrix_coefficients: u(8)
RETURN_FALSE_ON_FAIL(
destination->WriteUInt8(static_cast<uint8_t>(color_space.matrix())));
return true;
}
bool CopyOrRewriteVideoSignalTypeInfo(
rtc::BitBuffer* source,
rtc::BitBufferWriter* destination,
const ColorSpace* color_space,
SpsVuiRewriter::ParseResult* out_vui_rewritten) {
// Read.
uint32_t video_signal_type_present_flag;
uint32_t video_format = 5; // H264 default: unspecified
uint32_t video_full_range_flag = 0; // H264 default: limited
uint32_t colour_description_present_flag = 0;
uint8_t colour_primaries = 3; // H264 default: unspecified
uint8_t transfer_characteristics = 3; // H264 default: unspecified
uint8_t matrix_coefficients = 3; // H264 default: unspecified
RETURN_FALSE_ON_FAIL(source->ReadBits(&video_signal_type_present_flag, 1));
if (video_signal_type_present_flag) {
RETURN_FALSE_ON_FAIL(source->ReadBits(&video_format, 3));
RETURN_FALSE_ON_FAIL(source->ReadBits(&video_full_range_flag, 1));
RETURN_FALSE_ON_FAIL(source->ReadBits(&colour_description_present_flag, 1));
if (colour_description_present_flag) {
RETURN_FALSE_ON_FAIL(source->ReadUInt8(&colour_primaries));
RETURN_FALSE_ON_FAIL(source->ReadUInt8(&transfer_characteristics));
RETURN_FALSE_ON_FAIL(source->ReadUInt8(&matrix_coefficients));
}
}
// Update.
uint32_t video_signal_type_present_flag_override =
video_signal_type_present_flag;
uint32_t video_format_override = video_format;
uint32_t video_full_range_flag_override = video_full_range_flag;
uint32_t colour_description_present_flag_override =
colour_description_present_flag;
uint8_t colour_primaries_override = colour_primaries;
uint8_t transfer_characteristics_override = transfer_characteristics;
uint8_t matrix_coefficients_override = matrix_coefficients;
if (color_space) {
if (IsDefaultColorSpace(*color_space)) {
video_signal_type_present_flag_override = 0;
} else {
video_signal_type_present_flag_override = 1;
video_format_override = 5; // unspecified
if (color_space->range() == ColorSpace::RangeID::kFull) {
video_full_range_flag_override = 1;
} else {
// ColorSpace::RangeID::kInvalid and kDerived are treated as limited.
video_full_range_flag_override = 0;
}
colour_description_present_flag_override =
color_space->primaries() != ColorSpace::PrimaryID::kUnspecified ||
color_space->transfer() != ColorSpace::TransferID::kUnspecified ||
color_space->matrix() != ColorSpace::MatrixID::kUnspecified;
colour_primaries_override =
static_cast<uint8_t>(color_space->primaries());
transfer_characteristics_override =
static_cast<uint8_t>(color_space->transfer());
matrix_coefficients_override =
static_cast<uint8_t>(color_space->matrix());
}
}
// Write.
RETURN_FALSE_ON_FAIL(
destination->WriteBits(video_signal_type_present_flag_override, 1));
if (video_signal_type_present_flag_override) {
RETURN_FALSE_ON_FAIL(destination->WriteBits(video_format_override, 3));
RETURN_FALSE_ON_FAIL(
destination->WriteBits(video_full_range_flag_override, 1));
RETURN_FALSE_ON_FAIL(
destination->WriteBits(colour_description_present_flag_override, 1));
if (colour_description_present_flag_override) {
RETURN_FALSE_ON_FAIL(destination->WriteUInt8(colour_primaries_override));
RETURN_FALSE_ON_FAIL(
destination->WriteUInt8(transfer_characteristics_override));
RETURN_FALSE_ON_FAIL(
destination->WriteUInt8(matrix_coefficients_override));
}
}
if (video_signal_type_present_flag_override !=
video_signal_type_present_flag ||
video_format_override != video_format ||
video_full_range_flag_override != video_full_range_flag ||
colour_description_present_flag_override !=
colour_description_present_flag ||
colour_primaries_override != colour_primaries ||
transfer_characteristics_override != transfer_characteristics ||
matrix_coefficients_override != matrix_coefficients) {
*out_vui_rewritten = SpsVuiRewriter::ParseResult::kVuiRewritten;
}
return true;
}
bool CopyRemainingBits(rtc::BitBuffer* source,
rtc::BitBufferWriter* destination) {
uint32_t bits_tmp;

18
common_video/h264/sps_vui_rewriter.h
View File

@ -16,20 +16,19 @@
#include <stdint.h>
#include "absl/types/optional.h"
#include "api/video/color_space.h"
#include "common_video/h264/sps_parser.h"
#include "rtc_base/buffer.h"
#include "rtc_base/copy_on_write_buffer.h"
namespace webrtc {
// A class that can parse an SPS block of a NAL unit and if necessary
// creates a copy with updated settings to allow for faster decoding for streams
// that use picture order count type 0. Streams in that format incur additional
// delay because it allows decode order to differ from render order.
// The mechanism used is to rewrite (edit or add) the SPS's VUI to contain
// restrictions on the maximum number of reordered pictures. This reduces
// latency significantly, though it still adds about a frame of latency to
// decoding.
// A class that can parse an SPS+VUI and if necessary creates a copy with
// updated parameters.
// The rewriter disables frame buffering. This should force decoders to deliver
// decoded frame immediately and, thus, reduce latency.
// The rewriter updates video signal type parameters if external parameters are
// provided.
class SpsVuiRewriter : private SpsParser {
public:
enum class ParseResult { kFailure, kVuiOk, kVuiRewritten };
@ -48,6 +47,7 @@ class SpsVuiRewriter : private SpsParser {
const uint8_t* buffer,
size_t length,
absl::optional<SpsParser::SpsState>* sps,
const ColorSpace* color_space,
rtc::Buffer* destination,
Direction Direction);
@ -61,6 +61,7 @@ class SpsVuiRewriter : private SpsParser {
size_t num_nalus,
const size_t* nalu_offsets,
const size_t* nalu_lengths,
const ColorSpace* color_space,
rtc::CopyOnWriteBuffer* output_buffer,
size_t* output_nalu_offsets,
size_t* output_nalu_lengths);
@ -70,6 +71,7 @@ class SpsVuiRewriter : private SpsParser {
const uint8_t* buffer,
size_t length,
absl::optional<SpsParser::SpsState>* sps,
const ColorSpace* color_space,
rtc::Buffer* destination);
static void UpdateStats(ParseResult result, Direction direction);

292
common_video/h264/sps_vui_rewriter_unittest.cc
View File

@ -11,6 +11,7 @@
#include <cstdint>
#include <vector>
#include "api/video/color_space.h"
#include "common_video/h264/h264_common.h"
#include "common_video/h264/sps_vui_rewriter.h"
#include "rtc_base/bit_buffer.h"
@ -21,6 +22,7 @@
namespace webrtc {
namespace {
enum SpsMode {
kNoRewriteRequired_VuiOptimal,
kRewriteRequired_NoVui,
@ -37,13 +39,159 @@ static const uint8_t kSpsNaluType[] = {H264::NaluType::kSps};
static const uint8_t kIdr1[] = {H264::NaluType::kIdr, 0xFF, 0x00, 0x00, 0x04};
static const uint8_t kIdr2[] = {H264::NaluType::kIdr, 0xFF, 0x00, 0x11};
struct VuiHeader {
uint32_t vui_parameters_present_flag;
uint32_t bitstream_restriction_flag;
uint32_t max_num_reorder_frames;
uint32_t max_dec_frame_buffering;
uint32_t video_signal_type_present_flag;
uint32_t video_full_range_flag;
uint32_t colour_description_present_flag;
uint8_t colour_primaries;
uint8_t transfer_characteristics;
uint8_t matrix_coefficients;
};
static const VuiHeader kVuiNotPresent = {
/* vui_parameters_present_flag= */ 0,
/* bitstream_restriction_flag= */ 0,
/* max_num_reorder_frames= */ 0,
/* max_dec_frame_buffering= */ 0,
/* video_signal_type_present_flag= */ 0,
/* video_full_range_flag= */ 0,
/* colour_description_present_flag= */ 0,
/* colour_primaries= */ 0,
/* transfer_characteristics= */ 0,
/* matrix_coefficients= */ 0};
static const VuiHeader kVuiNoBitstreamRestriction = {
/* vui_parameters_present_flag= */ 1,
/* bitstream_restriction_flag= */ 0,
/* max_num_reorder_frames= */ 0,
/* max_dec_frame_buffering= */ 0,
/* video_signal_type_present_flag= */ 0,
/* video_full_range_flag= */ 0,
/* colour_description_present_flag= */ 0,
/* colour_primaries= */ 0,
/* transfer_characteristics= */ 0,
/* matrix_coefficients= */ 0};
static const VuiHeader kVuiNoFrameBuffering = {
/* vui_parameters_present_flag= */ 1,
/* bitstream_restriction_flag= */ 1,
/* max_num_reorder_frames= */ 0,
/* max_dec_frame_buffering= */ 1,
/* video_signal_type_present_flag= */ 0,
/* video_full_range_flag= */ 0,
/* colour_description_present_flag= */ 0,
/* colour_primaries= */ 0,
/* transfer_characteristics= */ 0,
/* matrix_coefficients= */ 0};
static const VuiHeader kVuiFrameBuffering = {
/* vui_parameters_present_flag= */ 1,
/* bitstream_restriction_flag= */ 1,
/* max_num_reorder_frames= */ 3,
/* max_dec_frame_buffering= */ 3,
/* video_signal_type_present_flag= */ 0,
/* video_full_range_flag= */ 0,
/* colour_description_present_flag= */ 0,
/* colour_primaries= */ 0,
/* transfer_characteristics= */ 0,
/* matrix_coefficients= */ 0};
static const VuiHeader kVuiNoVideoSignalType = {
/* vui_parameters_present_flag= */ 1,
/* bitstream_restriction_flag= */ 1,
/* max_num_reorder_frames= */ 0,
/* max_dec_frame_buffering= */ 1,
/* video_signal_type_present_flag= */ 0,
/* video_full_range_flag= */ 0,
/* colour_description_present_flag= */ 0,
/* colour_primaries= */ 0,
/* transfer_characteristics= */ 0,
/* matrix_coefficients= */ 0};
static const VuiHeader kVuiLimitedRangeNoColourDescription = {
/* vui_parameters_present_flag= */ 1,
/* bitstream_restriction_flag= */ 1,
/* max_num_reorder_frames= */ 0,
/* max_dec_frame_buffering= */ 1,
/* video_signal_type_present_flag= */ 1,
/* video_full_range_flag= */ 0,
/* colour_description_present_flag= */ 0,
/* colour_primaries= */ 0,
/* transfer_characteristics= */ 0,
/* matrix_coefficients= */ 0};
static const VuiHeader kVuiFullRangeNoColourDescription = {
/* vui_parameters_present_flag= */ 1,
/* bitstream_restriction_flag= */ 1,
/* max_num_reorder_frames= */ 0,
/* max_dec_frame_buffering= */ 1,
/* video_signal_type_present_flag= */ 1,
/* video_full_range_flag= */ 1,
/* colour_description_present_flag= */ 0,
/* colour_primaries= */ 0,
/* transfer_characteristics= */ 0,
/* matrix_coefficients= */ 0};
static const VuiHeader kVuiLimitedRangeBt709Color = {
/* vui_parameters_present_flag= */ 1,
/* bitstream_restriction_flag= */ 1,
/* max_num_reorder_frames= */ 0,
/* max_dec_frame_buffering= */ 1,
/* video_signal_type_present_flag= */ 1,
/* video_full_range_flag= */ 0,
/* colour_description_present_flag= */ 1,
/* colour_primaries= */ 1,
/* transfer_characteristics= */ 1,
/* matrix_coefficients= */ 1};
static const webrtc::ColorSpace kColorSpaceH264Default(
ColorSpace::PrimaryID::kUnspecified,
ColorSpace::TransferID::kUnspecified,
ColorSpace::MatrixID::kUnspecified,
ColorSpace::RangeID::kLimited);
static const webrtc::ColorSpace kColorSpacePrimariesBt709(
ColorSpace::PrimaryID::kBT709,
ColorSpace::TransferID::kUnspecified,
ColorSpace::MatrixID::kUnspecified,
ColorSpace::RangeID::kLimited);
static const webrtc::ColorSpace kColorSpaceTransferBt709(
ColorSpace::PrimaryID::kUnspecified,
ColorSpace::TransferID::kBT709,
ColorSpace::MatrixID::kUnspecified,
ColorSpace::RangeID::kLimited);
static const webrtc::ColorSpace kColorSpaceMatrixBt709(
ColorSpace::PrimaryID::kUnspecified,
ColorSpace::TransferID::kUnspecified,
ColorSpace::MatrixID::kBT709,
ColorSpace::RangeID::kLimited);
static const webrtc::ColorSpace kColorSpaceFullRange(
ColorSpace::PrimaryID::kBT709,
ColorSpace::TransferID::kUnspecified,
ColorSpace::MatrixID::kUnspecified,
ColorSpace::RangeID::kFull);
static const webrtc::ColorSpace kColorSpaceBt709LimitedRange(
ColorSpace::PrimaryID::kBT709,
ColorSpace::TransferID::kBT709,
ColorSpace::MatrixID::kBT709,
ColorSpace::RangeID::kLimited);
} // namespace
// Generates a fake SPS with basically everything empty and with characteristics
// based off SpsMode.
// Pass in a buffer of at least kSpsBufferMaxSize.
// The fake SPS that this generates also always has at least one emulation byte
// at offset 2, since the first two bytes are always 0, and has a 0x3 as the
// level_idc, to make sure the parser doesn't eat all 0x3 bytes.
void GenerateFakeSps(SpsMode mode, rtc::Buffer* out_buffer) {
void GenerateFakeSps(const VuiHeader& vui, rtc::Buffer* out_buffer) {
uint8_t rbsp[kSpsBufferMaxSize] = {0};
rtc::BitBufferWriter writer(rbsp, kSpsBufferMaxSize);
// Profile byte.
@ -97,16 +245,31 @@ void GenerateFakeSps(SpsMode mode, rtc::Buffer* out_buffer) {
// Finally! The VUI.
// vui_parameters_present_flag: u(1)
if (mode == kRewriteRequired_NoVui) {
writer.WriteBits(0, 1);
} else {
writer.WriteBits(1, 1);
// VUI time. 8 flags to ignore followed by the bitstream restriction flag.
writer.WriteBits(0, 8);
if (mode == kRewriteRequired_NoBitstreamRestriction) {
writer.WriteBits(0, 1);
} else {
writer.WriteBits(1, 1);
writer.WriteBits(vui.vui_parameters_present_flag, 1);
if (vui.vui_parameters_present_flag) {
// aspect_ratio_info_present_flag, overscan_info_present_flag. Both u(1).
writer.WriteBits(0, 2);
writer.WriteBits(vui.video_signal_type_present_flag, 1);
if (vui.video_signal_type_present_flag) {
// video_format: u(3). 5 = Unspecified
writer.WriteBits(5, 3);
writer.WriteBits(vui.video_full_range_flag, 1);
writer.WriteBits(vui.colour_description_present_flag, 1);
if (vui.colour_description_present_flag) {
writer.WriteUInt8(vui.colour_primaries);
writer.WriteUInt8(vui.transfer_characteristics);
writer.WriteUInt8(vui.matrix_coefficients);
}
}
// chroma_loc_info_present_flag, timing_info_present_flag,
// nal_hrd_parameters_present_flag, vcl_hrd_parameters_present_flag,
// pic_struct_present_flag, All u(1)
writer.WriteBits(0, 5);
writer.WriteBits(vui.bitstream_restriction_flag, 1);
if (vui.bitstream_restriction_flag) {
// Write some defaults. Shouldn't matter for parsing, though.
// motion_vectors_over_pic_boundaries_flag: u(1)
writer.WriteBits(1, 1);
@ -120,15 +283,8 @@ void GenerateFakeSps(SpsMode mode, rtc::Buffer* out_buffer) {
writer.WriteExponentialGolomb(16);
// Next are the limits we care about.
// max_num_reorder_frames: ue(v)
// max_dec_frame_buffering: ue(v)
if (mode == kRewriteRequired_VuiSuboptimal) {
writer.WriteExponentialGolomb(4);
writer.WriteExponentialGolomb(4);
} else {
writer.WriteExponentialGolomb(0);
writer.WriteExponentialGolomb(1);
}
writer.WriteExponentialGolomb(vui.max_num_reorder_frames);
writer.WriteExponentialGolomb(vui.max_dec_frame_buffering);
}
}
@ -142,21 +298,23 @@ void GenerateFakeSps(SpsMode mode, rtc::Buffer* out_buffer) {
H264::WriteRbsp(rbsp, byte_count, out_buffer);
}
void TestSps(SpsMode mode, SpsVuiRewriter::ParseResult expected_parse_result) {
void TestSps(const VuiHeader& vui,
const ColorSpace* color_space,
SpsVuiRewriter::ParseResult expected_parse_result) {
rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE);
rtc::Buffer original_sps;
GenerateFakeSps(mode, &original_sps);
GenerateFakeSps(vui, &original_sps);
absl::optional<SpsParser::SpsState> sps;
rtc::Buffer rewritten_sps;
SpsVuiRewriter::ParseResult result = SpsVuiRewriter::ParseAndRewriteSps(
original_sps.data(), original_sps.size(), &sps, &rewritten_sps,
SpsVuiRewriter::Direction::kIncoming);
original_sps.data(), original_sps.size(), &sps, color_space,
&rewritten_sps, SpsVuiRewriter::Direction::kIncoming);
EXPECT_EQ(expected_parse_result, result);
ASSERT_TRUE(sps);
EXPECT_EQ(sps->width, kWidth);
EXPECT_EQ(sps->height, kHeight);
if (mode != kRewriteRequired_NoVui) {
if (vui.vui_parameters_present_flag) {
EXPECT_EQ(sps->vui_params_present, 1u);
}
@ -164,7 +322,7 @@ void TestSps(SpsMode mode, SpsVuiRewriter::ParseResult expected_parse_result) {
// Ensure that added/rewritten SPS is parsable.
rtc::Buffer tmp;
result = SpsVuiRewriter::ParseAndRewriteSps(
rewritten_sps.data(), rewritten_sps.size(), &sps, &tmp,
rewritten_sps.data(), rewritten_sps.size(), &sps, nullptr, &tmp,
SpsVuiRewriter::Direction::kIncoming);
EXPECT_EQ(SpsVuiRewriter::ParseResult::kVuiOk, result);
ASSERT_TRUE(sps);
@ -174,27 +332,67 @@ void TestSps(SpsMode mode, SpsVuiRewriter::ParseResult expected_parse_result) {
}
}
#define REWRITE_TEST(test_name, mode, expected_parse_result) \
TEST(SpsVuiRewriterTest, test_name) { TestSps(mode, expected_parse_result); }
class SpsVuiRewriterTest : public ::testing::Test,
public ::testing::WithParamInterface<
::testing::tuple<VuiHeader,
const ColorSpace*,
SpsVuiRewriter::ParseResult>> {
};
REWRITE_TEST(VuiAlreadyOptimal,
kNoRewriteRequired_VuiOptimal,
SpsVuiRewriter::ParseResult::kVuiOk)
REWRITE_TEST(RewriteFullVui,
kRewriteRequired_NoVui,
SpsVuiRewriter::ParseResult::kVuiRewritten)
REWRITE_TEST(AddBitstreamRestriction,
kRewriteRequired_NoBitstreamRestriction,
SpsVuiRewriter::ParseResult::kVuiRewritten)
REWRITE_TEST(RewriteSuboptimalVui,
kRewriteRequired_VuiSuboptimal,
SpsVuiRewriter::ParseResult::kVuiRewritten)
TEST_P(SpsVuiRewriterTest, RewriteVui) {
VuiHeader vui = ::testing::get<0>(GetParam());
const ColorSpace* color_space = ::testing::get<1>(GetParam());
SpsVuiRewriter::ParseResult expected_parse_result =
::testing::get<2>(GetParam());
TestSps(vui, color_space, expected_parse_result);
}
TEST(SpsVuiRewriterTest, ParseOutgoingBitstreamOptimalVui) {
INSTANTIATE_TEST_SUITE_P(
,
SpsVuiRewriterTest,
::testing::Values(
std::make_tuple(kVuiNoFrameBuffering,
nullptr,
SpsVuiRewriter::ParseResult::kVuiOk),
std::make_tuple(kVuiNoVideoSignalType,
&kColorSpaceH264Default,
SpsVuiRewriter::ParseResult::kVuiOk),
std::make_tuple(kVuiLimitedRangeBt709Color,
&kColorSpaceBt709LimitedRange,
SpsVuiRewriter::ParseResult::kVuiOk),
std::make_tuple(kVuiNotPresent,
nullptr,
SpsVuiRewriter::ParseResult::kVuiRewritten),
std::make_tuple(kVuiNoBitstreamRestriction,
nullptr,
SpsVuiRewriter::ParseResult::kVuiRewritten),
std::make_tuple(kVuiFrameBuffering,
nullptr,
SpsVuiRewriter::ParseResult::kVuiRewritten),
std::make_tuple(kVuiLimitedRangeNoColourDescription,
&kColorSpaceFullRange,
SpsVuiRewriter::ParseResult::kVuiRewritten),
std::make_tuple(kVuiNoVideoSignalType,
&kColorSpacePrimariesBt709,
SpsVuiRewriter::ParseResult::kVuiRewritten),
std::make_tuple(kVuiNoVideoSignalType,
&kColorSpaceTransferBt709,
SpsVuiRewriter::ParseResult::kVuiRewritten),
std::make_tuple(kVuiNoVideoSignalType,
&kColorSpaceMatrixBt709,
SpsVuiRewriter::ParseResult::kVuiRewritten),
std::make_tuple(kVuiFullRangeNoColourDescription,
&kColorSpaceH264Default,
SpsVuiRewriter::ParseResult::kVuiRewritten),
std::make_tuple(kVuiLimitedRangeBt709Color,
&kColorSpaceH264Default,
SpsVuiRewriter::ParseResult::kVuiRewritten)));
TEST(SpsVuiRewriterOutgoingVuiTest, ParseOutgoingBitstreamOptimalVui) {
rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE);
rtc::Buffer optimal_sps;
GenerateFakeSps(kNoRewriteRequired_VuiOptimal, &optimal_sps);
GenerateFakeSps(kVuiNoFrameBuffering, &optimal_sps);
rtc::Buffer buffer;
const size_t kNumNalus = 2;
@ -214,7 +412,7 @@ TEST(SpsVuiRewriterTest, ParseOutgoingBitstreamOptimalVui) {
size_t modified_nalu_lengths[kNumNalus];
SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps(
buffer, kNumNalus, nalu_offsets, nalu_lengths, &modified_buffer,
buffer, kNumNalus, nalu_offsets, nalu_lengths, nullptr, &modified_buffer,
modified_nalu_offsets, modified_nalu_lengths);
EXPECT_THAT(
@ -229,11 +427,11 @@ TEST(SpsVuiRewriterTest, ParseOutgoingBitstreamOptimalVui) {
::testing::ElementsAreArray(nalu_lengths, kNumNalus));
}
TEST(SpsVuiRewriterTest, ParseOutgoingBitstreamNoVui) {
TEST(SpsVuiRewriterOutgoingVuiTest, ParseOutgoingBitstreamNoVui) {
rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE);
rtc::Buffer sps;
GenerateFakeSps(kRewriteRequired_NoVui, &sps);
GenerateFakeSps(kVuiNotPresent, &sps);
rtc::Buffer buffer;
const size_t kNumNalus = 3;
@ -254,7 +452,7 @@ TEST(SpsVuiRewriterTest, ParseOutgoingBitstreamNoVui) {
buffer.AppendData(kIdr2);
rtc::Buffer optimal_sps;
GenerateFakeSps(kNoRewriteRequired_VuiOptimal, &optimal_sps);
GenerateFakeSps(kVuiNoFrameBuffering, &optimal_sps);
rtc::Buffer expected_buffer;
size_t expected_nalu_offsets[kNumNalus];
@ -278,7 +476,7 @@ TEST(SpsVuiRewriterTest, ParseOutgoingBitstreamNoVui) {
size_t modified_nalu_lengths[kNumNalus];
SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps(
buffer, kNumNalus, nalu_offsets, nalu_lengths, &modified_buffer,
buffer, kNumNalus, nalu_offsets, nalu_lengths, nullptr, &modified_buffer,
modified_nalu_offsets, modified_nalu_lengths);
EXPECT_THAT(

2
modules/rtp_rtcp/source/rtp_format_h264.cc
View File

@ -458,7 +458,7 @@ bool RtpDepacketizerH264::ProcessStapAOrSingleNalu(
SpsVuiRewriter::ParseResult result = SpsVuiRewriter::ParseAndRewriteSps(
&payload_data[start_offset], end_offset - start_offset, &sps,
output_buffer.get(), SpsVuiRewriter::Direction::kIncoming);
nullptr, output_buffer.get(), SpsVuiRewriter::Direction::kIncoming);
if (result == SpsVuiRewriter::ParseResult::kVuiRewritten) {
if (modified_buffer_) {

3
video/frame_encode_metadata_writer.cc
View File

@ -207,7 +207,8 @@ FrameEncodeMetadataWriter::UpdateBitstream(
SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps(
buffer, fragmentation->fragmentationVectorSize,
fragmentation->fragmentationOffset, fragmentation->fragmentationLength,
&modified_buffer, modified_fragmentation->fragmentationOffset,
encoded_image->ColorSpace(), &modified_buffer,
modified_fragmentation->fragmentationOffset,
modified_fragmentation->fragmentationLength);
encoded_image->SetEncodedData(modified_buffer);