diff --git a/common_video/h264/sps_vui_rewriter.cc b/common_video/h264/sps_vui_rewriter.cc index b6cb4ed257..8c62495733 100644 --- a/common_video/h264/sps_vui_rewriter.cc +++ b/common_video/h264/sps_vui_rewriter.cc @@ -15,6 +15,7 @@ #include #include +#include "api/video/color_space.h" #include "common_video/h264/h264_common.h" #include "common_video/h264/sps_parser.h" #include "rtc_base/bit_buffer.h" @@ -73,14 +74,22 @@ enum SpsValidEvent { bool CopyAndRewriteVui(const SpsParser::SpsState& sps, rtc::BitBuffer* source, rtc::BitBufferWriter* destination, + const webrtc::ColorSpace* color_space, SpsVuiRewriter::ParseResult* out_vui_rewritten); bool CopyHrdParameters(rtc::BitBuffer* source, rtc::BitBufferWriter* destination); bool AddBitstreamRestriction(rtc::BitBufferWriter* destination, uint32_t max_num_ref_frames); +bool IsDefaultColorSpace(const ColorSpace& color_space); +bool AddVideoSignalTypeInfo(rtc::BitBufferWriter* destination, + const ColorSpace& color_space); +bool CopyOrRewriteVideoSignalTypeInfo( + rtc::BitBuffer* source, + rtc::BitBufferWriter* destination, + const ColorSpace* color_space, + SpsVuiRewriter::ParseResult* out_vui_rewritten); bool CopyRemainingBits(rtc::BitBuffer* source, rtc::BitBufferWriter* destination); - } // namespace void SpsVuiRewriter::UpdateStats(ParseResult result, Direction direction) { @@ -116,6 +125,7 @@ SpsVuiRewriter::ParseResult SpsVuiRewriter::ParseAndRewriteSps( const uint8_t* buffer, size_t length, absl::optional* sps, + const webrtc::ColorSpace* color_space, rtc::Buffer* destination) { // Create temporary RBSP decoded buffer of the payload (exlcuding the // leading nalu type header byte (the SpsParser uses only the payload). @@ -151,7 +161,7 @@ SpsVuiRewriter::ParseResult SpsVuiRewriter::ParseAndRewriteSps( sps_writer.Seek(byte_offset, bit_offset); ParseResult vui_updated; - if (!CopyAndRewriteVui(*sps_state, &source_buffer, &sps_writer, + if (!CopyAndRewriteVui(*sps_state, &source_buffer, &sps_writer, color_space, &vui_updated)) { RTC_LOG(LS_ERROR) << "Failed to parse/copy SPS VUI."; return ParseResult::kFailure; @@ -190,9 +200,11 @@ SpsVuiRewriter::ParseResult SpsVuiRewriter::ParseAndRewriteSps( const uint8_t* buffer, size_t length, absl::optional* sps, + const webrtc::ColorSpace* color_space, rtc::Buffer* destination, Direction direction) { - ParseResult result = ParseAndRewriteSps(buffer, length, sps, destination); + ParseResult result = + ParseAndRewriteSps(buffer, length, sps, color_space, destination); UpdateStats(result, direction); return result; } @@ -202,6 +214,7 @@ void SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps( size_t num_nalus, const size_t* nalu_offsets, const size_t* nalu_lengths, + const webrtc::ColorSpace* color_space, rtc::CopyOnWriteBuffer* output_buffer, size_t* output_nalu_offsets, size_t* output_nalu_lengths) { @@ -244,7 +257,7 @@ void SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps( ParseResult result = ParseAndRewriteSps( nalu_ptr + H264::kNaluTypeSize, nalu_length - H264::kNaluTypeSize, - &sps, &output_nalu, Direction::kOutgoing); + &sps, color_space, &output_nalu, Direction::kOutgoing); if (result == ParseResult::kVuiRewritten) { updated_sps = true; output_nalu_offsets[i] = output_buffer->size(); @@ -265,14 +278,16 @@ void SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps( } namespace { - bool CopyAndRewriteVui(const SpsParser::SpsState& sps, rtc::BitBuffer* source, rtc::BitBufferWriter* destination, + const webrtc::ColorSpace* color_space, SpsVuiRewriter::ParseResult* out_vui_rewritten) { uint32_t golomb_tmp; uint32_t bits_tmp; + *out_vui_rewritten = SpsVuiRewriter::ParseResult::kVuiOk; + // // vui_parameters_present_flag: u(1) // @@ -283,12 +298,27 @@ bool CopyAndRewriteVui(const SpsParser::SpsState& sps, // (2) rewrite frame reordering values so no reordering is allowed. if (!sps.vui_params_present) { // Write a simple VUI with the parameters we want and 0 for all other flags. - // There are 8 flags to be off before the bitstream restriction flag. - RETURN_FALSE_ON_FAIL(destination->WriteBits(0, 8)); + + // aspect_ratio_info_present_flag, overscan_info_present_flag. Both u(1). + RETURN_FALSE_ON_FAIL(destination->WriteBits(0, 2)); + + uint32_t video_signal_type_present_flag = + (color_space && !IsDefaultColorSpace(*color_space)) ? 1 : 0; + RETURN_FALSE_ON_FAIL( + destination->WriteBits(video_signal_type_present_flag, 1)); + if (video_signal_type_present_flag) { + RETURN_FALSE_ON_FAIL(AddVideoSignalTypeInfo(destination, *color_space)); + } + // chroma_loc_info_present_flag, timing_info_present_flag, + // nal_hrd_parameters_present_flag, vcl_hrd_parameters_present_flag, + // pic_struct_present_flag, All u(1) + RETURN_FALSE_ON_FAIL(destination->WriteBits(0, 5)); // bitstream_restriction_flag: u(1) RETURN_FALSE_ON_FAIL(destination->WriteBits(1, 1)); RETURN_FALSE_ON_FAIL( AddBitstreamRestriction(destination, sps.max_num_ref_frames)); + + *out_vui_rewritten = SpsVuiRewriter::ParseResult::kVuiRewritten; } else { // Parse out the full VUI. // aspect_ratio_info_present_flag: u(1) @@ -307,19 +337,10 @@ bool CopyAndRewriteVui(const SpsParser::SpsState& sps, // overscan_appropriate_flag: u(1) COPY_BITS(source, destination, bits_tmp, 1); } - // video_signal_type_present_flag: u(1) - COPY_BITS(source, destination, bits_tmp, 1); - if (bits_tmp == 1) { - // video_format + video_full_range_flag: u(3) + u(1) - COPY_BITS(source, destination, bits_tmp, 4); - // colour_description_present_flag: u(1) - COPY_BITS(source, destination, bits_tmp, 1); - if (bits_tmp == 1) { - // colour_primaries, transfer_characteristics, matrix_coefficients: - // u(8) each. - COPY_BITS(source, destination, bits_tmp, 24); - } - } + + CopyOrRewriteVideoSignalTypeInfo(source, destination, color_space, + out_vui_rewritten); + // chroma_loc_info_present_flag: u(1) COPY_BITS(source, destination, bits_tmp, 1); if (bits_tmp == 1) { @@ -364,6 +385,7 @@ bool CopyAndRewriteVui(const SpsParser::SpsState& sps, // We're adding one from scratch. RETURN_FALSE_ON_FAIL( AddBitstreamRestriction(destination, sps.max_num_ref_frames)); + *out_vui_rewritten = SpsVuiRewriter::ParseResult::kVuiRewritten; } else { // We're replacing. // motion_vectors_over_pic_boundaries_flag: u(1) @@ -387,18 +409,15 @@ bool CopyAndRewriteVui(const SpsParser::SpsState& sps, source->ReadExponentialGolomb(&max_num_reorder_frames)); RETURN_FALSE_ON_FAIL( source->ReadExponentialGolomb(&max_dec_frame_buffering)); - if (max_num_reorder_frames == 0 && - max_dec_frame_buffering <= sps.max_num_ref_frames) { - RTC_LOG(LS_INFO) << "VUI bitstream already contains an optimal VUI."; - *out_vui_rewritten = SpsVuiRewriter::ParseResult::kVuiOk; - return true; - } RETURN_FALSE_ON_FAIL(destination->WriteExponentialGolomb(0)); RETURN_FALSE_ON_FAIL( destination->WriteExponentialGolomb(sps.max_num_ref_frames)); + if (max_num_reorder_frames != 0 || + max_dec_frame_buffering > sps.max_num_ref_frames) { + *out_vui_rewritten = SpsVuiRewriter::ParseResult::kVuiRewritten; + } } } - *out_vui_rewritten = SpsVuiRewriter::ParseResult::kVuiRewritten; return true; } @@ -461,6 +480,129 @@ bool AddBitstreamRestriction(rtc::BitBufferWriter* destination, return true; } +bool IsDefaultColorSpace(const ColorSpace& color_space) { + return color_space.range() != ColorSpace::RangeID::kFull && + color_space.primaries() == ColorSpace::PrimaryID::kUnspecified && + color_space.transfer() == ColorSpace::TransferID::kUnspecified && + color_space.matrix() == ColorSpace::MatrixID::kUnspecified; +} + +bool AddVideoSignalTypeInfo(rtc::BitBufferWriter* destination, + const ColorSpace& color_space) { + // video_format: u(3). + RETURN_FALSE_ON_FAIL(destination->WriteBits(5, 3)); // 5 = Unspecified + // video_full_range_flag: u(1) + RETURN_FALSE_ON_FAIL(destination->WriteBits( + color_space.range() == ColorSpace::RangeID::kFull ? 1 : 0, 1)); + // colour_description_present_flag: u(1) + RETURN_FALSE_ON_FAIL(destination->WriteBits(1, 1)); + // colour_primaries: u(8) + RETURN_FALSE_ON_FAIL( + destination->WriteUInt8(static_cast(color_space.primaries()))); + // transfer_characteristics: u(8) + RETURN_FALSE_ON_FAIL( + destination->WriteUInt8(static_cast(color_space.transfer()))); + // matrix_coefficients: u(8) + RETURN_FALSE_ON_FAIL( + destination->WriteUInt8(static_cast(color_space.matrix()))); + return true; +} + +bool CopyOrRewriteVideoSignalTypeInfo( + rtc::BitBuffer* source, + rtc::BitBufferWriter* destination, + const ColorSpace* color_space, + SpsVuiRewriter::ParseResult* out_vui_rewritten) { + // Read. + uint32_t video_signal_type_present_flag; + uint32_t video_format = 5; // H264 default: unspecified + uint32_t video_full_range_flag = 0; // H264 default: limited + uint32_t colour_description_present_flag = 0; + uint8_t colour_primaries = 3; // H264 default: unspecified + uint8_t transfer_characteristics = 3; // H264 default: unspecified + uint8_t matrix_coefficients = 3; // H264 default: unspecified + RETURN_FALSE_ON_FAIL(source->ReadBits(&video_signal_type_present_flag, 1)); + if (video_signal_type_present_flag) { + RETURN_FALSE_ON_FAIL(source->ReadBits(&video_format, 3)); + RETURN_FALSE_ON_FAIL(source->ReadBits(&video_full_range_flag, 1)); + RETURN_FALSE_ON_FAIL(source->ReadBits(&colour_description_present_flag, 1)); + if (colour_description_present_flag) { + RETURN_FALSE_ON_FAIL(source->ReadUInt8(&colour_primaries)); + RETURN_FALSE_ON_FAIL(source->ReadUInt8(&transfer_characteristics)); + RETURN_FALSE_ON_FAIL(source->ReadUInt8(&matrix_coefficients)); + } + } + + // Update. + uint32_t video_signal_type_present_flag_override = + video_signal_type_present_flag; + uint32_t video_format_override = video_format; + uint32_t video_full_range_flag_override = video_full_range_flag; + uint32_t colour_description_present_flag_override = + colour_description_present_flag; + uint8_t colour_primaries_override = colour_primaries; + uint8_t transfer_characteristics_override = transfer_characteristics; + uint8_t matrix_coefficients_override = matrix_coefficients; + if (color_space) { + if (IsDefaultColorSpace(*color_space)) { + video_signal_type_present_flag_override = 0; + } else { + video_signal_type_present_flag_override = 1; + video_format_override = 5; // unspecified + + if (color_space->range() == ColorSpace::RangeID::kFull) { + video_full_range_flag_override = 1; + } else { + // ColorSpace::RangeID::kInvalid and kDerived are treated as limited. + video_full_range_flag_override = 0; + } + + colour_description_present_flag_override = + color_space->primaries() != ColorSpace::PrimaryID::kUnspecified || + color_space->transfer() != ColorSpace::TransferID::kUnspecified || + color_space->matrix() != ColorSpace::MatrixID::kUnspecified; + colour_primaries_override = + static_cast(color_space->primaries()); + transfer_characteristics_override = + static_cast(color_space->transfer()); + matrix_coefficients_override = + static_cast(color_space->matrix()); + } + } + + // Write. + RETURN_FALSE_ON_FAIL( + destination->WriteBits(video_signal_type_present_flag_override, 1)); + if (video_signal_type_present_flag_override) { + RETURN_FALSE_ON_FAIL(destination->WriteBits(video_format_override, 3)); + RETURN_FALSE_ON_FAIL( + destination->WriteBits(video_full_range_flag_override, 1)); + RETURN_FALSE_ON_FAIL( + destination->WriteBits(colour_description_present_flag_override, 1)); + if (colour_description_present_flag_override) { + RETURN_FALSE_ON_FAIL(destination->WriteUInt8(colour_primaries_override)); + RETURN_FALSE_ON_FAIL( + destination->WriteUInt8(transfer_characteristics_override)); + RETURN_FALSE_ON_FAIL( + destination->WriteUInt8(matrix_coefficients_override)); + } + } + + if (video_signal_type_present_flag_override != + video_signal_type_present_flag || + video_format_override != video_format || + video_full_range_flag_override != video_full_range_flag || + colour_description_present_flag_override != + colour_description_present_flag || + colour_primaries_override != colour_primaries || + transfer_characteristics_override != transfer_characteristics || + matrix_coefficients_override != matrix_coefficients) { + *out_vui_rewritten = SpsVuiRewriter::ParseResult::kVuiRewritten; + } + + return true; +} + bool CopyRemainingBits(rtc::BitBuffer* source, rtc::BitBufferWriter* destination) { uint32_t bits_tmp; diff --git a/common_video/h264/sps_vui_rewriter.h b/common_video/h264/sps_vui_rewriter.h index 250b641644..0590b5a031 100644 --- a/common_video/h264/sps_vui_rewriter.h +++ b/common_video/h264/sps_vui_rewriter.h @@ -16,20 +16,19 @@ #include #include "absl/types/optional.h" +#include "api/video/color_space.h" #include "common_video/h264/sps_parser.h" #include "rtc_base/buffer.h" #include "rtc_base/copy_on_write_buffer.h" namespace webrtc { -// A class that can parse an SPS block of a NAL unit and if necessary -// creates a copy with updated settings to allow for faster decoding for streams -// that use picture order count type 0. Streams in that format incur additional -// delay because it allows decode order to differ from render order. -// The mechanism used is to rewrite (edit or add) the SPS's VUI to contain -// restrictions on the maximum number of reordered pictures. This reduces -// latency significantly, though it still adds about a frame of latency to -// decoding. +// A class that can parse an SPS+VUI and if necessary creates a copy with +// updated parameters. +// The rewriter disables frame buffering. This should force decoders to deliver +// decoded frame immediately and, thus, reduce latency. +// The rewriter updates video signal type parameters if external parameters are +// provided. class SpsVuiRewriter : private SpsParser { public: enum class ParseResult { kFailure, kVuiOk, kVuiRewritten }; @@ -48,6 +47,7 @@ class SpsVuiRewriter : private SpsParser { const uint8_t* buffer, size_t length, absl::optional* sps, + const ColorSpace* color_space, rtc::Buffer* destination, Direction Direction); @@ -61,6 +61,7 @@ class SpsVuiRewriter : private SpsParser { size_t num_nalus, const size_t* nalu_offsets, const size_t* nalu_lengths, + const ColorSpace* color_space, rtc::CopyOnWriteBuffer* output_buffer, size_t* output_nalu_offsets, size_t* output_nalu_lengths); @@ -70,6 +71,7 @@ class SpsVuiRewriter : private SpsParser { const uint8_t* buffer, size_t length, absl::optional* sps, + const ColorSpace* color_space, rtc::Buffer* destination); static void UpdateStats(ParseResult result, Direction direction); diff --git a/common_video/h264/sps_vui_rewriter_unittest.cc b/common_video/h264/sps_vui_rewriter_unittest.cc index 263bfefaf0..870981fdab 100644 --- a/common_video/h264/sps_vui_rewriter_unittest.cc +++ b/common_video/h264/sps_vui_rewriter_unittest.cc @@ -11,6 +11,7 @@ #include #include +#include "api/video/color_space.h" #include "common_video/h264/h264_common.h" #include "common_video/h264/sps_vui_rewriter.h" #include "rtc_base/bit_buffer.h" @@ -21,6 +22,7 @@ namespace webrtc { +namespace { enum SpsMode { kNoRewriteRequired_VuiOptimal, kRewriteRequired_NoVui, @@ -37,13 +39,159 @@ static const uint8_t kSpsNaluType[] = {H264::NaluType::kSps}; static const uint8_t kIdr1[] = {H264::NaluType::kIdr, 0xFF, 0x00, 0x00, 0x04}; static const uint8_t kIdr2[] = {H264::NaluType::kIdr, 0xFF, 0x00, 0x11}; +struct VuiHeader { + uint32_t vui_parameters_present_flag; + uint32_t bitstream_restriction_flag; + uint32_t max_num_reorder_frames; + uint32_t max_dec_frame_buffering; + uint32_t video_signal_type_present_flag; + uint32_t video_full_range_flag; + uint32_t colour_description_present_flag; + uint8_t colour_primaries; + uint8_t transfer_characteristics; + uint8_t matrix_coefficients; +}; + +static const VuiHeader kVuiNotPresent = { + /* vui_parameters_present_flag= */ 0, + /* bitstream_restriction_flag= */ 0, + /* max_num_reorder_frames= */ 0, + /* max_dec_frame_buffering= */ 0, + /* video_signal_type_present_flag= */ 0, + /* video_full_range_flag= */ 0, + /* colour_description_present_flag= */ 0, + /* colour_primaries= */ 0, + /* transfer_characteristics= */ 0, + /* matrix_coefficients= */ 0}; + +static const VuiHeader kVuiNoBitstreamRestriction = { + /* vui_parameters_present_flag= */ 1, + /* bitstream_restriction_flag= */ 0, + /* max_num_reorder_frames= */ 0, + /* max_dec_frame_buffering= */ 0, + /* video_signal_type_present_flag= */ 0, + /* video_full_range_flag= */ 0, + /* colour_description_present_flag= */ 0, + /* colour_primaries= */ 0, + /* transfer_characteristics= */ 0, + /* matrix_coefficients= */ 0}; + +static const VuiHeader kVuiNoFrameBuffering = { + /* vui_parameters_present_flag= */ 1, + /* bitstream_restriction_flag= */ 1, + /* max_num_reorder_frames= */ 0, + /* max_dec_frame_buffering= */ 1, + /* video_signal_type_present_flag= */ 0, + /* video_full_range_flag= */ 0, + /* colour_description_present_flag= */ 0, + /* colour_primaries= */ 0, + /* transfer_characteristics= */ 0, + /* matrix_coefficients= */ 0}; + +static const VuiHeader kVuiFrameBuffering = { + /* vui_parameters_present_flag= */ 1, + /* bitstream_restriction_flag= */ 1, + /* max_num_reorder_frames= */ 3, + /* max_dec_frame_buffering= */ 3, + /* video_signal_type_present_flag= */ 0, + /* video_full_range_flag= */ 0, + /* colour_description_present_flag= */ 0, + /* colour_primaries= */ 0, + /* transfer_characteristics= */ 0, + /* matrix_coefficients= */ 0}; + +static const VuiHeader kVuiNoVideoSignalType = { + /* vui_parameters_present_flag= */ 1, + /* bitstream_restriction_flag= */ 1, + /* max_num_reorder_frames= */ 0, + /* max_dec_frame_buffering= */ 1, + /* video_signal_type_present_flag= */ 0, + /* video_full_range_flag= */ 0, + /* colour_description_present_flag= */ 0, + /* colour_primaries= */ 0, + /* transfer_characteristics= */ 0, + /* matrix_coefficients= */ 0}; + +static const VuiHeader kVuiLimitedRangeNoColourDescription = { + /* vui_parameters_present_flag= */ 1, + /* bitstream_restriction_flag= */ 1, + /* max_num_reorder_frames= */ 0, + /* max_dec_frame_buffering= */ 1, + /* video_signal_type_present_flag= */ 1, + /* video_full_range_flag= */ 0, + /* colour_description_present_flag= */ 0, + /* colour_primaries= */ 0, + /* transfer_characteristics= */ 0, + /* matrix_coefficients= */ 0}; + +static const VuiHeader kVuiFullRangeNoColourDescription = { + /* vui_parameters_present_flag= */ 1, + /* bitstream_restriction_flag= */ 1, + /* max_num_reorder_frames= */ 0, + /* max_dec_frame_buffering= */ 1, + /* video_signal_type_present_flag= */ 1, + /* video_full_range_flag= */ 1, + /* colour_description_present_flag= */ 0, + /* colour_primaries= */ 0, + /* transfer_characteristics= */ 0, + /* matrix_coefficients= */ 0}; + +static const VuiHeader kVuiLimitedRangeBt709Color = { + /* vui_parameters_present_flag= */ 1, + /* bitstream_restriction_flag= */ 1, + /* max_num_reorder_frames= */ 0, + /* max_dec_frame_buffering= */ 1, + /* video_signal_type_present_flag= */ 1, + /* video_full_range_flag= */ 0, + /* colour_description_present_flag= */ 1, + /* colour_primaries= */ 1, + /* transfer_characteristics= */ 1, + /* matrix_coefficients= */ 1}; + +static const webrtc::ColorSpace kColorSpaceH264Default( + ColorSpace::PrimaryID::kUnspecified, + ColorSpace::TransferID::kUnspecified, + ColorSpace::MatrixID::kUnspecified, + ColorSpace::RangeID::kLimited); + +static const webrtc::ColorSpace kColorSpacePrimariesBt709( + ColorSpace::PrimaryID::kBT709, + ColorSpace::TransferID::kUnspecified, + ColorSpace::MatrixID::kUnspecified, + ColorSpace::RangeID::kLimited); + +static const webrtc::ColorSpace kColorSpaceTransferBt709( + ColorSpace::PrimaryID::kUnspecified, + ColorSpace::TransferID::kBT709, + ColorSpace::MatrixID::kUnspecified, + ColorSpace::RangeID::kLimited); + +static const webrtc::ColorSpace kColorSpaceMatrixBt709( + ColorSpace::PrimaryID::kUnspecified, + ColorSpace::TransferID::kUnspecified, + ColorSpace::MatrixID::kBT709, + ColorSpace::RangeID::kLimited); + +static const webrtc::ColorSpace kColorSpaceFullRange( + ColorSpace::PrimaryID::kBT709, + ColorSpace::TransferID::kUnspecified, + ColorSpace::MatrixID::kUnspecified, + ColorSpace::RangeID::kFull); + +static const webrtc::ColorSpace kColorSpaceBt709LimitedRange( + ColorSpace::PrimaryID::kBT709, + ColorSpace::TransferID::kBT709, + ColorSpace::MatrixID::kBT709, + ColorSpace::RangeID::kLimited); +} // namespace + // Generates a fake SPS with basically everything empty and with characteristics // based off SpsMode. // Pass in a buffer of at least kSpsBufferMaxSize. // The fake SPS that this generates also always has at least one emulation byte // at offset 2, since the first two bytes are always 0, and has a 0x3 as the // level_idc, to make sure the parser doesn't eat all 0x3 bytes. -void GenerateFakeSps(SpsMode mode, rtc::Buffer* out_buffer) { +void GenerateFakeSps(const VuiHeader& vui, rtc::Buffer* out_buffer) { uint8_t rbsp[kSpsBufferMaxSize] = {0}; rtc::BitBufferWriter writer(rbsp, kSpsBufferMaxSize); // Profile byte. @@ -97,16 +245,31 @@ void GenerateFakeSps(SpsMode mode, rtc::Buffer* out_buffer) { // Finally! The VUI. // vui_parameters_present_flag: u(1) - if (mode == kRewriteRequired_NoVui) { - writer.WriteBits(0, 1); - } else { - writer.WriteBits(1, 1); - // VUI time. 8 flags to ignore followed by the bitstream restriction flag. - writer.WriteBits(0, 8); - if (mode == kRewriteRequired_NoBitstreamRestriction) { - writer.WriteBits(0, 1); - } else { - writer.WriteBits(1, 1); + writer.WriteBits(vui.vui_parameters_present_flag, 1); + if (vui.vui_parameters_present_flag) { + // aspect_ratio_info_present_flag, overscan_info_present_flag. Both u(1). + writer.WriteBits(0, 2); + + writer.WriteBits(vui.video_signal_type_present_flag, 1); + if (vui.video_signal_type_present_flag) { + // video_format: u(3). 5 = Unspecified + writer.WriteBits(5, 3); + writer.WriteBits(vui.video_full_range_flag, 1); + writer.WriteBits(vui.colour_description_present_flag, 1); + if (vui.colour_description_present_flag) { + writer.WriteUInt8(vui.colour_primaries); + writer.WriteUInt8(vui.transfer_characteristics); + writer.WriteUInt8(vui.matrix_coefficients); + } + } + + // chroma_loc_info_present_flag, timing_info_present_flag, + // nal_hrd_parameters_present_flag, vcl_hrd_parameters_present_flag, + // pic_struct_present_flag, All u(1) + writer.WriteBits(0, 5); + + writer.WriteBits(vui.bitstream_restriction_flag, 1); + if (vui.bitstream_restriction_flag) { // Write some defaults. Shouldn't matter for parsing, though. // motion_vectors_over_pic_boundaries_flag: u(1) writer.WriteBits(1, 1); @@ -120,15 +283,8 @@ void GenerateFakeSps(SpsMode mode, rtc::Buffer* out_buffer) { writer.WriteExponentialGolomb(16); // Next are the limits we care about. - // max_num_reorder_frames: ue(v) - // max_dec_frame_buffering: ue(v) - if (mode == kRewriteRequired_VuiSuboptimal) { - writer.WriteExponentialGolomb(4); - writer.WriteExponentialGolomb(4); - } else { - writer.WriteExponentialGolomb(0); - writer.WriteExponentialGolomb(1); - } + writer.WriteExponentialGolomb(vui.max_num_reorder_frames); + writer.WriteExponentialGolomb(vui.max_dec_frame_buffering); } } @@ -142,21 +298,23 @@ void GenerateFakeSps(SpsMode mode, rtc::Buffer* out_buffer) { H264::WriteRbsp(rbsp, byte_count, out_buffer); } -void TestSps(SpsMode mode, SpsVuiRewriter::ParseResult expected_parse_result) { +void TestSps(const VuiHeader& vui, + const ColorSpace* color_space, + SpsVuiRewriter::ParseResult expected_parse_result) { rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE); rtc::Buffer original_sps; - GenerateFakeSps(mode, &original_sps); + GenerateFakeSps(vui, &original_sps); absl::optional sps; rtc::Buffer rewritten_sps; SpsVuiRewriter::ParseResult result = SpsVuiRewriter::ParseAndRewriteSps( - original_sps.data(), original_sps.size(), &sps, &rewritten_sps, - SpsVuiRewriter::Direction::kIncoming); + original_sps.data(), original_sps.size(), &sps, color_space, + &rewritten_sps, SpsVuiRewriter::Direction::kIncoming); EXPECT_EQ(expected_parse_result, result); ASSERT_TRUE(sps); EXPECT_EQ(sps->width, kWidth); EXPECT_EQ(sps->height, kHeight); - if (mode != kRewriteRequired_NoVui) { + if (vui.vui_parameters_present_flag) { EXPECT_EQ(sps->vui_params_present, 1u); } @@ -164,7 +322,7 @@ void TestSps(SpsMode mode, SpsVuiRewriter::ParseResult expected_parse_result) { // Ensure that added/rewritten SPS is parsable. rtc::Buffer tmp; result = SpsVuiRewriter::ParseAndRewriteSps( - rewritten_sps.data(), rewritten_sps.size(), &sps, &tmp, + rewritten_sps.data(), rewritten_sps.size(), &sps, nullptr, &tmp, SpsVuiRewriter::Direction::kIncoming); EXPECT_EQ(SpsVuiRewriter::ParseResult::kVuiOk, result); ASSERT_TRUE(sps); @@ -174,27 +332,67 @@ void TestSps(SpsMode mode, SpsVuiRewriter::ParseResult expected_parse_result) { } } -#define REWRITE_TEST(test_name, mode, expected_parse_result) \ - TEST(SpsVuiRewriterTest, test_name) { TestSps(mode, expected_parse_result); } +class SpsVuiRewriterTest : public ::testing::Test, + public ::testing::WithParamInterface< + ::testing::tuple> { +}; -REWRITE_TEST(VuiAlreadyOptimal, - kNoRewriteRequired_VuiOptimal, - SpsVuiRewriter::ParseResult::kVuiOk) -REWRITE_TEST(RewriteFullVui, - kRewriteRequired_NoVui, - SpsVuiRewriter::ParseResult::kVuiRewritten) -REWRITE_TEST(AddBitstreamRestriction, - kRewriteRequired_NoBitstreamRestriction, - SpsVuiRewriter::ParseResult::kVuiRewritten) -REWRITE_TEST(RewriteSuboptimalVui, - kRewriteRequired_VuiSuboptimal, - SpsVuiRewriter::ParseResult::kVuiRewritten) +TEST_P(SpsVuiRewriterTest, RewriteVui) { + VuiHeader vui = ::testing::get<0>(GetParam()); + const ColorSpace* color_space = ::testing::get<1>(GetParam()); + SpsVuiRewriter::ParseResult expected_parse_result = + ::testing::get<2>(GetParam()); + TestSps(vui, color_space, expected_parse_result); +} -TEST(SpsVuiRewriterTest, ParseOutgoingBitstreamOptimalVui) { +INSTANTIATE_TEST_SUITE_P( + , + SpsVuiRewriterTest, + ::testing::Values( + std::make_tuple(kVuiNoFrameBuffering, + nullptr, + SpsVuiRewriter::ParseResult::kVuiOk), + std::make_tuple(kVuiNoVideoSignalType, + &kColorSpaceH264Default, + SpsVuiRewriter::ParseResult::kVuiOk), + std::make_tuple(kVuiLimitedRangeBt709Color, + &kColorSpaceBt709LimitedRange, + SpsVuiRewriter::ParseResult::kVuiOk), + std::make_tuple(kVuiNotPresent, + nullptr, + SpsVuiRewriter::ParseResult::kVuiRewritten), + std::make_tuple(kVuiNoBitstreamRestriction, + nullptr, + SpsVuiRewriter::ParseResult::kVuiRewritten), + std::make_tuple(kVuiFrameBuffering, + nullptr, + SpsVuiRewriter::ParseResult::kVuiRewritten), + std::make_tuple(kVuiLimitedRangeNoColourDescription, + &kColorSpaceFullRange, + SpsVuiRewriter::ParseResult::kVuiRewritten), + std::make_tuple(kVuiNoVideoSignalType, + &kColorSpacePrimariesBt709, + SpsVuiRewriter::ParseResult::kVuiRewritten), + std::make_tuple(kVuiNoVideoSignalType, + &kColorSpaceTransferBt709, + SpsVuiRewriter::ParseResult::kVuiRewritten), + std::make_tuple(kVuiNoVideoSignalType, + &kColorSpaceMatrixBt709, + SpsVuiRewriter::ParseResult::kVuiRewritten), + std::make_tuple(kVuiFullRangeNoColourDescription, + &kColorSpaceH264Default, + SpsVuiRewriter::ParseResult::kVuiRewritten), + std::make_tuple(kVuiLimitedRangeBt709Color, + &kColorSpaceH264Default, + SpsVuiRewriter::ParseResult::kVuiRewritten))); + +TEST(SpsVuiRewriterOutgoingVuiTest, ParseOutgoingBitstreamOptimalVui) { rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE); rtc::Buffer optimal_sps; - GenerateFakeSps(kNoRewriteRequired_VuiOptimal, &optimal_sps); + GenerateFakeSps(kVuiNoFrameBuffering, &optimal_sps); rtc::Buffer buffer; const size_t kNumNalus = 2; @@ -214,7 +412,7 @@ TEST(SpsVuiRewriterTest, ParseOutgoingBitstreamOptimalVui) { size_t modified_nalu_lengths[kNumNalus]; SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps( - buffer, kNumNalus, nalu_offsets, nalu_lengths, &modified_buffer, + buffer, kNumNalus, nalu_offsets, nalu_lengths, nullptr, &modified_buffer, modified_nalu_offsets, modified_nalu_lengths); EXPECT_THAT( @@ -229,11 +427,11 @@ TEST(SpsVuiRewriterTest, ParseOutgoingBitstreamOptimalVui) { ::testing::ElementsAreArray(nalu_lengths, kNumNalus)); } -TEST(SpsVuiRewriterTest, ParseOutgoingBitstreamNoVui) { +TEST(SpsVuiRewriterOutgoingVuiTest, ParseOutgoingBitstreamNoVui) { rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE); rtc::Buffer sps; - GenerateFakeSps(kRewriteRequired_NoVui, &sps); + GenerateFakeSps(kVuiNotPresent, &sps); rtc::Buffer buffer; const size_t kNumNalus = 3; @@ -254,7 +452,7 @@ TEST(SpsVuiRewriterTest, ParseOutgoingBitstreamNoVui) { buffer.AppendData(kIdr2); rtc::Buffer optimal_sps; - GenerateFakeSps(kNoRewriteRequired_VuiOptimal, &optimal_sps); + GenerateFakeSps(kVuiNoFrameBuffering, &optimal_sps); rtc::Buffer expected_buffer; size_t expected_nalu_offsets[kNumNalus]; @@ -278,7 +476,7 @@ TEST(SpsVuiRewriterTest, ParseOutgoingBitstreamNoVui) { size_t modified_nalu_lengths[kNumNalus]; SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps( - buffer, kNumNalus, nalu_offsets, nalu_lengths, &modified_buffer, + buffer, kNumNalus, nalu_offsets, nalu_lengths, nullptr, &modified_buffer, modified_nalu_offsets, modified_nalu_lengths); EXPECT_THAT( diff --git a/modules/rtp_rtcp/source/rtp_format_h264.cc b/modules/rtp_rtcp/source/rtp_format_h264.cc index 0922935ed5..28cc2fb624 100644 --- a/modules/rtp_rtcp/source/rtp_format_h264.cc +++ b/modules/rtp_rtcp/source/rtp_format_h264.cc @@ -458,7 +458,7 @@ bool RtpDepacketizerH264::ProcessStapAOrSingleNalu( SpsVuiRewriter::ParseResult result = SpsVuiRewriter::ParseAndRewriteSps( &payload_data[start_offset], end_offset - start_offset, &sps, - output_buffer.get(), SpsVuiRewriter::Direction::kIncoming); + nullptr, output_buffer.get(), SpsVuiRewriter::Direction::kIncoming); if (result == SpsVuiRewriter::ParseResult::kVuiRewritten) { if (modified_buffer_) { diff --git a/video/frame_encode_metadata_writer.cc b/video/frame_encode_metadata_writer.cc index 13d63728a4..950e714026 100644 --- a/video/frame_encode_metadata_writer.cc +++ b/video/frame_encode_metadata_writer.cc @@ -207,7 +207,8 @@ FrameEncodeMetadataWriter::UpdateBitstream( SpsVuiRewriter::ParseOutgoingBitstreamAndRewriteSps( buffer, fragmentation->fragmentationVectorSize, fragmentation->fragmentationOffset, fragmentation->fragmentationLength, - &modified_buffer, modified_fragmentation->fragmentationOffset, + encoded_image->ColorSpace(), &modified_buffer, + modified_fragmentation->fragmentationOffset, modified_fragmentation->fragmentationLength); encoded_image->SetEncodedData(modified_buffer);