From c13f4be5f4070cfab6e669e35de5bf86b2b80361 Mon Sep 17 00:00:00 2001 From: Johannes Kron Date: Wed, 12 Dec 2018 09:52:53 +0100 Subject: [PATCH] Add chroma siting to color space RTP extension - Add chroma siting to color space RTP extension. - Use 16 bits for max/min luminance. - Change denominator of chromaticity and luminance. - Add RTC_DCHECKs to protect against overflows. Bug: webrtc:8651 Change-Id: If8b95bad6241381224eaba9c5bccce06a65a9195 Reviewed-on: https://webrtc-review.googlesource.com/c/113804 Commit-Queue: Johannes Kron Reviewed-by: Danil Chapovalov Reviewed-by: Niels Moller Cr-Commit-Position: refs/heads/master@{#25990} --- api/video/hdr_metadata.h | 39 +++- .../rtp_rtcp/source/rtp_header_extensions.cc | 183 +++++++++++------- .../rtp_rtcp/source/rtp_header_extensions.h | 15 +- .../rtp_rtcp/source/rtp_packet_unittest.cc | 13 +- 4 files changed, 165 insertions(+), 85 deletions(-) diff --git a/api/video/hdr_metadata.h b/api/video/hdr_metadata.h index 676a900c99..3be1d59883 100644 --- a/api/video/hdr_metadata.h +++ b/api/video/hdr_metadata.h @@ -21,14 +21,19 @@ struct HdrMasteringMetadata { struct Chromaticity { // xy chromaticity coordinates must be calculated as specified in ISO // 11664-3:2012 Section 7, and must be specified with four decimal places. - // The x coordinate must be in the range [0.0001, 0.7400] and the y - // coordinate must be in the range [0.0001, 0.8400]. + // The x coordinate should be in the range [0.0001, 0.7400] and the y + // coordinate should be in the range [0.0001, 0.8400]. Valid range [0.0000, + // 1.0000]. float x = 0.0f; float y = 0.0f; bool operator==(const Chromaticity& rhs) const { return x == rhs.x && y == rhs.y; } + bool Validate() const { + return x >= 0.0 && x <= 1.0 && y >= 0.0 && y <= 1.0; + } + Chromaticity(); }; @@ -41,13 +46,13 @@ struct HdrMasteringMetadata { Chromaticity white_point; // The nominal maximum display luminance of the mastering display. Specified - // in the unit candela/m2. The value must be in the range [5, 10000] with zero - // decimal places. + // in the unit candela/m2. The value should be in the range [5, 10000] with + // zero decimal places. Valid range [0, 20000]. float luminance_max = 0.0f; // The nominal minimum display luminance of the mastering display. Specified - // in the unit candela/m2. The value must be in the range [0.0001, 5.0000] - // with four decimal places. + // in the unit candela/m2. The value should be in the range [0.0001, 5.0000] + // with four decimal places. Valid range [0.0000, 5.0000]. float luminance_min = 0.0f; HdrMasteringMetadata(); @@ -58,6 +63,13 @@ struct HdrMasteringMetadata { (luminance_max == rhs.luminance_max) && (luminance_min == rhs.luminance_min)); } + + bool Validate() const { + return luminance_max >= 0.0 && luminance_max <= 20000.0 && + luminance_min >= 0.0 && luminance_min <= 5.0 && + primary_r.Validate() && primary_g.Validate() && + primary_b.Validate() && white_point.Validate(); + } }; // High dynamic range (HDR) metadata common for HDR10 and WebM/VP9-based HDR @@ -66,11 +78,11 @@ struct HdrMasteringMetadata { struct HdrMetadata { HdrMasteringMetadata mastering_metadata; // Max content light level (CLL), i.e. maximum brightness level present in the - // stream, in nits. 1 nit = 1 candela/m2. - uint32_t max_content_light_level = 0; + // stream, in nits. 1 nit = 1 candela/m2. Valid range [0, 20000]. + int max_content_light_level = 0; // Max frame-average light level (FALL), i.e. maximum average brightness of - // the brightest frame in the stream, in nits. - uint32_t max_frame_average_light_level = 0; + // the brightest frame in the stream, in nits. Valid range [0, 20000]. + int max_frame_average_light_level = 0; HdrMetadata(); @@ -80,6 +92,13 @@ struct HdrMetadata { (max_frame_average_light_level == rhs.max_frame_average_light_level) && (mastering_metadata == rhs.mastering_metadata)); } + + bool Validate() const { + return max_content_light_level >= 0 && max_content_light_level <= 20000 && + max_frame_average_light_level >= 0 && + max_frame_average_light_level <= 20000 && + mastering_metadata.Validate(); + } }; } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.cc b/modules/rtp_rtcp/source/rtp_header_extensions.cc index 171109237b..38c2d5ac7d 100644 --- a/modules/rtp_rtcp/source/rtp_header_extensions.cc +++ b/modules/rtp_rtcp/source/rtp_header_extensions.cc @@ -12,6 +12,7 @@ #include #include +#include #include "modules/rtp_rtcp/include/rtp_cvo.h" #include "modules/rtp_rtcp/source/byte_io.h" @@ -444,30 +445,30 @@ bool FrameMarkingExtension::Write(rtc::ArrayView data, // 0 1 2 3 // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | ID | length=30 | Primaries | Transfer | +// | ID | length=28 | primaries | transfer | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | Matrix | Range | luminance_max | +// | matrix |range+chr.sit. | luminance_max | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | | luminance_min | +// | luminance_min | mastering_metadata.| // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | mastering_metadata.primary_r.x and .y | +// |primary_r.x and .y | mastering_metadata.| // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | mastering_metadata.primary_g.x and .y | +// |primary_g.x and .y | mastering_metadata.| // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | mastering_metadata.primary_b.x and .y | +// |primary_b.x and .y | mastering_metadata.| // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | mastering_metadata.white.x and .y | -// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | max_content_light_level | max_frame_average_light_level | +// |white.x and .y | max_content_light_level | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | max_frame_average_light_level | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ // // Data layout of color space w/o HDR metadata (one-byte RTP header extension) // 0 1 2 3 // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | ID | L = 3 | Primaries | Transfer | Matrix | +// | ID | L = 3 | primaries | transfer | matrix | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -// | Range | +// |range+chr.sit. | // +-+-+-+-+-+-+-+-+ constexpr RTPExtensionType ColorSpaceExtension::kId; @@ -483,13 +484,21 @@ bool ColorSpaceExtension::Parse(rtc::ArrayView data, size_t offset = 0; // Read color space information. - if (!color_space->set_primaries_from_uint8(data.data()[offset++])) + if (!color_space->set_primaries_from_uint8(data[offset++])) return false; - if (!color_space->set_transfer_from_uint8(data.data()[offset++])) + if (!color_space->set_transfer_from_uint8(data[offset++])) return false; - if (!color_space->set_matrix_from_uint8(data.data()[offset++])) + if (!color_space->set_matrix_from_uint8(data[offset++])) return false; - if (!color_space->set_range_from_uint8(data.data()[offset++])) + + uint8_t range_and_chroma_siting = data[offset++]; + if (!color_space->set_range_from_uint8((range_and_chroma_siting >> 4) & 0x03)) + return false; + if (!color_space->set_chroma_siting_horizontal_from_uint8( + (range_and_chroma_siting >> 2) & 0x03)) + return false; + if (!color_space->set_chroma_siting_vertical_from_uint8( + range_and_chroma_siting & 0x03)) return false; // Read HDR metadata if it exists, otherwise clear it. @@ -497,26 +506,9 @@ bool ColorSpaceExtension::Parse(rtc::ArrayView data, color_space->set_hdr_metadata(nullptr); } else { HdrMetadata hdr_metadata; - offset += ParseLuminance(data.data() + offset, - &hdr_metadata.mastering_metadata.luminance_max, - kLuminanceMaxDenominator); - offset += ParseLuminance(data.data() + offset, - &hdr_metadata.mastering_metadata.luminance_min, - kLuminanceMinDenominator); - offset += ParseChromaticity(data.data() + offset, - &hdr_metadata.mastering_metadata.primary_r); - offset += ParseChromaticity(data.data() + offset, - &hdr_metadata.mastering_metadata.primary_g); - offset += ParseChromaticity(data.data() + offset, - &hdr_metadata.mastering_metadata.primary_b); - offset += ParseChromaticity(data.data() + offset, - &hdr_metadata.mastering_metadata.white_point); - hdr_metadata.max_content_light_level = - ByteReader::ReadBigEndian(data.data() + offset); - offset += 2; - hdr_metadata.max_frame_average_light_level = - ByteReader::ReadBigEndian(data.data() + offset); - offset += 2; + offset += ParseHdrMetadata(data.subview(offset), &hdr_metadata); + if (!hdr_metadata.Validate()) + return false; color_space->set_hdr_metadata(&hdr_metadata); } RTC_DCHECK_EQ(ValueSize(*color_space), offset); @@ -528,40 +520,67 @@ bool ColorSpaceExtension::Write(rtc::ArrayView data, RTC_DCHECK_EQ(data.size(), ValueSize(color_space)); size_t offset = 0; // Write color space information. - data.data()[offset++] = static_cast(color_space.primaries()); - data.data()[offset++] = static_cast(color_space.transfer()); - data.data()[offset++] = static_cast(color_space.matrix()); - data.data()[offset++] = static_cast(color_space.range()); + data[offset++] = static_cast(color_space.primaries()); + data[offset++] = static_cast(color_space.transfer()); + data[offset++] = static_cast(color_space.matrix()); + data[offset++] = CombineRangeAndChromaSiting( + color_space.range(), color_space.chroma_siting_horizontal(), + color_space.chroma_siting_vertical()); // Write HDR metadata if it exists. if (color_space.hdr_metadata()) { - const HdrMetadata& hdr_metadata = *color_space.hdr_metadata(); - offset += WriteLuminance(data.data() + offset, - hdr_metadata.mastering_metadata.luminance_max, - kLuminanceMaxDenominator); - offset += WriteLuminance(data.data() + offset, - hdr_metadata.mastering_metadata.luminance_min, - kLuminanceMinDenominator); - offset += WriteChromaticity(data.data() + offset, - hdr_metadata.mastering_metadata.primary_r); - offset += WriteChromaticity(data.data() + offset, - hdr_metadata.mastering_metadata.primary_g); - offset += WriteChromaticity(data.data() + offset, - hdr_metadata.mastering_metadata.primary_b); - offset += WriteChromaticity(data.data() + offset, - hdr_metadata.mastering_metadata.white_point); - - ByteWriter::WriteBigEndian(data.data() + offset, - hdr_metadata.max_content_light_level); - offset += 2; - ByteWriter::WriteBigEndian( - data.data() + offset, hdr_metadata.max_frame_average_light_level); - offset += 2; + offset += + WriteHdrMetadata(data.subview(offset), *color_space.hdr_metadata()); } RTC_DCHECK_EQ(ValueSize(color_space), offset); return true; } +// Combines range and chroma siting into one byte with the following bit layout: +// bits 0-1 Chroma siting vertical. +// 2-3 Chroma siting horizontal. +// 4-5 Range. +// 6-7 Unused. +uint8_t ColorSpaceExtension::CombineRangeAndChromaSiting( + ColorSpace::RangeID range, + ColorSpace::ChromaSiting chroma_siting_horizontal, + ColorSpace::ChromaSiting chroma_siting_vertical) { + RTC_DCHECK_LE(static_cast(range), 3); + RTC_DCHECK_LE(static_cast(chroma_siting_horizontal), 3); + RTC_DCHECK_LE(static_cast(chroma_siting_vertical), 3); + return (static_cast(range) << 4) | + (static_cast(chroma_siting_horizontal) << 2) | + static_cast(chroma_siting_vertical); +} + +size_t ColorSpaceExtension::ParseHdrMetadata(rtc::ArrayView data, + HdrMetadata* hdr_metadata) { + RTC_DCHECK_EQ(data.size(), + kValueSizeBytes - kValueSizeBytesWithoutHdrMetadata); + size_t offset = 0; + offset += ParseLuminance(data.data() + offset, + &hdr_metadata->mastering_metadata.luminance_max, + kLuminanceMaxDenominator); + offset += ParseLuminance(data.data() + offset, + &hdr_metadata->mastering_metadata.luminance_min, + kLuminanceMinDenominator); + offset += ParseChromaticity(data.data() + offset, + &hdr_metadata->mastering_metadata.primary_r); + offset += ParseChromaticity(data.data() + offset, + &hdr_metadata->mastering_metadata.primary_g); + offset += ParseChromaticity(data.data() + offset, + &hdr_metadata->mastering_metadata.primary_b); + offset += ParseChromaticity(data.data() + offset, + &hdr_metadata->mastering_metadata.white_point); + hdr_metadata->max_content_light_level = + ByteReader::ReadBigEndian(data.data() + offset); + offset += 2; + hdr_metadata->max_frame_average_light_level = + ByteReader::ReadBigEndian(data.data() + offset); + offset += 2; + return offset; +} + size_t ColorSpaceExtension::ParseChromaticity( const uint8_t* data, HdrMasteringMetadata::Chromaticity* p) { @@ -576,16 +595,48 @@ size_t ColorSpaceExtension::ParseChromaticity( size_t ColorSpaceExtension::ParseLuminance(const uint8_t* data, float* f, int denominator) { - uint32_t luminance_scaled = ByteReader::ReadBigEndian(data); + uint16_t luminance_scaled = ByteReader::ReadBigEndian(data); *f = static_cast(luminance_scaled) / denominator; - return 3; // Return number of bytes read. + return 2; // Return number of bytes read. +} + +size_t ColorSpaceExtension::WriteHdrMetadata(rtc::ArrayView data, + const HdrMetadata& hdr_metadata) { + RTC_DCHECK_EQ(data.size(), + kValueSizeBytes - kValueSizeBytesWithoutHdrMetadata); + RTC_DCHECK(hdr_metadata.Validate()); + size_t offset = 0; + offset += WriteLuminance(data.data() + offset, + hdr_metadata.mastering_metadata.luminance_max, + kLuminanceMaxDenominator); + offset += WriteLuminance(data.data() + offset, + hdr_metadata.mastering_metadata.luminance_min, + kLuminanceMinDenominator); + offset += WriteChromaticity(data.data() + offset, + hdr_metadata.mastering_metadata.primary_r); + offset += WriteChromaticity(data.data() + offset, + hdr_metadata.mastering_metadata.primary_g); + offset += WriteChromaticity(data.data() + offset, + hdr_metadata.mastering_metadata.primary_b); + offset += WriteChromaticity(data.data() + offset, + hdr_metadata.mastering_metadata.white_point); + + ByteWriter::WriteBigEndian(data.data() + offset, + hdr_metadata.max_content_light_level); + offset += 2; + ByteWriter::WriteBigEndian( + data.data() + offset, hdr_metadata.max_frame_average_light_level); + offset += 2; + return offset; } size_t ColorSpaceExtension::WriteChromaticity( uint8_t* data, const HdrMasteringMetadata::Chromaticity& p) { RTC_DCHECK_GE(p.x, 0.0f); + RTC_DCHECK_LE(p.x, 1.0f); RTC_DCHECK_GE(p.y, 0.0f); + RTC_DCHECK_LE(p.y, 1.0f); ByteWriter::WriteBigEndian( data, std::round(p.x * kChromaticityDenominator)); ByteWriter::WriteBigEndian( @@ -597,8 +648,10 @@ size_t ColorSpaceExtension::WriteLuminance(uint8_t* data, float f, int denominator) { RTC_DCHECK_GE(f, 0.0f); - ByteWriter::WriteBigEndian(data, std::round(f * denominator)); - return 3; // Return number of bytes written. + float upscaled_value = f * denominator; + RTC_DCHECK_LE(upscaled_value, std::numeric_limits::max()); + ByteWriter::WriteBigEndian(data, std::round(upscaled_value)); + return 2; // Return number of bytes written. } bool BaseRtpStringExtension::Parse(rtc::ArrayView data, diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.h b/modules/rtp_rtcp/source/rtp_header_extensions.h index c1eaf8c92e..9f4a28b701 100644 --- a/modules/rtp_rtcp/source/rtp_header_extensions.h +++ b/modules/rtp_rtcp/source/rtp_header_extensions.h @@ -186,7 +186,7 @@ class ColorSpaceExtension { public: using value_type = ColorSpace; static constexpr RTPExtensionType kId = kRtpExtensionColorSpace; - static constexpr uint8_t kValueSizeBytes = 30; + static constexpr uint8_t kValueSizeBytes = 28; static constexpr uint8_t kValueSizeBytesWithoutHdrMetadata = 4; static constexpr const char kUri[] = "http://www.webrtc.org/experiments/rtp-hdrext/color-space"; @@ -201,12 +201,21 @@ class ColorSpaceExtension { const ColorSpace& color_space); private: - static constexpr int kChromaticityDenominator = 10000; // 0.0001 resolution. - static constexpr int kLuminanceMaxDenominator = 100; // 0.01 resolution. + static constexpr int kChromaticityDenominator = 50000; // 0.00002 resolution. + static constexpr int kLuminanceMaxDenominator = 1; // 1 resolution. static constexpr int kLuminanceMinDenominator = 10000; // 0.0001 resolution. + + static uint8_t CombineRangeAndChromaSiting( + ColorSpace::RangeID range, + ColorSpace::ChromaSiting chroma_siting_horizontal, + ColorSpace::ChromaSiting chroma_siting_vertical); + static size_t ParseHdrMetadata(rtc::ArrayView data, + HdrMetadata* hdr_metadata); static size_t ParseChromaticity(const uint8_t* data, HdrMasteringMetadata::Chromaticity* p); static size_t ParseLuminance(const uint8_t* data, float* f, int denominator); + static size_t WriteHdrMetadata(rtc::ArrayView data, + const HdrMetadata& hdr_metadata); static size_t WriteChromaticity(uint8_t* data, const HdrMasteringMetadata::Chromaticity& p); static size_t WriteLuminance(uint8_t* data, float f, int denominator); diff --git a/modules/rtp_rtcp/source/rtp_packet_unittest.cc b/modules/rtp_rtcp/source/rtp_packet_unittest.cc index b1c0e42525..e53125b4aa 100644 --- a/modules/rtp_rtcp/source/rtp_packet_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_packet_unittest.cc @@ -205,14 +205,13 @@ HdrMetadata CreateTestHdrMetadata() { } ColorSpace CreateTestColorSpace(bool with_hdr_metadata) { - ColorSpace color_space( + HdrMetadata hdr_metadata = CreateTestHdrMetadata(); + return ColorSpace( ColorSpace::PrimaryID::kBT709, ColorSpace::TransferID::kGAMMA22, - ColorSpace::MatrixID::kSMPTE2085, ColorSpace::RangeID::kFull); - if (with_hdr_metadata) { - HdrMetadata hdr_metadata = CreateTestHdrMetadata(); - color_space.set_hdr_metadata(&hdr_metadata); - } - return color_space; + ColorSpace::MatrixID::kSMPTE2085, ColorSpace::RangeID::kFull, + ColorSpace::ChromaSiting::kCollocated, + ColorSpace::ChromaSiting::kCollocated, + with_hdr_metadata ? &hdr_metadata : nullptr); } void TestCreateAndParseColorSpaceExtension(bool with_hdr_metadata) {