Add chroma siting to color space RTP extension

- Add chroma siting to color space RTP extension.
- Use 16 bits for max/min luminance.
- Change denominator of chromaticity and luminance.
- Add RTC_DCHECKs to protect against overflows.

Bug: webrtc:8651
Change-Id: If8b95bad6241381224eaba9c5bccce06a65a9195
Reviewed-on: https://webrtc-review.googlesource.com/c/113804
Commit-Queue: Johannes Kron <kron@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25990}
This commit is contained in:
Johannes Kron
2018-12-12 09:52:53 +01:00
committed by Commit Bot
parent 0697ce2a76
commit c13f4be5f4
4 changed files with 165 additions and 85 deletions

View File

@ -21,14 +21,19 @@ struct HdrMasteringMetadata {
struct Chromaticity {
// xy chromaticity coordinates must be calculated as specified in ISO
// 11664-3:2012 Section 7, and must be specified with four decimal places.
// The x coordinate must be in the range [0.0001, 0.7400] and the y
// coordinate must be in the range [0.0001, 0.8400].
// The x coordinate should be in the range [0.0001, 0.7400] and the y
// coordinate should be in the range [0.0001, 0.8400]. Valid range [0.0000,
// 1.0000].
float x = 0.0f;
float y = 0.0f;
bool operator==(const Chromaticity& rhs) const {
return x == rhs.x && y == rhs.y;
}
bool Validate() const {
return x >= 0.0 && x <= 1.0 && y >= 0.0 && y <= 1.0;
}
Chromaticity();
};
@ -41,13 +46,13 @@ struct HdrMasteringMetadata {
Chromaticity white_point;
// The nominal maximum display luminance of the mastering display. Specified
// in the unit candela/m2. The value must be in the range [5, 10000] with zero
// decimal places.
// in the unit candela/m2. The value should be in the range [5, 10000] with
// zero decimal places. Valid range [0, 20000].
float luminance_max = 0.0f;
// The nominal minimum display luminance of the mastering display. Specified
// in the unit candela/m2. The value must be in the range [0.0001, 5.0000]
// with four decimal places.
// in the unit candela/m2. The value should be in the range [0.0001, 5.0000]
// with four decimal places. Valid range [0.0000, 5.0000].
float luminance_min = 0.0f;
HdrMasteringMetadata();
@ -58,6 +63,13 @@ struct HdrMasteringMetadata {
(luminance_max == rhs.luminance_max) &&
(luminance_min == rhs.luminance_min));
}
bool Validate() const {
return luminance_max >= 0.0 && luminance_max <= 20000.0 &&
luminance_min >= 0.0 && luminance_min <= 5.0 &&
primary_r.Validate() && primary_g.Validate() &&
primary_b.Validate() && white_point.Validate();
}
};
// High dynamic range (HDR) metadata common for HDR10 and WebM/VP9-based HDR
@ -66,11 +78,11 @@ struct HdrMasteringMetadata {
struct HdrMetadata {
HdrMasteringMetadata mastering_metadata;
// Max content light level (CLL), i.e. maximum brightness level present in the
// stream, in nits. 1 nit = 1 candela/m2.
uint32_t max_content_light_level = 0;
// stream, in nits. 1 nit = 1 candela/m2. Valid range [0, 20000].
int max_content_light_level = 0;
// Max frame-average light level (FALL), i.e. maximum average brightness of
// the brightest frame in the stream, in nits.
uint32_t max_frame_average_light_level = 0;
// the brightest frame in the stream, in nits. Valid range [0, 20000].
int max_frame_average_light_level = 0;
HdrMetadata();
@ -80,6 +92,13 @@ struct HdrMetadata {
(max_frame_average_light_level == rhs.max_frame_average_light_level) &&
(mastering_metadata == rhs.mastering_metadata));
}
bool Validate() const {
return max_content_light_level >= 0 && max_content_light_level <= 20000 &&
max_frame_average_light_level >= 0 &&
max_frame_average_light_level <= 20000 &&
mastering_metadata.Validate();
}
};
} // namespace webrtc

View File

@ -12,6 +12,7 @@
#include <string.h>
#include <cmath>
#include <limits>
#include "modules/rtp_rtcp/include/rtp_cvo.h"
#include "modules/rtp_rtcp/source/byte_io.h"
@ -444,30 +445,30 @@ bool FrameMarkingExtension::Write(rtc::ArrayView<uint8_t> data,
// 0 1 2 3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | ID | length=30 | Primaries | Transfer |
// | ID | length=28 | primaries | transfer |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | Matrix | Range | luminance_max |
// | matrix |range+chr.sit. | luminance_max |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | | luminance_min |
// | luminance_min | mastering_metadata.|
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | mastering_metadata.primary_r.x and .y |
// |primary_r.x and .y | mastering_metadata.|
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | mastering_metadata.primary_g.x and .y |
// |primary_g.x and .y | mastering_metadata.|
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | mastering_metadata.primary_b.x and .y |
// |primary_b.x and .y | mastering_metadata.|
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | mastering_metadata.white.x and .y |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | max_content_light_level | max_frame_average_light_level |
// |white.x and .y | max_content_light_level |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | max_frame_average_light_level |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
//
// Data layout of color space w/o HDR metadata (one-byte RTP header extension)
// 0 1 2 3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | ID | L = 3 | Primaries | Transfer | Matrix |
// | ID | L = 3 | primaries | transfer | matrix |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | Range |
// |range+chr.sit. |
// +-+-+-+-+-+-+-+-+
constexpr RTPExtensionType ColorSpaceExtension::kId;
@ -483,13 +484,21 @@ bool ColorSpaceExtension::Parse(rtc::ArrayView<const uint8_t> data,
size_t offset = 0;
// Read color space information.
if (!color_space->set_primaries_from_uint8(data.data()[offset++]))
if (!color_space->set_primaries_from_uint8(data[offset++]))
return false;
if (!color_space->set_transfer_from_uint8(data.data()[offset++]))
if (!color_space->set_transfer_from_uint8(data[offset++]))
return false;
if (!color_space->set_matrix_from_uint8(data.data()[offset++]))
if (!color_space->set_matrix_from_uint8(data[offset++]))
return false;
if (!color_space->set_range_from_uint8(data.data()[offset++]))
uint8_t range_and_chroma_siting = data[offset++];
if (!color_space->set_range_from_uint8((range_and_chroma_siting >> 4) & 0x03))
return false;
if (!color_space->set_chroma_siting_horizontal_from_uint8(
(range_and_chroma_siting >> 2) & 0x03))
return false;
if (!color_space->set_chroma_siting_vertical_from_uint8(
range_and_chroma_siting & 0x03))
return false;
// Read HDR metadata if it exists, otherwise clear it.
@ -497,26 +506,9 @@ bool ColorSpaceExtension::Parse(rtc::ArrayView<const uint8_t> data,
color_space->set_hdr_metadata(nullptr);
} else {
HdrMetadata hdr_metadata;
offset += ParseLuminance(data.data() + offset,
&hdr_metadata.mastering_metadata.luminance_max,
kLuminanceMaxDenominator);
offset += ParseLuminance(data.data() + offset,
&hdr_metadata.mastering_metadata.luminance_min,
kLuminanceMinDenominator);
offset += ParseChromaticity(data.data() + offset,
&hdr_metadata.mastering_metadata.primary_r);
offset += ParseChromaticity(data.data() + offset,
&hdr_metadata.mastering_metadata.primary_g);
offset += ParseChromaticity(data.data() + offset,
&hdr_metadata.mastering_metadata.primary_b);
offset += ParseChromaticity(data.data() + offset,
&hdr_metadata.mastering_metadata.white_point);
hdr_metadata.max_content_light_level =
ByteReader<uint16_t>::ReadBigEndian(data.data() + offset);
offset += 2;
hdr_metadata.max_frame_average_light_level =
ByteReader<uint16_t>::ReadBigEndian(data.data() + offset);
offset += 2;
offset += ParseHdrMetadata(data.subview(offset), &hdr_metadata);
if (!hdr_metadata.Validate())
return false;
color_space->set_hdr_metadata(&hdr_metadata);
}
RTC_DCHECK_EQ(ValueSize(*color_space), offset);
@ -528,40 +520,67 @@ bool ColorSpaceExtension::Write(rtc::ArrayView<uint8_t> data,
RTC_DCHECK_EQ(data.size(), ValueSize(color_space));
size_t offset = 0;
// Write color space information.
data.data()[offset++] = static_cast<uint8_t>(color_space.primaries());
data.data()[offset++] = static_cast<uint8_t>(color_space.transfer());
data.data()[offset++] = static_cast<uint8_t>(color_space.matrix());
data.data()[offset++] = static_cast<uint8_t>(color_space.range());
data[offset++] = static_cast<uint8_t>(color_space.primaries());
data[offset++] = static_cast<uint8_t>(color_space.transfer());
data[offset++] = static_cast<uint8_t>(color_space.matrix());
data[offset++] = CombineRangeAndChromaSiting(
color_space.range(), color_space.chroma_siting_horizontal(),
color_space.chroma_siting_vertical());
// Write HDR metadata if it exists.
if (color_space.hdr_metadata()) {
const HdrMetadata& hdr_metadata = *color_space.hdr_metadata();
offset += WriteLuminance(data.data() + offset,
hdr_metadata.mastering_metadata.luminance_max,
kLuminanceMaxDenominator);
offset += WriteLuminance(data.data() + offset,
hdr_metadata.mastering_metadata.luminance_min,
kLuminanceMinDenominator);
offset += WriteChromaticity(data.data() + offset,
hdr_metadata.mastering_metadata.primary_r);
offset += WriteChromaticity(data.data() + offset,
hdr_metadata.mastering_metadata.primary_g);
offset += WriteChromaticity(data.data() + offset,
hdr_metadata.mastering_metadata.primary_b);
offset += WriteChromaticity(data.data() + offset,
hdr_metadata.mastering_metadata.white_point);
ByteWriter<uint16_t>::WriteBigEndian(data.data() + offset,
hdr_metadata.max_content_light_level);
offset += 2;
ByteWriter<uint16_t>::WriteBigEndian(
data.data() + offset, hdr_metadata.max_frame_average_light_level);
offset += 2;
offset +=
WriteHdrMetadata(data.subview(offset), *color_space.hdr_metadata());
}
RTC_DCHECK_EQ(ValueSize(color_space), offset);
return true;
}
// Combines range and chroma siting into one byte with the following bit layout:
// bits 0-1 Chroma siting vertical.
// 2-3 Chroma siting horizontal.
// 4-5 Range.
// 6-7 Unused.
uint8_t ColorSpaceExtension::CombineRangeAndChromaSiting(
ColorSpace::RangeID range,
ColorSpace::ChromaSiting chroma_siting_horizontal,
ColorSpace::ChromaSiting chroma_siting_vertical) {
RTC_DCHECK_LE(static_cast<uint8_t>(range), 3);
RTC_DCHECK_LE(static_cast<uint8_t>(chroma_siting_horizontal), 3);
RTC_DCHECK_LE(static_cast<uint8_t>(chroma_siting_vertical), 3);
return (static_cast<uint8_t>(range) << 4) |
(static_cast<uint8_t>(chroma_siting_horizontal) << 2) |
static_cast<uint8_t>(chroma_siting_vertical);
}
size_t ColorSpaceExtension::ParseHdrMetadata(rtc::ArrayView<const uint8_t> data,
HdrMetadata* hdr_metadata) {
RTC_DCHECK_EQ(data.size(),
kValueSizeBytes - kValueSizeBytesWithoutHdrMetadata);
size_t offset = 0;
offset += ParseLuminance(data.data() + offset,
&hdr_metadata->mastering_metadata.luminance_max,
kLuminanceMaxDenominator);
offset += ParseLuminance(data.data() + offset,
&hdr_metadata->mastering_metadata.luminance_min,
kLuminanceMinDenominator);
offset += ParseChromaticity(data.data() + offset,
&hdr_metadata->mastering_metadata.primary_r);
offset += ParseChromaticity(data.data() + offset,
&hdr_metadata->mastering_metadata.primary_g);
offset += ParseChromaticity(data.data() + offset,
&hdr_metadata->mastering_metadata.primary_b);
offset += ParseChromaticity(data.data() + offset,
&hdr_metadata->mastering_metadata.white_point);
hdr_metadata->max_content_light_level =
ByteReader<uint16_t>::ReadBigEndian(data.data() + offset);
offset += 2;
hdr_metadata->max_frame_average_light_level =
ByteReader<uint16_t>::ReadBigEndian(data.data() + offset);
offset += 2;
return offset;
}
size_t ColorSpaceExtension::ParseChromaticity(
const uint8_t* data,
HdrMasteringMetadata::Chromaticity* p) {
@ -576,16 +595,48 @@ size_t ColorSpaceExtension::ParseChromaticity(
size_t ColorSpaceExtension::ParseLuminance(const uint8_t* data,
float* f,
int denominator) {
uint32_t luminance_scaled = ByteReader<uint32_t, 3>::ReadBigEndian(data);
uint16_t luminance_scaled = ByteReader<uint16_t>::ReadBigEndian(data);
*f = static_cast<float>(luminance_scaled) / denominator;
return 3; // Return number of bytes read.
return 2; // Return number of bytes read.
}
size_t ColorSpaceExtension::WriteHdrMetadata(rtc::ArrayView<uint8_t> data,
const HdrMetadata& hdr_metadata) {
RTC_DCHECK_EQ(data.size(),
kValueSizeBytes - kValueSizeBytesWithoutHdrMetadata);
RTC_DCHECK(hdr_metadata.Validate());
size_t offset = 0;
offset += WriteLuminance(data.data() + offset,
hdr_metadata.mastering_metadata.luminance_max,
kLuminanceMaxDenominator);
offset += WriteLuminance(data.data() + offset,
hdr_metadata.mastering_metadata.luminance_min,
kLuminanceMinDenominator);
offset += WriteChromaticity(data.data() + offset,
hdr_metadata.mastering_metadata.primary_r);
offset += WriteChromaticity(data.data() + offset,
hdr_metadata.mastering_metadata.primary_g);
offset += WriteChromaticity(data.data() + offset,
hdr_metadata.mastering_metadata.primary_b);
offset += WriteChromaticity(data.data() + offset,
hdr_metadata.mastering_metadata.white_point);
ByteWriter<uint16_t>::WriteBigEndian(data.data() + offset,
hdr_metadata.max_content_light_level);
offset += 2;
ByteWriter<uint16_t>::WriteBigEndian(
data.data() + offset, hdr_metadata.max_frame_average_light_level);
offset += 2;
return offset;
}
size_t ColorSpaceExtension::WriteChromaticity(
uint8_t* data,
const HdrMasteringMetadata::Chromaticity& p) {
RTC_DCHECK_GE(p.x, 0.0f);
RTC_DCHECK_LE(p.x, 1.0f);
RTC_DCHECK_GE(p.y, 0.0f);
RTC_DCHECK_LE(p.y, 1.0f);
ByteWriter<uint16_t>::WriteBigEndian(
data, std::round(p.x * kChromaticityDenominator));
ByteWriter<uint16_t>::WriteBigEndian(
@ -597,8 +648,10 @@ size_t ColorSpaceExtension::WriteLuminance(uint8_t* data,
float f,
int denominator) {
RTC_DCHECK_GE(f, 0.0f);
ByteWriter<uint32_t, 3>::WriteBigEndian(data, std::round(f * denominator));
return 3; // Return number of bytes written.
float upscaled_value = f * denominator;
RTC_DCHECK_LE(upscaled_value, std::numeric_limits<uint16_t>::max());
ByteWriter<uint16_t>::WriteBigEndian(data, std::round(upscaled_value));
return 2; // Return number of bytes written.
}
bool BaseRtpStringExtension::Parse(rtc::ArrayView<const uint8_t> data,

View File

@ -186,7 +186,7 @@ class ColorSpaceExtension {
public:
using value_type = ColorSpace;
static constexpr RTPExtensionType kId = kRtpExtensionColorSpace;
static constexpr uint8_t kValueSizeBytes = 30;
static constexpr uint8_t kValueSizeBytes = 28;
static constexpr uint8_t kValueSizeBytesWithoutHdrMetadata = 4;
static constexpr const char kUri[] =
"http://www.webrtc.org/experiments/rtp-hdrext/color-space";
@ -201,12 +201,21 @@ class ColorSpaceExtension {
const ColorSpace& color_space);
private:
static constexpr int kChromaticityDenominator = 10000; // 0.0001 resolution.
static constexpr int kLuminanceMaxDenominator = 100; // 0.01 resolution.
static constexpr int kChromaticityDenominator = 50000; // 0.00002 resolution.
static constexpr int kLuminanceMaxDenominator = 1; // 1 resolution.
static constexpr int kLuminanceMinDenominator = 10000; // 0.0001 resolution.
static uint8_t CombineRangeAndChromaSiting(
ColorSpace::RangeID range,
ColorSpace::ChromaSiting chroma_siting_horizontal,
ColorSpace::ChromaSiting chroma_siting_vertical);
static size_t ParseHdrMetadata(rtc::ArrayView<const uint8_t> data,
HdrMetadata* hdr_metadata);
static size_t ParseChromaticity(const uint8_t* data,
HdrMasteringMetadata::Chromaticity* p);
static size_t ParseLuminance(const uint8_t* data, float* f, int denominator);
static size_t WriteHdrMetadata(rtc::ArrayView<uint8_t> data,
const HdrMetadata& hdr_metadata);
static size_t WriteChromaticity(uint8_t* data,
const HdrMasteringMetadata::Chromaticity& p);
static size_t WriteLuminance(uint8_t* data, float f, int denominator);

View File

@ -205,14 +205,13 @@ HdrMetadata CreateTestHdrMetadata() {
}
ColorSpace CreateTestColorSpace(bool with_hdr_metadata) {
ColorSpace color_space(
HdrMetadata hdr_metadata = CreateTestHdrMetadata();
return ColorSpace(
ColorSpace::PrimaryID::kBT709, ColorSpace::TransferID::kGAMMA22,
ColorSpace::MatrixID::kSMPTE2085, ColorSpace::RangeID::kFull);
if (with_hdr_metadata) {
HdrMetadata hdr_metadata = CreateTestHdrMetadata();
color_space.set_hdr_metadata(&hdr_metadata);
}
return color_space;
ColorSpace::MatrixID::kSMPTE2085, ColorSpace::RangeID::kFull,
ColorSpace::ChromaSiting::kCollocated,
ColorSpace::ChromaSiting::kCollocated,
with_hdr_metadata ? &hdr_metadata : nullptr);
}
void TestCreateAndParseColorSpaceExtension(bool with_hdr_metadata) {