diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index 454e480ca1..d767048cc0 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -11,6 +11,8 @@ import("../../webrtc.gni") rtc_source_set("video_frame") { visibility = [ "*" ] sources = [ + "color_space.cc", + "color_space.h", "video_content_type.cc", "video_content_type.h", "video_frame.cc", diff --git a/api/video/color_space.cc b/api/video/color_space.cc new file mode 100644 index 0000000000..a8be5cd828 --- /dev/null +++ b/api/video/color_space.cc @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/color_space.h" + +namespace webrtc { + +ColorSpace::ColorSpace() = default; + +ColorSpace::ColorSpace(PrimaryID primaries, + TransferID transfer, + MatrixID matrix, + RangeID range) + : primaries_(primaries), + transfer_(transfer), + matrix_(matrix), + range_(range) {} + +ColorSpace::PrimaryID ColorSpace::primaries() const { + return primaries_; +} + +ColorSpace::TransferID ColorSpace::transfer() const { + return transfer_; +} + +ColorSpace::MatrixID ColorSpace::matrix() const { + return matrix_; +} + +ColorSpace::RangeID ColorSpace::range() const { + return range_; +} + +} // namespace webrtc diff --git a/api/video/color_space.h b/api/video/color_space.h new file mode 100644 index 0000000000..736a170d06 --- /dev/null +++ b/api/video/color_space.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_COLOR_SPACE_H_ +#define API_VIDEO_COLOR_SPACE_H_ + +namespace webrtc { + +// Used to represent a color space for the purpose of color conversion. This +// class only represents color information that can be transferred through the +// bitstream of WebRTC's internal supported codecs: +// - VP9 supports color profiles, see VP9 Bitstream & Decoding Process +// Specification Version 0.6 Section 7.2.2 "Color config semantics" available +// from https://www.webmproject.org. +// TODO(emircan): Extract these values from decode and add to the existing ones. +// - VP8 only supports BT.601, see +// https://tools.ietf.org/html/rfc6386#section-9.2 +// - H264 supports different color primaries, transfer characteristics, matrix +// coefficients and range. See T-REC-H.264 E.2.1, "VUI parameters semantics", +// available from https://www.itu.int/rec/T-REC-H.264. +class ColorSpace { + public: + enum class PrimaryID { + kInvalid, + kBT709, + kSMPTE170M, // Identical to BT601 + kSMPTE240M, + kBT2020, + }; + + enum class TransferID { + kInvalid, + kBT709, + kSMPTE170M, + kSMPTE240M, + kBT2020, + kBT2020_10, + kIEC61966_2_1, + }; + + enum class MatrixID { + kInvalid, + kBT709, + kSMPTE170M, + kSMPTE240M, + kBT2020_NCL, + }; + + enum class RangeID { + kInvalid, + // Limited Rec. 709 color range with RGB values ranging from 16 to 235. + kLimited, + // Full RGB color range with RGB valees from 0 to 255. + kFull, + }; + + ColorSpace(); + ColorSpace(PrimaryID primaries, + TransferID transfer, + MatrixID matrix, + RangeID full_range); + + PrimaryID primaries() const; + TransferID transfer() const; + MatrixID matrix() const; + RangeID range() const; + + private: + PrimaryID primaries_ = PrimaryID::kInvalid; + TransferID transfer_ = TransferID::kInvalid; + MatrixID matrix_ = MatrixID::kInvalid; + RangeID range_ = RangeID::kInvalid; +}; + +} // namespace webrtc + +#endif // API_VIDEO_COLOR_SPACE_H_ diff --git a/api/video/video_frame.cc b/api/video/video_frame.cc index e8fb29d742..5521ad8a58 100644 --- a/api/video/video_frame.cc +++ b/api/video/video_frame.cc @@ -15,6 +15,55 @@ namespace webrtc { +VideoFrame::Builder::Builder() = default; + +VideoFrame::Builder::~Builder() = default; + +VideoFrame VideoFrame::Builder::build() { + return VideoFrame(video_frame_buffer_, timestamp_us_, timestamp_rtp_, + ntp_time_ms_, rotation_, color_space_); +} + +VideoFrame::Builder& VideoFrame::Builder::set_video_frame_buffer( + const rtc::scoped_refptr& buffer) { + video_frame_buffer_ = buffer; + return *this; +} + +VideoFrame::Builder& VideoFrame::Builder::set_timestamp_ms( + int64_t timestamp_ms) { + timestamp_us_ = timestamp_ms * rtc::kNumMicrosecsPerMillisec; + return *this; +} + +VideoFrame::Builder& VideoFrame::Builder::set_timestamp_us( + int64_t timestamp_us) { + timestamp_us_ = timestamp_us; + return *this; +} + +VideoFrame::Builder& VideoFrame::Builder::set_timestamp_rtp( + uint32_t timestamp_rtp) { + timestamp_rtp_ = timestamp_rtp; + return *this; +} + +VideoFrame::Builder& VideoFrame::Builder::set_ntp_time_ms(int64_t ntp_time_ms) { + ntp_time_ms_ = ntp_time_ms; + return *this; +} + +VideoFrame::Builder& VideoFrame::Builder::set_rotation(VideoRotation rotation) { + rotation_ = rotation; + return *this; +} + +VideoFrame::Builder& VideoFrame::Builder::set_color_space( + const ColorSpace& color_space) { + color_space_ = color_space; + return *this; +} + VideoFrame::VideoFrame(const rtc::scoped_refptr& buffer, webrtc::VideoRotation rotation, int64_t timestamp_us) @@ -36,6 +85,19 @@ VideoFrame::VideoFrame(const rtc::scoped_refptr& buffer, RTC_DCHECK(buffer); } +VideoFrame::VideoFrame(const rtc::scoped_refptr& buffer, + int64_t timestamp_us, + uint32_t timestamp_rtp, + int64_t ntp_time_ms, + VideoRotation rotation, + const absl::optional& color_space) + : video_frame_buffer_(buffer), + timestamp_rtp_(timestamp_rtp), + ntp_time_ms_(ntp_time_ms), + timestamp_us_(timestamp_us), + rotation_(rotation), + color_space_(color_space) {} + VideoFrame::~VideoFrame() = default; VideoFrame::VideoFrame(const VideoFrame&) = default; diff --git a/api/video/video_frame.h b/api/video/video_frame.h index bd08346cb5..dcb533e29f 100644 --- a/api/video/video_frame.h +++ b/api/video/video_frame.h @@ -13,6 +13,8 @@ #include +#include "absl/types/optional.h" +#include "api/video/color_space.h" #include "api/video/video_frame_buffer.h" #include "api/video/video_rotation.h" @@ -20,12 +22,35 @@ namespace webrtc { class VideoFrame { public: - // Preferred constructor. + // Preferred way of building VideoFrame objects. + class Builder { + public: + Builder(); + ~Builder(); + + VideoFrame build(); + Builder& set_video_frame_buffer( + const rtc::scoped_refptr& buffer); + Builder& set_timestamp_ms(int64_t timestamp_ms); + Builder& set_timestamp_us(int64_t timestamp_us); + Builder& set_timestamp_rtp(uint32_t timestamp_rtp); + Builder& set_ntp_time_ms(int64_t ntp_time_ms); + Builder& set_rotation(VideoRotation rotation); + Builder& set_color_space(const ColorSpace& color_space); + + private: + rtc::scoped_refptr video_frame_buffer_; + int64_t timestamp_us_ = 0; + uint32_t timestamp_rtp_ = 0; + int64_t ntp_time_ms_ = 0; + VideoRotation rotation_ = kVideoRotation_0; + absl::optional color_space_; + }; + + // To be deprecated. Migrate all use to Builder. VideoFrame(const rtc::scoped_refptr& buffer, webrtc::VideoRotation rotation, int64_t timestamp_us); - - // For use by the parts of the pipeline that needs the RTP 90kHz timestamp. VideoFrame(const rtc::scoped_refptr& buffer, uint32_t timestamp_rtp, int64_t render_time_ms, @@ -85,6 +110,9 @@ class VideoFrame { VideoRotation rotation() const { return rotation_; } void set_rotation(VideoRotation rotation) { rotation_ = rotation; } + // Set Color space when available. + absl::optional color_space() const { return color_space_; } + // Get render time in milliseconds. // TODO(nisse): Deprecated. Migrate all users to timestamp_us(). int64_t render_time_ms() const; @@ -100,12 +128,20 @@ class VideoFrame { } private: + VideoFrame(const rtc::scoped_refptr& buffer, + int64_t timestamp_us, + uint32_t timestamp_rtp, + int64_t ntp_time_ms, + VideoRotation rotation, + const absl::optional& color_space); + // An opaque reference counted handle that stores the pixel data. rtc::scoped_refptr video_frame_buffer_; uint32_t timestamp_rtp_; int64_t ntp_time_ms_; int64_t timestamp_us_; VideoRotation rotation_; + absl::optional color_space_; }; } // namespace webrtc diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 2a2c1deb37..50037996fe 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -479,6 +479,7 @@ rtc_static_library("webrtc_vp9") { ":webrtc_vp9_helpers", "..:module_api", "../..:webrtc_common", + "../../api/video:video_frame", "../../api/video:video_frame_i010", "../../api/video_codecs:video_codecs_api", "../../common_video", @@ -718,6 +719,7 @@ if (rtc_include_tests) { "../../api:create_videocodec_test_fixture_api", "../../api:mock_video_codec_factory", "../../api:videocodec_test_fixture_api", + "../../api/video:video_frame", "../../api/video:video_frame_i420", "../../api/video_codecs:rtc_software_fallback_wrappers", "../../api/video_codecs:video_codecs_api", diff --git a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc index 1d899aea9c..ec4efe86da 100644 --- a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc +++ b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc @@ -8,6 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "api/video/color_space.h" #include "api/video/i420_buffer.h" #include "common_video/libyuv/include/webrtc_libyuv.h" #include "media/base/vp9_profile.h" @@ -85,6 +86,12 @@ TEST_F(TestVp9Impl, EncodeDecode) { ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp)); ASSERT_TRUE(decoded_frame); EXPECT_GT(I420PSNR(input_frame, decoded_frame.get()), 36); + + const ColorSpace color_space = decoded_frame->color_space().value(); + EXPECT_EQ(ColorSpace::PrimaryID::kInvalid, color_space.primaries()); + EXPECT_EQ(ColorSpace::TransferID::kInvalid, color_space.transfer()); + EXPECT_EQ(ColorSpace::MatrixID::kInvalid, color_space.matrix()); + EXPECT_EQ(ColorSpace::RangeID::kLimited, color_space.range()); } // We only test the encoder here, since the decoded frame rotation is set based diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc index 56ff936bc1..da0ec5b264 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -21,6 +21,7 @@ #include "vpx/vpx_encoder.h" #include "absl/memory/memory.h" +#include "api/video/color_space.h" #include "api/video/i010_buffer.h" #include "common_video/include/video_frame_buffer.h" #include "common_video/libyuv/include/webrtc_libyuv.h" @@ -1178,10 +1179,72 @@ int VP9DecoderImpl::ReturnFrame(const vpx_image_t* img, RTC_NOTREACHED(); return WEBRTC_VIDEO_CODEC_NO_OUTPUT; } - VideoFrame decoded_image(img_wrapped_buffer, timestamp, - 0 /* render_time_ms */, webrtc::kVideoRotation_0); - decoded_image.set_ntp_time_ms(ntp_time_ms); + ColorSpace::PrimaryID primaries = ColorSpace::PrimaryID::kInvalid; + ColorSpace::TransferID transfer = ColorSpace::TransferID::kInvalid; + ColorSpace::MatrixID matrix = ColorSpace::MatrixID::kInvalid; + switch (img->cs) { + case VPX_CS_BT_601: + case VPX_CS_SMPTE_170: + primaries = ColorSpace::PrimaryID::kSMPTE170M; + transfer = ColorSpace::TransferID::kSMPTE170M; + matrix = ColorSpace::MatrixID::kSMPTE170M; + break; + case VPX_CS_SMPTE_240: + primaries = ColorSpace::PrimaryID::kSMPTE240M; + transfer = ColorSpace::TransferID::kSMPTE240M; + matrix = ColorSpace::MatrixID::kSMPTE240M; + break; + case VPX_CS_BT_709: + primaries = ColorSpace::PrimaryID::kBT709; + transfer = ColorSpace::TransferID::kBT709; + matrix = ColorSpace::MatrixID::kBT709; + break; + case VPX_CS_BT_2020: + primaries = ColorSpace::PrimaryID::kBT2020; + switch (img->bit_depth) { + case 8: + transfer = ColorSpace::TransferID::kBT709; + break; + case 10: + transfer = ColorSpace::TransferID::kBT2020_10; + break; + default: + RTC_NOTREACHED(); + break; + } + matrix = ColorSpace::MatrixID::kBT2020_NCL; + break; + case VPX_CS_SRGB: + primaries = ColorSpace::PrimaryID::kBT709; + transfer = ColorSpace::TransferID::kIEC61966_2_1; + matrix = ColorSpace::MatrixID::kBT709; + break; + default: + break; + } + + ColorSpace::RangeID range = ColorSpace::RangeID::kInvalid; + switch (img->range) { + case VPX_CR_STUDIO_RANGE: + range = ColorSpace::RangeID::kLimited; + break; + case VPX_CR_FULL_RANGE: + range = ColorSpace::RangeID::kFull; + break; + default: + break; + } + + VideoFrame decoded_image = + VideoFrame::Builder() + .set_video_frame_buffer(img_wrapped_buffer) + .set_timestamp_ms(0) + .set_timestamp_rtp(timestamp) + .set_ntp_time_ms(ntp_time_ms) + .set_rotation(webrtc::kVideoRotation_0) + .set_color_space(ColorSpace(primaries, transfer, matrix, range)) + .build(); decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp); return WEBRTC_VIDEO_CODEC_OK; }