zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add color space information to webrtc::VideoFrame and extract from VP9

Browse Source 
This CL is the first step for introducing color space information in webrtc.
- Add ColorSpace class listing color profiles.
- Add ColorSpace as a member of webrtc::VideoFrame.
- Make use of this class by extracting info from VP9 decoder.

Bug: webrtc:9522
Change-Id: I5e2514efee2a193bddb4459261387f2d40e936ad
Reviewed-on: https://webrtc-review.googlesource.com/88540
Reviewed-by: Niklas Enbom <niklas.enbom@webrtc.org>
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Commit-Queue: Emircan Uysaler <emircan@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23988}
This commit is contained in:
Emircan Uysaler
2018-07-16 10:01:49 -07:00
committed by Commit Bot
parent d48b5f5a41
commit 800787f03b
8 changed files with 304 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
api/video/BUILD.gn
View File

@ -11,6 +11,8 @@ import("../../webrtc.gni")
rtc_source_set("video_frame") {
visibility = [ "*" ]
sources = [
"color_space.cc",
"color_space.h",
"video_content_type.cc",
"video_content_type.h",
"video_frame.cc",

42
api/video/color_space.cc Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/video/color_space.h"
namespace webrtc {
ColorSpace::ColorSpace() = default;
ColorSpace::ColorSpace(PrimaryID primaries,
TransferID transfer,
MatrixID matrix,
RangeID range)
: primaries_(primaries),
transfer_(transfer),
matrix_(matrix),
range_(range) {}
ColorSpace::PrimaryID ColorSpace::primaries() const {
return primaries_;
}
ColorSpace::TransferID ColorSpace::transfer() const {
return transfer_;
}
ColorSpace::MatrixID ColorSpace::matrix() const {
return matrix_;
}
ColorSpace::RangeID ColorSpace::range() const {
return range_;
}
} // namespace webrtc

84
api/video/color_space.h Normal file
View File

@ -0,0 +1,84 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef API_VIDEO_COLOR_SPACE_H_
#define API_VIDEO_COLOR_SPACE_H_
namespace webrtc {
// Used to represent a color space for the purpose of color conversion. This
// class only represents color information that can be transferred through the
// bitstream of WebRTC's internal supported codecs:
// - VP9 supports color profiles, see VP9 Bitstream & Decoding Process
// Specification Version 0.6 Section 7.2.2 "Color config semantics" available
// from https://www.webmproject.org.
// TODO(emircan): Extract these values from decode and add to the existing ones.
// - VP8 only supports BT.601, see
// https://tools.ietf.org/html/rfc6386#section-9.2
// - H264 supports different color primaries, transfer characteristics, matrix
// coefficients and range. See T-REC-H.264 E.2.1, "VUI parameters semantics",
// available from https://www.itu.int/rec/T-REC-H.264.
class ColorSpace {
public:
enum class PrimaryID {
kInvalid,
kBT709,
kSMPTE170M, // Identical to BT601
kSMPTE240M,
kBT2020,
};
enum class TransferID {
kInvalid,
kBT709,
kSMPTE170M,
kSMPTE240M,
kBT2020,
kBT2020_10,
kIEC61966_2_1,
};
enum class MatrixID {
kInvalid,
kBT709,
kSMPTE170M,
kSMPTE240M,
kBT2020_NCL,
};
enum class RangeID {
kInvalid,
// Limited Rec. 709 color range with RGB values ranging from 16 to 235.
kLimited,
// Full RGB color range with RGB valees from 0 to 255.
kFull,
};
ColorSpace();
ColorSpace(PrimaryID primaries,
TransferID transfer,
MatrixID matrix,
RangeID full_range);
PrimaryID primaries() const;
TransferID transfer() const;
MatrixID matrix() const;
RangeID range() const;
private:
PrimaryID primaries_ = PrimaryID::kInvalid;
TransferID transfer_ = TransferID::kInvalid;
MatrixID matrix_ = MatrixID::kInvalid;
RangeID range_ = RangeID::kInvalid;
};
} // namespace webrtc
#endif // API_VIDEO_COLOR_SPACE_H_

62
api/video/video_frame.cc
View File

@ -15,6 +15,55 @@
namespace webrtc {
VideoFrame::Builder::Builder() = default;
VideoFrame::Builder::~Builder() = default;
VideoFrame VideoFrame::Builder::build() {
return VideoFrame(video_frame_buffer_, timestamp_us_, timestamp_rtp_,
ntp_time_ms_, rotation_, color_space_);
}
VideoFrame::Builder& VideoFrame::Builder::set_video_frame_buffer(
const rtc::scoped_refptr<VideoFrameBuffer>& buffer) {
video_frame_buffer_ = buffer;
return *this;
}
VideoFrame::Builder& VideoFrame::Builder::set_timestamp_ms(
int64_t timestamp_ms) {
timestamp_us_ = timestamp_ms * rtc::kNumMicrosecsPerMillisec;
return *this;
}
VideoFrame::Builder& VideoFrame::Builder::set_timestamp_us(
int64_t timestamp_us) {
timestamp_us_ = timestamp_us;
return *this;
}
VideoFrame::Builder& VideoFrame::Builder::set_timestamp_rtp(
uint32_t timestamp_rtp) {
timestamp_rtp_ = timestamp_rtp;
return *this;
}
VideoFrame::Builder& VideoFrame::Builder::set_ntp_time_ms(int64_t ntp_time_ms) {
ntp_time_ms_ = ntp_time_ms;
return *this;
}
VideoFrame::Builder& VideoFrame::Builder::set_rotation(VideoRotation rotation) {
rotation_ = rotation;
return *this;
}
VideoFrame::Builder& VideoFrame::Builder::set_color_space(
const ColorSpace& color_space) {
color_space_ = color_space;
return *this;
}
VideoFrame::VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
webrtc::VideoRotation rotation,
int64_t timestamp_us)
@ -36,6 +85,19 @@ VideoFrame::VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
RTC_DCHECK(buffer);
}
VideoFrame::VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
int64_t timestamp_us,
uint32_t timestamp_rtp,
int64_t ntp_time_ms,
VideoRotation rotation,
const absl::optional<ColorSpace>& color_space)
: video_frame_buffer_(buffer),
timestamp_rtp_(timestamp_rtp),
ntp_time_ms_(ntp_time_ms),
timestamp_us_(timestamp_us),
rotation_(rotation),
color_space_(color_space) {}
VideoFrame::~VideoFrame() = default;
VideoFrame::VideoFrame(const VideoFrame&) = default;

42
api/video/video_frame.h
View File

@ -13,6 +13,8 @@
#include <stdint.h>
#include "absl/types/optional.h"
#include "api/video/color_space.h"
#include "api/video/video_frame_buffer.h"
#include "api/video/video_rotation.h"
@ -20,12 +22,35 @@ namespace webrtc {
class VideoFrame {
public:
// Preferred constructor.
// Preferred way of building VideoFrame objects.
class Builder {
public:
Builder();
~Builder();
VideoFrame build();
Builder& set_video_frame_buffer(
const rtc::scoped_refptr<VideoFrameBuffer>& buffer);
Builder& set_timestamp_ms(int64_t timestamp_ms);
Builder& set_timestamp_us(int64_t timestamp_us);
Builder& set_timestamp_rtp(uint32_t timestamp_rtp);
Builder& set_ntp_time_ms(int64_t ntp_time_ms);
Builder& set_rotation(VideoRotation rotation);
Builder& set_color_space(const ColorSpace& color_space);
private:
rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_;
int64_t timestamp_us_ = 0;
uint32_t timestamp_rtp_ = 0;
int64_t ntp_time_ms_ = 0;
VideoRotation rotation_ = kVideoRotation_0;
absl::optional<ColorSpace> color_space_;
};
// To be deprecated. Migrate all use to Builder.
VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
webrtc::VideoRotation rotation,
int64_t timestamp_us);
// For use by the parts of the pipeline that needs the RTP 90kHz timestamp.
VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
uint32_t timestamp_rtp,
int64_t render_time_ms,
@ -85,6 +110,9 @@ class VideoFrame {
VideoRotation rotation() const { return rotation_; }
void set_rotation(VideoRotation rotation) { rotation_ = rotation; }
// Set Color space when available.
absl::optional<ColorSpace> color_space() const { return color_space_; }
// Get render time in milliseconds.
// TODO(nisse): Deprecated. Migrate all users to timestamp_us().
int64_t render_time_ms() const;
@ -100,12 +128,20 @@ class VideoFrame {
}
private:
VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
int64_t timestamp_us,
uint32_t timestamp_rtp,
int64_t ntp_time_ms,
VideoRotation rotation,
const absl::optional<ColorSpace>& color_space);
// An opaque reference counted handle that stores the pixel data.
rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_;
uint32_t timestamp_rtp_;
int64_t ntp_time_ms_;
int64_t timestamp_us_;
VideoRotation rotation_;
absl::optional<ColorSpace> color_space_;
};
} // namespace webrtc

2
modules/video_coding/BUILD.gn
View File

@ -479,6 +479,7 @@ rtc_static_library("webrtc_vp9") {
":webrtc_vp9_helpers",
"..:module_api",
"../..:webrtc_common",
"../../api/video:video_frame",
"../../api/video:video_frame_i010",
"../../api/video_codecs:video_codecs_api",
"../../common_video",
@ -718,6 +719,7 @@ if (rtc_include_tests) {
"../../api:create_videocodec_test_fixture_api",
"../../api:mock_video_codec_factory",
"../../api:videocodec_test_fixture_api",
"../../api/video:video_frame",
"../../api/video:video_frame_i420",
"../../api/video_codecs:rtc_software_fallback_wrappers",
"../../api/video_codecs:video_codecs_api",

7
modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
View File

@ -8,6 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "api/video/color_space.h"
#include "api/video/i420_buffer.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "media/base/vp9_profile.h"
@ -85,6 +86,12 @@ TEST_F(TestVp9Impl, EncodeDecode) {
ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
ASSERT_TRUE(decoded_frame);
EXPECT_GT(I420PSNR(input_frame, decoded_frame.get()), 36);
const ColorSpace color_space = decoded_frame->color_space().value();
EXPECT_EQ(ColorSpace::PrimaryID::kInvalid, color_space.primaries());
EXPECT_EQ(ColorSpace::TransferID::kInvalid, color_space.transfer());
EXPECT_EQ(ColorSpace::MatrixID::kInvalid, color_space.matrix());
EXPECT_EQ(ColorSpace::RangeID::kLimited, color_space.range());
}
// We only test the encoder here, since the decoded frame rotation is set based

69
modules/video_coding/codecs/vp9/vp9_impl.cc
View File

@ -21,6 +21,7 @@
#include "vpx/vpx_encoder.h"
#include "absl/memory/memory.h"
#include "api/video/color_space.h"
#include "api/video/i010_buffer.h"
#include "common_video/include/video_frame_buffer.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
@ -1178,10 +1179,72 @@ int VP9DecoderImpl::ReturnFrame(const vpx_image_t* img,
RTC_NOTREACHED();
return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
}
VideoFrame decoded_image(img_wrapped_buffer, timestamp,
0 /* render_time_ms */, webrtc::kVideoRotation_0);
decoded_image.set_ntp_time_ms(ntp_time_ms);
ColorSpace::PrimaryID primaries = ColorSpace::PrimaryID::kInvalid;
ColorSpace::TransferID transfer = ColorSpace::TransferID::kInvalid;
ColorSpace::MatrixID matrix = ColorSpace::MatrixID::kInvalid;
switch (img->cs) {
case VPX_CS_BT_601:
case VPX_CS_SMPTE_170:
primaries = ColorSpace::PrimaryID::kSMPTE170M;
transfer = ColorSpace::TransferID::kSMPTE170M;
matrix = ColorSpace::MatrixID::kSMPTE170M;
break;
case VPX_CS_SMPTE_240:
primaries = ColorSpace::PrimaryID::kSMPTE240M;
transfer = ColorSpace::TransferID::kSMPTE240M;
matrix = ColorSpace::MatrixID::kSMPTE240M;
break;
case VPX_CS_BT_709:
primaries = ColorSpace::PrimaryID::kBT709;
transfer = ColorSpace::TransferID::kBT709;
matrix = ColorSpace::MatrixID::kBT709;
break;
case VPX_CS_BT_2020:
primaries = ColorSpace::PrimaryID::kBT2020;
switch (img->bit_depth) {
case 8:
transfer = ColorSpace::TransferID::kBT709;
break;
case 10:
transfer = ColorSpace::TransferID::kBT2020_10;
break;
default:
RTC_NOTREACHED();
break;
}
matrix = ColorSpace::MatrixID::kBT2020_NCL;
break;
case VPX_CS_SRGB:
primaries = ColorSpace::PrimaryID::kBT709;
transfer = ColorSpace::TransferID::kIEC61966_2_1;
matrix = ColorSpace::MatrixID::kBT709;
break;
default:
break;
}
ColorSpace::RangeID range = ColorSpace::RangeID::kInvalid;
switch (img->range) {
case VPX_CR_STUDIO_RANGE:
range = ColorSpace::RangeID::kLimited;
break;
case VPX_CR_FULL_RANGE:
range = ColorSpace::RangeID::kFull;
break;
default:
break;
}
VideoFrame decoded_image =
VideoFrame::Builder()
.set_video_frame_buffer(img_wrapped_buffer)
.set_timestamp_ms(0)
.set_timestamp_rtp(timestamp)
.set_ntp_time_ms(ntp_time_ms)
.set_rotation(webrtc::kVideoRotation_0)
.set_color_space(ColorSpace(primaries, transfer, matrix, range))
.build();
decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp);
return WEBRTC_VIDEO_CODEC_OK;
}