From d4c16b131f68aaf68dc376978174bb3d585fa13d Mon Sep 17 00:00:00 2001 From: Emircan Uysaler Date: Mon, 23 Jul 2018 17:55:25 -0700 Subject: [PATCH] Extract color space from H264 decoder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes use of ColorSpace class to extract info from H264 stream. Bug: webrtc:9522 Change-Id: I651d16707260bb2867b1eda95dd4956d62c47279 Reviewed-on: https://webrtc-review.googlesource.com/90180 Reviewed-by: Erik Språng Reviewed-by: Niklas Enbom Commit-Queue: Emircan Uysaler Cr-Commit-Position: refs/heads/master@{#24085} --- api/video/color_space.h | 27 ++- modules/video_coding/BUILD.gn | 3 + .../codecs/h264/h264_color_space.cc | 171 ++++++++++++++++++ .../codecs/h264/h264_color_space.h | 27 +++ .../codecs/h264/h264_decoder_impl.cc | 38 ++-- .../codecs/h264/test/h264_impl_unittest.cc | 7 + 6 files changed, 260 insertions(+), 13 deletions(-) create mode 100644 modules/video_coding/codecs/h264/h264_color_space.cc create mode 100644 modules/video_coding/codecs/h264/h264_color_space.h diff --git a/api/video/color_space.h b/api/video/color_space.h index 736a170d06..8102647b6f 100644 --- a/api/video/color_space.h +++ b/api/video/color_space.h @@ -30,27 +30,50 @@ class ColorSpace { enum class PrimaryID { kInvalid, kBT709, + kBT470M, + kBT470BG, kSMPTE170M, // Identical to BT601 kSMPTE240M, + kFILM, kBT2020, + kSMPTEST428, + kSMPTEST431, + kSMPTEST432, + kJEDECP22, }; enum class TransferID { kInvalid, kBT709, + kGAMMA22, + kGAMMA28, kSMPTE170M, kSMPTE240M, - kBT2020, - kBT2020_10, + kLINEAR, + kLOG, + kLOG_SQRT, + kIEC61966_2_4, + kBT1361_ECG, kIEC61966_2_1, + kBT2020_10, + kBT2020_12, + kSMPTEST2084, + kSMPTEST428, + kARIB_STD_B67, }; enum class MatrixID { kInvalid, + kRGB, kBT709, + kFCC, + kBT470BG, kSMPTE170M, kSMPTE240M, + kYCOCG, kBT2020_NCL, + kBT2020_CL, + kSMPTE2085, }; enum class RangeID { diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 7437603c7a..9e11cce76a 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -296,6 +296,7 @@ rtc_static_library("webrtc_h264") { deps = [ ":video_codec_interface", ":video_coding_utility", + "../../api/video:video_frame", "../../api/video:video_frame_i420", "../../api/video_codecs:video_codecs_api", "../../media:rtc_h264_profile_id", @@ -310,6 +311,8 @@ rtc_static_library("webrtc_h264") { if (rtc_use_h264) { defines += [ "WEBRTC_USE_H264" ] sources += [ + "codecs/h264/h264_color_space.cc", + "codecs/h264/h264_color_space.h", "codecs/h264/h264_decoder_impl.cc", "codecs/h264/h264_decoder_impl.h", "codecs/h264/h264_encoder_impl.cc", diff --git a/modules/video_coding/codecs/h264/h264_color_space.cc b/modules/video_coding/codecs/h264/h264_color_space.cc new file mode 100644 index 0000000000..b48ccfb4b5 --- /dev/null +++ b/modules/video_coding/codecs/h264/h264_color_space.cc @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/codecs/h264/h264_color_space.h" + +namespace webrtc { + +ColorSpace ExtractH264ColorSpace(AVCodecContext* codec) { + ColorSpace::PrimaryID primaries = ColorSpace::PrimaryID::kInvalid; + switch (codec->color_primaries) { + case AVCOL_PRI_BT709: + primaries = ColorSpace::PrimaryID::kBT709; + break; + case AVCOL_PRI_BT470M: + primaries = ColorSpace::PrimaryID::kBT470M; + break; + case AVCOL_PRI_BT470BG: + primaries = ColorSpace::PrimaryID::kBT470BG; + break; + case AVCOL_PRI_SMPTE170M: + primaries = ColorSpace::PrimaryID::kSMPTE170M; + break; + case AVCOL_PRI_SMPTE240M: + primaries = ColorSpace::PrimaryID::kSMPTE240M; + break; + case AVCOL_PRI_FILM: + primaries = ColorSpace::PrimaryID::kFILM; + break; + case AVCOL_PRI_BT2020: + primaries = ColorSpace::PrimaryID::kBT2020; + break; + case AVCOL_PRI_SMPTE428: + primaries = ColorSpace::PrimaryID::kSMPTEST428; + break; + case AVCOL_PRI_SMPTE431: + primaries = ColorSpace::PrimaryID::kSMPTEST431; + break; + case AVCOL_PRI_SMPTE432: + primaries = ColorSpace::PrimaryID::kSMPTEST432; + break; + case AVCOL_PRI_JEDEC_P22: + primaries = ColorSpace::PrimaryID::kJEDECP22; + break; + case AVCOL_PRI_RESERVED0: + case AVCOL_PRI_UNSPECIFIED: + case AVCOL_PRI_RESERVED: + default: + break; + } + + ColorSpace::TransferID transfer = ColorSpace::TransferID::kInvalid; + switch (codec->color_trc) { + case AVCOL_TRC_BT709: + transfer = ColorSpace::TransferID::kBT709; + break; + case AVCOL_TRC_GAMMA22: + transfer = ColorSpace::TransferID::kGAMMA22; + break; + case AVCOL_TRC_GAMMA28: + transfer = ColorSpace::TransferID::kGAMMA28; + break; + case AVCOL_TRC_SMPTE170M: + transfer = ColorSpace::TransferID::kSMPTE170M; + break; + case AVCOL_TRC_SMPTE240M: + transfer = ColorSpace::TransferID::kSMPTE240M; + break; + case AVCOL_TRC_LINEAR: + transfer = ColorSpace::TransferID::kLINEAR; + break; + case AVCOL_TRC_LOG: + transfer = ColorSpace::TransferID::kLOG; + break; + case AVCOL_TRC_LOG_SQRT: + transfer = ColorSpace::TransferID::kLOG_SQRT; + break; + case AVCOL_TRC_IEC61966_2_4: + transfer = ColorSpace::TransferID::kIEC61966_2_4; + break; + case AVCOL_TRC_BT1361_ECG: + transfer = ColorSpace::TransferID::kBT1361_ECG; + break; + case AVCOL_TRC_IEC61966_2_1: + transfer = ColorSpace::TransferID::kIEC61966_2_1; + break; + case AVCOL_TRC_BT2020_10: + transfer = ColorSpace::TransferID::kBT2020_10; + break; + case AVCOL_TRC_BT2020_12: + transfer = ColorSpace::TransferID::kBT2020_12; + break; + case AVCOL_TRC_SMPTE2084: + transfer = ColorSpace::TransferID::kSMPTEST2084; + break; + case AVCOL_TRC_SMPTE428: + transfer = ColorSpace::TransferID::kSMPTEST428; + break; + case AVCOL_TRC_ARIB_STD_B67: + transfer = ColorSpace::TransferID::kARIB_STD_B67; + break; + case AVCOL_TRC_RESERVED0: + case AVCOL_TRC_UNSPECIFIED: + case AVCOL_TRC_RESERVED: + default: + break; + } + + ColorSpace::MatrixID matrix = ColorSpace::MatrixID::kInvalid; + switch (codec->colorspace) { + case AVCOL_SPC_RGB: + matrix = ColorSpace::MatrixID::kRGB; + break; + case AVCOL_SPC_BT709: + matrix = ColorSpace::MatrixID::kBT709; + break; + case AVCOL_SPC_FCC: + matrix = ColorSpace::MatrixID::kFCC; + break; + case AVCOL_SPC_BT470BG: + matrix = ColorSpace::MatrixID::kBT470BG; + break; + case AVCOL_SPC_SMPTE170M: + matrix = ColorSpace::MatrixID::kSMPTE170M; + break; + case AVCOL_SPC_SMPTE240M: + matrix = ColorSpace::MatrixID::kSMPTE240M; + break; + case AVCOL_SPC_YCGCO: + matrix = ColorSpace::MatrixID::kYCOCG; + break; + case AVCOL_SPC_BT2020_NCL: + matrix = ColorSpace::MatrixID::kBT2020_NCL; + break; + case AVCOL_SPC_BT2020_CL: + matrix = ColorSpace::MatrixID::kBT2020_CL; + break; + case AVCOL_SPC_SMPTE2085: + matrix = ColorSpace::MatrixID::kSMPTE2085; + break; + case AVCOL_SPC_CHROMA_DERIVED_NCL: + case AVCOL_SPC_CHROMA_DERIVED_CL: + case AVCOL_SPC_ICTCP: + case AVCOL_SPC_UNSPECIFIED: + case AVCOL_SPC_RESERVED: + default: + break; + } + + ColorSpace::RangeID range = ColorSpace::RangeID::kInvalid; + switch (codec->color_range) { + case AVCOL_RANGE_MPEG: + range = ColorSpace::RangeID::kLimited; + break; + case AVCOL_RANGE_JPEG: + range = ColorSpace::RangeID::kFull; + break; + case AVCOL_RANGE_UNSPECIFIED: + default: + break; + } + return ColorSpace(primaries, transfer, matrix, range); +} + +} // namespace webrtc diff --git a/modules/video_coding/codecs/h264/h264_color_space.h b/modules/video_coding/codecs/h264/h264_color_space.h new file mode 100644 index 0000000000..5608ab1f05 --- /dev/null +++ b/modules/video_coding/codecs/h264/h264_color_space.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_CODECS_H264_H264_COLOR_SPACE_H_ +#define MODULES_VIDEO_CODING_CODECS_H264_H264_COLOR_SPACE_H_ + +#include "api/video/color_space.h" + +extern "C" { +#include "third_party/ffmpeg/libavcodec/avcodec.h" +} // extern "C" + +namespace webrtc { + +// Helper class for extracting color space information from H264 stream. +ColorSpace ExtractH264ColorSpace(AVCodecContext* codec); + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_CODECS_H264_H264_COLOR_SPACE_H_ diff --git a/modules/video_coding/codecs/h264/h264_decoder_impl.cc b/modules/video_coding/codecs/h264/h264_decoder_impl.cc index e91def4e03..ae3de069c5 100644 --- a/modules/video_coding/codecs/h264/h264_decoder_impl.cc +++ b/modules/video_coding/codecs/h264/h264_decoder_impl.cc @@ -20,8 +20,10 @@ extern "C" { #include "third_party/ffmpeg/libavutil/imgutils.h" } // extern "C" +#include "api/video/color_space.h" #include "api/video/i420_buffer.h" #include "common_video/include/video_frame_buffer.h" +#include "modules/video_coding/codecs/h264/h264_color_space.h" #include "rtc_base/checks.h" #include "rtc_base/criticalsection.h" #include "rtc_base/keep_ref_until_done.h" @@ -288,15 +290,24 @@ int32_t H264DecoderImpl::Decode(const EncodedImage& input_image, RTC_DCHECK_EQ(av_frame_->reordered_opaque, frame_timestamp_us); // Obtain the |video_frame| containing the decoded image. - VideoFrame* video_frame = static_cast( - av_buffer_get_opaque(av_frame_->buf[0])); - RTC_DCHECK(video_frame); + VideoFrame* input_frame = + static_cast(av_buffer_get_opaque(av_frame_->buf[0])); + RTC_DCHECK(input_frame); rtc::scoped_refptr i420_buffer = - video_frame->video_frame_buffer()->GetI420(); + input_frame->video_frame_buffer()->GetI420(); RTC_CHECK_EQ(av_frame_->data[kYPlaneIndex], i420_buffer->DataY()); RTC_CHECK_EQ(av_frame_->data[kUPlaneIndex], i420_buffer->DataU()); RTC_CHECK_EQ(av_frame_->data[kVPlaneIndex], i420_buffer->DataV()); - video_frame->set_timestamp(input_image._timeStamp); + + const ColorSpace& color_space = ExtractH264ColorSpace(av_context_.get()); + VideoFrame decoded_frame = + VideoFrame::Builder() + .set_video_frame_buffer(input_frame->video_frame_buffer()) + .set_timestamp_us(input_frame->timestamp_us()) + .set_timestamp_rtp(input_image._timeStamp) + .set_rotation(input_frame->rotation()) + .set_color_space(color_space) + .build(); absl::optional qp; // TODO(sakal): Maybe it is possible to get QP directly from FFmpeg. @@ -319,19 +330,24 @@ int32_t H264DecoderImpl::Decode(const EncodedImage& input_image, i420_buffer->DataU(), i420_buffer->StrideU(), i420_buffer->DataV(), i420_buffer->StrideV(), rtc::KeepRefUntilDone(i420_buffer))); - VideoFrame cropped_frame( - cropped_buf, video_frame->timestamp(), video_frame->render_time_ms(), - video_frame->rotation()); + VideoFrame cropped_frame = + VideoFrame::Builder() + .set_video_frame_buffer(cropped_buf) + .set_timestamp_ms(decoded_frame.render_time_ms()) + .set_timestamp_rtp(decoded_frame.timestamp()) + .set_rotation(decoded_frame.rotation()) + .set_color_space(color_space) + .build(); // TODO(nisse): Timestamp and rotation are all zero here. Change decoder // interface to pass a VideoFrameBuffer instead of a VideoFrame? decoded_image_callback_->Decoded(cropped_frame, absl::nullopt, qp); } else { // Return decoded frame. - decoded_image_callback_->Decoded(*video_frame, absl::nullopt, qp); + decoded_image_callback_->Decoded(decoded_frame, absl::nullopt, qp); } - // Stop referencing it, possibly freeing |video_frame|. + // Stop referencing it, possibly freeing |input_frame|. av_frame_unref(av_frame_.get()); - video_frame = nullptr; + input_frame = nullptr; return WEBRTC_VIDEO_CODEC_OK; } diff --git a/modules/video_coding/codecs/h264/test/h264_impl_unittest.cc b/modules/video_coding/codecs/h264/test/h264_impl_unittest.cc index 88ef5f20cc..14bb6bcc47 100644 --- a/modules/video_coding/codecs/h264/test/h264_impl_unittest.cc +++ b/modules/video_coding/codecs/h264/test/h264_impl_unittest.cc @@ -8,6 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include "api/video/color_space.h" #include "common_video/libyuv/include/webrtc_libyuv.h" #include "modules/video_coding/codecs/h264/include/h264.h" #include "modules/video_coding/codecs/test/video_codec_unittest.h" @@ -54,6 +55,12 @@ TEST_F(TestH264Impl, MAYBE_EncodeDecode) { ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp)); ASSERT_TRUE(decoded_frame); EXPECT_GT(I420PSNR(input_frame, decoded_frame.get()), 36); + + const ColorSpace color_space = decoded_frame->color_space().value(); + EXPECT_EQ(ColorSpace::PrimaryID::kInvalid, color_space.primaries()); + EXPECT_EQ(ColorSpace::TransferID::kInvalid, color_space.transfer()); + EXPECT_EQ(ColorSpace::MatrixID::kInvalid, color_space.matrix()); + EXPECT_EQ(ColorSpace::RangeID::kLimited, color_space.range()); } TEST_F(TestH264Impl, MAYBE_DecodedQpEqualsEncodedQp) {