zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extract color space from H264 decoder

Browse Source 
Makes use of ColorSpace class to extract info from H264 stream.

Bug: webrtc:9522
Change-Id: I651d16707260bb2867b1eda95dd4956d62c47279
Reviewed-on: https://webrtc-review.googlesource.com/90180
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Niklas Enbom <niklas.enbom@webrtc.org>
Commit-Queue: Emircan Uysaler <emircan@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24085}
This commit is contained in:
Emircan Uysaler
2018-07-23 17:55:25 -07:00
committed by Commit Bot
parent e09e6ef947
commit d4c16b131f
6 changed files with 260 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
api/video/color_space.h
View File

@ -30,27 +30,50 @@ class ColorSpace {
enum class PrimaryID { enum class PrimaryID {
kInvalid, kInvalid,
kBT709, kBT709,
kBT470M,
kBT470BG,
kSMPTE170M, // Identical to BT601 kSMPTE170M, // Identical to BT601
kSMPTE240M, kSMPTE240M,
kFILM,
kBT2020, kBT2020,
kSMPTEST428,
kSMPTEST431,
kSMPTEST432,
kJEDECP22,
}; };
enum class TransferID { enum class TransferID {
kInvalid, kInvalid,
kBT709, kBT709,
kGAMMA22,
kGAMMA28,
kSMPTE170M, kSMPTE170M,
kSMPTE240M, kSMPTE240M,
kBT2020, kLINEAR,
kBT2020_10, kLOG,
kLOG_SQRT,
kIEC61966_2_4,
kBT1361_ECG,
kIEC61966_2_1, kIEC61966_2_1,
kBT2020_10,
kBT2020_12,
kSMPTEST2084,
kSMPTEST428,
kARIB_STD_B67,
}; };
enum class MatrixID { enum class MatrixID {
kInvalid, kInvalid,
kRGB,
kBT709, kBT709,
kFCC,
kBT470BG,
kSMPTE170M, kSMPTE170M,
kSMPTE240M, kSMPTE240M,
kYCOCG,
kBT2020_NCL, kBT2020_NCL,
kBT2020_CL,
kSMPTE2085,
}; };
enum class RangeID { enum class RangeID {

3
modules/video_coding/BUILD.gn
View File

@ -296,6 +296,7 @@ rtc_static_library("webrtc_h264") {
deps = [ deps = [
":video_codec_interface", ":video_codec_interface",
":video_coding_utility", ":video_coding_utility",
"../../api/video:video_frame",
"../../api/video:video_frame_i420", "../../api/video:video_frame_i420",
"../../api/video_codecs:video_codecs_api", "../../api/video_codecs:video_codecs_api",
"../../media:rtc_h264_profile_id", "../../media:rtc_h264_profile_id",
@ -310,6 +311,8 @@ rtc_static_library("webrtc_h264") {
if (rtc_use_h264) { if (rtc_use_h264) {
defines += [ "WEBRTC_USE_H264" ] defines += [ "WEBRTC_USE_H264" ]
sources += [ sources += [
"codecs/h264/h264_color_space.cc",
"codecs/h264/h264_color_space.h",
"codecs/h264/h264_decoder_impl.cc", "codecs/h264/h264_decoder_impl.cc",
"codecs/h264/h264_decoder_impl.h", "codecs/h264/h264_decoder_impl.h",
"codecs/h264/h264_encoder_impl.cc", "codecs/h264/h264_encoder_impl.cc",

171
modules/video_coding/codecs/h264/h264_color_space.cc Normal file
View File

@ -0,0 +1,171 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/video_coding/codecs/h264/h264_color_space.h"
namespace webrtc {
ColorSpace ExtractH264ColorSpace(AVCodecContext* codec) {
ColorSpace::PrimaryID primaries = ColorSpace::PrimaryID::kInvalid;
switch (codec->color_primaries) {
case AVCOL_PRI_BT709:
primaries = ColorSpace::PrimaryID::kBT709;
break;
case AVCOL_PRI_BT470M:
primaries = ColorSpace::PrimaryID::kBT470M;
break;
case AVCOL_PRI_BT470BG:
primaries = ColorSpace::PrimaryID::kBT470BG;
break;
case AVCOL_PRI_SMPTE170M:
primaries = ColorSpace::PrimaryID::kSMPTE170M;
break;
case AVCOL_PRI_SMPTE240M:
primaries = ColorSpace::PrimaryID::kSMPTE240M;
break;
case AVCOL_PRI_FILM:
primaries = ColorSpace::PrimaryID::kFILM;
break;
case AVCOL_PRI_BT2020:
primaries = ColorSpace::PrimaryID::kBT2020;
break;
case AVCOL_PRI_SMPTE428:
primaries = ColorSpace::PrimaryID::kSMPTEST428;
break;
case AVCOL_PRI_SMPTE431:
primaries = ColorSpace::PrimaryID::kSMPTEST431;
break;
case AVCOL_PRI_SMPTE432:
primaries = ColorSpace::PrimaryID::kSMPTEST432;
break;
case AVCOL_PRI_JEDEC_P22:
primaries = ColorSpace::PrimaryID::kJEDECP22;
break;
case AVCOL_PRI_RESERVED0:
case AVCOL_PRI_UNSPECIFIED:
case AVCOL_PRI_RESERVED:
default:
break;
}
ColorSpace::TransferID transfer = ColorSpace::TransferID::kInvalid;
switch (codec->color_trc) {
case AVCOL_TRC_BT709:
transfer = ColorSpace::TransferID::kBT709;
break;
case AVCOL_TRC_GAMMA22:
transfer = ColorSpace::TransferID::kGAMMA22;
break;
case AVCOL_TRC_GAMMA28:
transfer = ColorSpace::TransferID::kGAMMA28;
break;
case AVCOL_TRC_SMPTE170M:
transfer = ColorSpace::TransferID::kSMPTE170M;
break;
case AVCOL_TRC_SMPTE240M:
transfer = ColorSpace::TransferID::kSMPTE240M;
break;
case AVCOL_TRC_LINEAR:
transfer = ColorSpace::TransferID::kLINEAR;
break;
case AVCOL_TRC_LOG:
transfer = ColorSpace::TransferID::kLOG;
break;
case AVCOL_TRC_LOG_SQRT:
transfer = ColorSpace::TransferID::kLOG_SQRT;
break;
case AVCOL_TRC_IEC61966_2_4:
transfer = ColorSpace::TransferID::kIEC61966_2_4;
break;
case AVCOL_TRC_BT1361_ECG:
transfer = ColorSpace::TransferID::kBT1361_ECG;
break;
case AVCOL_TRC_IEC61966_2_1:
transfer = ColorSpace::TransferID::kIEC61966_2_1;
break;
case AVCOL_TRC_BT2020_10:
transfer = ColorSpace::TransferID::kBT2020_10;
break;
case AVCOL_TRC_BT2020_12:
transfer = ColorSpace::TransferID::kBT2020_12;
break;
case AVCOL_TRC_SMPTE2084:
transfer = ColorSpace::TransferID::kSMPTEST2084;
break;
case AVCOL_TRC_SMPTE428:
transfer = ColorSpace::TransferID::kSMPTEST428;
break;
case AVCOL_TRC_ARIB_STD_B67:
transfer = ColorSpace::TransferID::kARIB_STD_B67;
break;
case AVCOL_TRC_RESERVED0:
case AVCOL_TRC_UNSPECIFIED:
case AVCOL_TRC_RESERVED:
default:
break;
}
ColorSpace::MatrixID matrix = ColorSpace::MatrixID::kInvalid;
switch (codec->colorspace) {
case AVCOL_SPC_RGB:
matrix = ColorSpace::MatrixID::kRGB;
break;
case AVCOL_SPC_BT709:
matrix = ColorSpace::MatrixID::kBT709;
break;
case AVCOL_SPC_FCC:
matrix = ColorSpace::MatrixID::kFCC;
break;
case AVCOL_SPC_BT470BG:
matrix = ColorSpace::MatrixID::kBT470BG;
break;
case AVCOL_SPC_SMPTE170M:
matrix = ColorSpace::MatrixID::kSMPTE170M;
break;
case AVCOL_SPC_SMPTE240M:
matrix = ColorSpace::MatrixID::kSMPTE240M;
break;
case AVCOL_SPC_YCGCO:
matrix = ColorSpace::MatrixID::kYCOCG;
break;
case AVCOL_SPC_BT2020_NCL:
matrix = ColorSpace::MatrixID::kBT2020_NCL;
break;
case AVCOL_SPC_BT2020_CL:
matrix = ColorSpace::MatrixID::kBT2020_CL;
break;
case AVCOL_SPC_SMPTE2085:
matrix = ColorSpace::MatrixID::kSMPTE2085;
break;
case AVCOL_SPC_CHROMA_DERIVED_NCL:
case AVCOL_SPC_CHROMA_DERIVED_CL:
case AVCOL_SPC_ICTCP:
case AVCOL_SPC_UNSPECIFIED:
case AVCOL_SPC_RESERVED:
default:
break;
}
ColorSpace::RangeID range = ColorSpace::RangeID::kInvalid;
switch (codec->color_range) {
case AVCOL_RANGE_MPEG:
range = ColorSpace::RangeID::kLimited;
break;
case AVCOL_RANGE_JPEG:
range = ColorSpace::RangeID::kFull;
break;
case AVCOL_RANGE_UNSPECIFIED:
default:
break;
}
return ColorSpace(primaries, transfer, matrix, range);
}
} // namespace webrtc

27
modules/video_coding/codecs/h264/h264_color_space.h Normal file
View File

@ -0,0 +1,27 @@
/*
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_VIDEO_CODING_CODECS_H264_H264_COLOR_SPACE_H_
#define MODULES_VIDEO_CODING_CODECS_H264_H264_COLOR_SPACE_H_
#include "api/video/color_space.h"
extern "C" {
#include "third_party/ffmpeg/libavcodec/avcodec.h"
} // extern "C"
namespace webrtc {
// Helper class for extracting color space information from H264 stream.
ColorSpace ExtractH264ColorSpace(AVCodecContext* codec);
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_CODECS_H264_H264_COLOR_SPACE_H_

38
modules/video_coding/codecs/h264/h264_decoder_impl.cc
View File

@ -20,8 +20,10 @@ extern "C" {
#include "third_party/ffmpeg/libavutil/imgutils.h" #include "third_party/ffmpeg/libavutil/imgutils.h"
} // extern "C" } // extern "C"
#include "api/video/color_space.h"
#include "api/video/i420_buffer.h" #include "api/video/i420_buffer.h"
#include "common_video/include/video_frame_buffer.h" #include "common_video/include/video_frame_buffer.h"
#include "modules/video_coding/codecs/h264/h264_color_space.h"
#include "rtc_base/checks.h" #include "rtc_base/checks.h"
#include "rtc_base/criticalsection.h" #include "rtc_base/criticalsection.h"
#include "rtc_base/keep_ref_until_done.h" #include "rtc_base/keep_ref_until_done.h"
@ -288,15 +290,24 @@ int32_t H264DecoderImpl::Decode(const EncodedImage& input_image,
RTC_DCHECK_EQ(av_frame_->reordered_opaque, frame_timestamp_us); RTC_DCHECK_EQ(av_frame_->reordered_opaque, frame_timestamp_us);
// Obtain the |video_frame| containing the decoded image. // Obtain the |video_frame| containing the decoded image.
VideoFrame* video_frame = static_cast<VideoFrame*>( VideoFrame* input_frame =
av_buffer_get_opaque(av_frame_->buf[0])); static_cast<VideoFrame*>(av_buffer_get_opaque(av_frame_->buf[0]));
RTC_DCHECK(video_frame); RTC_DCHECK(input_frame);
rtc::scoped_refptr<webrtc::I420BufferInterface> i420_buffer = rtc::scoped_refptr<webrtc::I420BufferInterface> i420_buffer =
video_frame->video_frame_buffer()->GetI420(); input_frame->video_frame_buffer()->GetI420();
RTC_CHECK_EQ(av_frame_->data[kYPlaneIndex], i420_buffer->DataY()); RTC_CHECK_EQ(av_frame_->data[kYPlaneIndex], i420_buffer->DataY());
RTC_CHECK_EQ(av_frame_->data[kUPlaneIndex], i420_buffer->DataU()); RTC_CHECK_EQ(av_frame_->data[kUPlaneIndex], i420_buffer->DataU());
RTC_CHECK_EQ(av_frame_->data[kVPlaneIndex], i420_buffer->DataV()); RTC_CHECK_EQ(av_frame_->data[kVPlaneIndex], i420_buffer->DataV());
video_frame->set_timestamp(input_image._timeStamp);
const ColorSpace& color_space = ExtractH264ColorSpace(av_context_.get());
VideoFrame decoded_frame =
VideoFrame::Builder()
.set_video_frame_buffer(input_frame->video_frame_buffer())
.set_timestamp_us(input_frame->timestamp_us())
.set_timestamp_rtp(input_image._timeStamp)
.set_rotation(input_frame->rotation())
.set_color_space(color_space)
.build();
absl::optional<uint8_t> qp; absl::optional<uint8_t> qp;
// TODO(sakal): Maybe it is possible to get QP directly from FFmpeg. // TODO(sakal): Maybe it is possible to get QP directly from FFmpeg.
@ -319,19 +330,24 @@ int32_t H264DecoderImpl::Decode(const EncodedImage& input_image,
i420_buffer->DataU(), i420_buffer->StrideU(), i420_buffer->DataU(), i420_buffer->StrideU(),
i420_buffer->DataV(), i420_buffer->StrideV(), i420_buffer->DataV(), i420_buffer->StrideV(),
rtc::KeepRefUntilDone(i420_buffer))); rtc::KeepRefUntilDone(i420_buffer)));
VideoFrame cropped_frame( VideoFrame cropped_frame =
cropped_buf, video_frame->timestamp(), video_frame->render_time_ms(), VideoFrame::Builder()
video_frame->rotation()); .set_video_frame_buffer(cropped_buf)
.set_timestamp_ms(decoded_frame.render_time_ms())
.set_timestamp_rtp(decoded_frame.timestamp())
.set_rotation(decoded_frame.rotation())
.set_color_space(color_space)
.build();
// TODO(nisse): Timestamp and rotation are all zero here. Change decoder // TODO(nisse): Timestamp and rotation are all zero here. Change decoder
// interface to pass a VideoFrameBuffer instead of a VideoFrame? // interface to pass a VideoFrameBuffer instead of a VideoFrame?
decoded_image_callback_->Decoded(cropped_frame, absl::nullopt, qp); decoded_image_callback_->Decoded(cropped_frame, absl::nullopt, qp);
} else { } else {
// Return decoded frame. // Return decoded frame.
decoded_image_callback_->Decoded(*video_frame, absl::nullopt, qp); decoded_image_callback_->Decoded(decoded_frame, absl::nullopt, qp);
} }
// Stop referencing it, possibly freeing |video_frame|. // Stop referencing it, possibly freeing |input_frame|.
av_frame_unref(av_frame_.get()); av_frame_unref(av_frame_.get());
video_frame = nullptr; input_frame = nullptr;
return WEBRTC_VIDEO_CODEC_OK; return WEBRTC_VIDEO_CODEC_OK;
} }

7
modules/video_coding/codecs/h264/test/h264_impl_unittest.cc
View File

@ -8,6 +8,7 @@
* be found in the AUTHORS file in the root of the source tree. * be found in the AUTHORS file in the root of the source tree.
*/ */
#include "api/video/color_space.h"
#include "common_video/libyuv/include/webrtc_libyuv.h" #include "common_video/libyuv/include/webrtc_libyuv.h"
#include "modules/video_coding/codecs/h264/include/h264.h" #include "modules/video_coding/codecs/h264/include/h264.h"
#include "modules/video_coding/codecs/test/video_codec_unittest.h" #include "modules/video_coding/codecs/test/video_codec_unittest.h"
@ -54,6 +55,12 @@ TEST_F(TestH264Impl, MAYBE_EncodeDecode) {
ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp)); ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
ASSERT_TRUE(decoded_frame); ASSERT_TRUE(decoded_frame);
EXPECT_GT(I420PSNR(input_frame, decoded_frame.get()), 36); EXPECT_GT(I420PSNR(input_frame, decoded_frame.get()), 36);
const ColorSpace color_space = decoded_frame->color_space().value();
EXPECT_EQ(ColorSpace::PrimaryID::kInvalid, color_space.primaries());
EXPECT_EQ(ColorSpace::TransferID::kInvalid, color_space.transfer());
EXPECT_EQ(ColorSpace::MatrixID::kInvalid, color_space.matrix());
EXPECT_EQ(ColorSpace::RangeID::kLimited, color_space.range());
} }
TEST_F(TestH264Impl, MAYBE_DecodedQpEqualsEncodedQp) { TEST_F(TestH264Impl, MAYBE_DecodedQpEqualsEncodedQp) {