zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove NV12 to I420 conversion in webrtc AV1 Encoder.

Browse Source 
libaom supports for NV12 inputs for encoding av1 stream. It will reduce
unnecessary conversion from NV12 to I420 format.
(https://bugs.chromium.org/p/aomedia/issues/detail?id=3232&q=3232&can=2)

Bug: webrtc:13746
Change-Id: I1407227d1690b3f63cb6581eef5d587e5f418892
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/251920
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Shuhai Peng <shuhai.peng@intel.com>
Cr-Commit-Position: refs/heads/main@{#36111}
This commit is contained in:
Shuhai Peng
2022-03-02 20:07:25 +08:00
committed by WebRTC LUCI CQ
parent 0d13bbd4b1
commit 9558ab41eb
2 changed files with 80 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
modules/video_coding/codecs/av1/libaom_av1_encoder.cc
View File

@ -105,6 +105,8 @@ class LibaomAv1Encoder final : public VideoEncoder {
// Configures the encoder which buffers next frame updates and can reference.
void SetSvcRefFrameConfig(
const ScalableVideoController::LayerFrameConfig& layer_frame);
// If pixel format doesn't match, then reallocate.
void MaybeRewrapImgWithFormat(const aom_img_fmt_t fmt);
std::unique_ptr<ScalableVideoController> svc_controller_;
bool inited_;
@ -228,11 +230,10 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings,
cfg_.g_pass = AOM_RC_ONE_PASS; // One-pass rate control
cfg_.g_lag_in_frames = kLagInFrames; // No look ahead when lag equals 0.
// Creating a wrapper to the image - setting image data to nullptr. Actual
// pointer will be set in encode. Setting align to 1, as it is meaningless
// (actual memory is not allocated).
frame_for_encode_ =
aom_img_alloc(nullptr, AOM_IMG_FMT_I420, cfg_.g_w, cfg_.g_h, 1);
if (frame_for_encode_ != nullptr) {
aom_img_free(frame_for_encode_);
frame_for_encode_ = nullptr;
}
// Flag options: AOM_CODEC_USE_PSNR and AOM_CODEC_USE_HIGHBITDEPTH
aom_codec_flags_t flags = 0;
@ -555,6 +556,21 @@ int32_t LibaomAv1Encoder::Release() {
return WEBRTC_VIDEO_CODEC_OK;
}
void LibaomAv1Encoder::MaybeRewrapImgWithFormat(const aom_img_fmt_t fmt) {
if (!frame_for_encode_) {
frame_for_encode_ =
aom_img_wrap(nullptr, fmt, cfg_.g_w, cfg_.g_h, 1, nullptr);
} else if (frame_for_encode_->fmt != fmt) {
RTC_LOG(LS_INFO) << "Switching AV1 encoder pixel format to "
<< (fmt == AOM_IMG_FMT_NV12 ? "NV12" : "I420");
aom_img_free(frame_for_encode_);
frame_for_encode_ =
aom_img_wrap(nullptr, fmt, cfg_.g_w, cfg_.g_h, 1, nullptr);
}
// else no-op since the image is already in the right format.
}
int32_t LibaomAv1Encoder::Encode(
const VideoFrame& frame,
const std::vector<VideoFrameType>* frame_types) {
@ -574,29 +590,46 @@ int32_t LibaomAv1Encoder::Encode(
return WEBRTC_VIDEO_CODEC_ERROR;
}
rtc::scoped_refptr<VideoFrameBuffer> buffer = frame.video_frame_buffer();
absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
supported_formats = {VideoFrameBuffer::Type::kI420,
VideoFrameBuffer::Type::kNV12};
rtc::scoped_refptr<VideoFrameBuffer> mapped_buffer;
if (buffer->type() != VideoFrameBuffer::Type::kNative) {
// `buffer` is already mapped.
mapped_buffer = buffer;
} else {
// Attempt to map to one of the supported formats.
mapped_buffer = buffer->GetMappedFrameBuffer(supported_formats);
}
// Convert input frame to I420, if needed.
VideoFrame prepped_input_frame = frame;
if (prepped_input_frame.video_frame_buffer()->type() !=
VideoFrameBuffer::Type::kI420 &&
prepped_input_frame.video_frame_buffer()->type() !=
VideoFrameBuffer::Type::kI420A) {
if (!mapped_buffer ||
(absl::c_find(supported_formats, mapped_buffer->type()) ==
supported_formats.end() &&
mapped_buffer->type() != VideoFrameBuffer::Type::kI420A)) {
rtc::scoped_refptr<I420BufferInterface> converted_buffer(
prepped_input_frame.video_frame_buffer()->ToI420());
mapped_buffer->ToI420());
if (!converted_buffer) {
RTC_LOG(LS_ERROR) << "Failed to convert "
<< VideoFrameBufferTypeToString(
prepped_input_frame.video_frame_buffer()->type())
frame.video_frame_buffer()->type())
<< " image to I420. Can't encode frame.";
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
}
RTC_CHECK(converted_buffer->type() == VideoFrameBuffer::Type::kI420 ||
converted_buffer->type() == VideoFrameBuffer::Type::kI420A);
prepped_input_frame = VideoFrame(converted_buffer, frame.timestamp(),
frame.render_time_ms(), frame.rotation());
mapped_buffer = converted_buffer;
}
switch (mapped_buffer->type()) {
case VideoFrameBuffer::Type::kI420:
case VideoFrameBuffer::Type::kI420A: {
// Set frame_for_encode_ data pointers and strides.
auto i420_buffer = prepped_input_frame.video_frame_buffer()->GetI420();
MaybeRewrapImgWithFormat(AOM_IMG_FMT_I420);
auto i420_buffer = mapped_buffer->GetI420();
RTC_DCHECK(i420_buffer);
frame_for_encode_->planes[AOM_PLANE_Y] =
const_cast<unsigned char*>(i420_buffer->DataY());
frame_for_encode_->planes[AOM_PLANE_U] =
@ -606,6 +639,25 @@ int32_t LibaomAv1Encoder::Encode(
frame_for_encode_->stride[AOM_PLANE_Y] = i420_buffer->StrideY();
frame_for_encode_->stride[AOM_PLANE_U] = i420_buffer->StrideU();
frame_for_encode_->stride[AOM_PLANE_V] = i420_buffer->StrideV();
break;
}
case VideoFrameBuffer::Type::kNV12: {
MaybeRewrapImgWithFormat(AOM_IMG_FMT_NV12);
const NV12BufferInterface* nv12_buffer = mapped_buffer->GetNV12();
RTC_DCHECK(nv12_buffer);
frame_for_encode_->planes[AOM_PLANE_Y] =
const_cast<unsigned char*>(nv12_buffer->DataY());
frame_for_encode_->planes[AOM_PLANE_U] =
const_cast<unsigned char*>(nv12_buffer->DataUV());
frame_for_encode_->planes[AOM_PLANE_V] = nullptr;
frame_for_encode_->stride[AOM_PLANE_Y] = nv12_buffer->StrideY();
frame_for_encode_->stride[AOM_PLANE_U] = nv12_buffer->StrideUV();
frame_for_encode_->stride[AOM_PLANE_V] = 0;
break;
}
default:
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
}
const uint32_t duration =
kRtpTicksPerSecond / static_cast<float>(encoder_settings_.maxFramerate);
@ -805,7 +857,8 @@ VideoEncoder::EncoderInfo LibaomAv1Encoder::GetEncoderInfo() const {
info.has_trusted_rate_controller = true;
info.is_hardware_accelerated = false;
info.scaling_settings = VideoEncoder::ScalingSettings(kMinQindex, kMaxQindex);
info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420};
info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420,
VideoFrameBuffer::Type::kNV12};
if (SvcEnabled()) {
for (int sid = 0; sid < svc_params_->number_spatial_layers; ++sid) {
info.fps_allocation[sid].resize(svc_params_->number_temporal_layers);

2
video/video_stream_encoder_unittest.cc
View File

@ -8694,7 +8694,7 @@ constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false);
constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
#if defined(WEBRTC_USE_H264)