From fef0026f2f64a7aff21a70a30e5bb9e5a3fc2065 Mon Sep 17 00:00:00 2001 From: Andrey Logvin Date: Wed, 9 Mar 2022 11:37:17 +0000 Subject: [PATCH] Revert "Reland "remove NV12 to I420 conversion in webrtc AV1 Encoder."" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit d7031692e3ba9eed78ead07f8bf34a847ca1fce6. Reason for revert: Breaks downstream project Original change's description: > Reland "remove NV12 to I420 conversion in webrtc AV1 Encoder." > > This reverts commit 66557e1af3f95a70753e782224d13a6186ed0d2e. > > Reason for revert: Some downstream projects seem to have an old libaom version with no NV12 support yet. It will be updated soon. > > Original change's description: > > Revert "remove NV12 to I420 conversion in webrtc AV1 Encoder." > > > > This reverts commit 9558ab41eb4de39c62cda2dd1e559f5814a3a0c7. > > > > Reason for revert: speculative revert: breaks downstream project > > > > Original change's description: > > > remove NV12 to I420 conversion in webrtc AV1 Encoder. > > > > > > libaom supports for NV12 inputs for encoding av1 stream. It will reduce > > > unnecessary conversion from NV12 to I420 format. > > > (https://bugs.chromium.org/p/aomedia/issues/detail?id=3232&q=3232&can=2) > > > > > > Bug: webrtc:13746 > > > Change-Id: I1407227d1690b3f63cb6581eef5d587e5f418892 > > > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/251920 > > > Reviewed-by: Erik Språng > > > Reviewed-by: Ilya Nikolaevskiy > > > Commit-Queue: Shuhai Peng > > > Cr-Commit-Position: refs/heads/main@{#36111} > > > > Bug: webrtc:13746 > > Change-Id: Ie928f7f5b5992337a9d186fa70b7fdec20a33f00 > > No-Presubmit: true > > No-Tree-Checks: true > > No-Try: true > > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/253122 > > Bot-Commit: rubber-stamper@appspot.gserviceaccount.com > > Reviewed-by: Mirko Bonadei > > Commit-Queue: Artem Titov > > Owners-Override: Artem Titov > > Owners-Override: Mirko Bonadei > > Cr-Commit-Position: refs/heads/main@{#36114} > > Bug: webrtc:13746 > Change-Id: Ib26ff6204abceb863b03d55e5953797c9ca27fc2 > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/253215 > Bot-Commit: rubber-stamper@appspot.gserviceaccount.com > Reviewed-by: Erik Språng > Reviewed-by: Ilya Nikolaevskiy > Commit-Queue: Ilya Nikolaevskiy > Cr-Commit-Position: refs/heads/main@{#36156} Bug: webrtc:13746 Change-Id: Ia9f8024bf70a82f8e26cd7a80d3020ed796c1b40 No-Presubmit: true No-Tree-Checks: true No-Try: true Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/254262 Bot-Commit: rubber-stamper@appspot.gserviceaccount.com Reviewed-by: Andrey Logvin Commit-Queue: Andrey Logvin Reviewed-by: Ilya Nikolaevskiy Commit-Queue: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/main@{#36159} --- .../codecs/av1/libaom_av1_encoder.cc | 105 +++++------------- video/video_stream_encoder_unittest.cc | 2 +- 2 files changed, 27 insertions(+), 80 deletions(-) diff --git a/modules/video_coding/codecs/av1/libaom_av1_encoder.cc b/modules/video_coding/codecs/av1/libaom_av1_encoder.cc index 9f33acf24e..0e427be915 100644 --- a/modules/video_coding/codecs/av1/libaom_av1_encoder.cc +++ b/modules/video_coding/codecs/av1/libaom_av1_encoder.cc @@ -93,8 +93,6 @@ class LibaomAv1Encoder final : public VideoEncoder { // Configures the encoder which buffers next frame updates and can reference. void SetSvcRefFrameConfig( const ScalableVideoController::LayerFrameConfig& layer_frame); - // If pixel format doesn't match, then reallocate. - void MaybeRewrapImgWithFormat(const aom_img_fmt_t fmt); std::unique_ptr svc_controller_; bool inited_; @@ -218,10 +216,11 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings, cfg_.g_pass = AOM_RC_ONE_PASS; // One-pass rate control cfg_.g_lag_in_frames = kLagInFrames; // No look ahead when lag equals 0. - if (frame_for_encode_ != nullptr) { - aom_img_free(frame_for_encode_); - frame_for_encode_ = nullptr; - } + // Creating a wrapper to the image - setting image data to nullptr. Actual + // pointer will be set in encode. Setting align to 1, as it is meaningless + // (actual memory is not allocated). + frame_for_encode_ = + aom_img_alloc(nullptr, AOM_IMG_FMT_I420, cfg_.g_w, cfg_.g_h, 1); // Flag options: AOM_CODEC_USE_PSNR and AOM_CODEC_USE_HIGHBITDEPTH aom_codec_flags_t flags = 0; @@ -579,21 +578,6 @@ int32_t LibaomAv1Encoder::Release() { return WEBRTC_VIDEO_CODEC_OK; } -void LibaomAv1Encoder::MaybeRewrapImgWithFormat(const aom_img_fmt_t fmt) { - if (!frame_for_encode_) { - frame_for_encode_ = - aom_img_wrap(nullptr, fmt, cfg_.g_w, cfg_.g_h, 1, nullptr); - - } else if (frame_for_encode_->fmt != fmt) { - RTC_LOG(LS_INFO) << "Switching AV1 encoder pixel format to " - << (fmt == AOM_IMG_FMT_NV12 ? "NV12" : "I420"); - aom_img_free(frame_for_encode_); - frame_for_encode_ = - aom_img_wrap(nullptr, fmt, cfg_.g_w, cfg_.g_h, 1, nullptr); - } - // else no-op since the image is already in the right format. -} - int32_t LibaomAv1Encoder::Encode( const VideoFrame& frame, const std::vector* frame_types) { @@ -613,74 +597,38 @@ int32_t LibaomAv1Encoder::Encode( return WEBRTC_VIDEO_CODEC_ERROR; } - rtc::scoped_refptr buffer = frame.video_frame_buffer(); - absl::InlinedVector - supported_formats = {VideoFrameBuffer::Type::kI420, - VideoFrameBuffer::Type::kNV12}; - rtc::scoped_refptr mapped_buffer; - if (buffer->type() != VideoFrameBuffer::Type::kNative) { - // `buffer` is already mapped. - mapped_buffer = buffer; - } else { - // Attempt to map to one of the supported formats. - mapped_buffer = buffer->GetMappedFrameBuffer(supported_formats); - } - // Convert input frame to I420, if needed. - if (!mapped_buffer || - (absl::c_find(supported_formats, mapped_buffer->type()) == - supported_formats.end() && - mapped_buffer->type() != VideoFrameBuffer::Type::kI420A)) { + VideoFrame prepped_input_frame = frame; + if (prepped_input_frame.video_frame_buffer()->type() != + VideoFrameBuffer::Type::kI420 && + prepped_input_frame.video_frame_buffer()->type() != + VideoFrameBuffer::Type::kI420A) { rtc::scoped_refptr converted_buffer( - mapped_buffer->ToI420()); + prepped_input_frame.video_frame_buffer()->ToI420()); if (!converted_buffer) { RTC_LOG(LS_ERROR) << "Failed to convert " << VideoFrameBufferTypeToString( - frame.video_frame_buffer()->type()) + prepped_input_frame.video_frame_buffer()->type()) << " image to I420. Can't encode frame."; return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; } RTC_CHECK(converted_buffer->type() == VideoFrameBuffer::Type::kI420 || converted_buffer->type() == VideoFrameBuffer::Type::kI420A); - - mapped_buffer = converted_buffer; + prepped_input_frame = VideoFrame(converted_buffer, frame.timestamp(), + frame.render_time_ms(), frame.rotation()); } - switch (mapped_buffer->type()) { - case VideoFrameBuffer::Type::kI420: - case VideoFrameBuffer::Type::kI420A: { - // Set frame_for_encode_ data pointers and strides. - MaybeRewrapImgWithFormat(AOM_IMG_FMT_I420); - auto i420_buffer = mapped_buffer->GetI420(); - RTC_DCHECK(i420_buffer); - frame_for_encode_->planes[AOM_PLANE_Y] = - const_cast(i420_buffer->DataY()); - frame_for_encode_->planes[AOM_PLANE_U] = - const_cast(i420_buffer->DataU()); - frame_for_encode_->planes[AOM_PLANE_V] = - const_cast(i420_buffer->DataV()); - frame_for_encode_->stride[AOM_PLANE_Y] = i420_buffer->StrideY(); - frame_for_encode_->stride[AOM_PLANE_U] = i420_buffer->StrideU(); - frame_for_encode_->stride[AOM_PLANE_V] = i420_buffer->StrideV(); - break; - } - case VideoFrameBuffer::Type::kNV12: { - MaybeRewrapImgWithFormat(AOM_IMG_FMT_NV12); - const NV12BufferInterface* nv12_buffer = mapped_buffer->GetNV12(); - RTC_DCHECK(nv12_buffer); - frame_for_encode_->planes[AOM_PLANE_Y] = - const_cast(nv12_buffer->DataY()); - frame_for_encode_->planes[AOM_PLANE_U] = - const_cast(nv12_buffer->DataUV()); - frame_for_encode_->planes[AOM_PLANE_V] = nullptr; - frame_for_encode_->stride[AOM_PLANE_Y] = nv12_buffer->StrideY(); - frame_for_encode_->stride[AOM_PLANE_U] = nv12_buffer->StrideUV(); - frame_for_encode_->stride[AOM_PLANE_V] = 0; - break; - } - default: - return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; - } + // Set frame_for_encode_ data pointers and strides. + auto i420_buffer = prepped_input_frame.video_frame_buffer()->GetI420(); + frame_for_encode_->planes[AOM_PLANE_Y] = + const_cast(i420_buffer->DataY()); + frame_for_encode_->planes[AOM_PLANE_U] = + const_cast(i420_buffer->DataU()); + frame_for_encode_->planes[AOM_PLANE_V] = + const_cast(i420_buffer->DataV()); + frame_for_encode_->stride[AOM_PLANE_Y] = i420_buffer->StrideY(); + frame_for_encode_->stride[AOM_PLANE_U] = i420_buffer->StrideU(); + frame_for_encode_->stride[AOM_PLANE_V] = i420_buffer->StrideV(); const uint32_t duration = kRtpTicksPerSecond / static_cast(encoder_settings_.maxFramerate); @@ -880,8 +828,7 @@ VideoEncoder::EncoderInfo LibaomAv1Encoder::GetEncoderInfo() const { info.has_trusted_rate_controller = true; info.is_hardware_accelerated = false; info.scaling_settings = VideoEncoder::ScalingSettings(kMinQindex, kMaxQindex); - info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420, - VideoFrameBuffer::Type::kNV12}; + info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420}; if (SvcEnabled()) { for (int sid = 0; sid < svc_params_->number_spatial_layers; ++sid) { info.fps_allocation[sid].resize(svc_params_->number_temporal_layers); diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index b1ff12ed11..aea1988df9 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -8694,7 +8694,7 @@ constexpr std::pair kVP8DisallowConversion = constexpr std::pair kVP9DisallowConversion = std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false); constexpr std::pair kAV1AllowConversion = - std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false); + std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true); constexpr std::pair kMultiplexDisallowConversion = std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false); #if defined(WEBRTC_USE_H264)