diff --git a/modules/video_coding/codecs/av1/libaom_av1_encoder.cc b/modules/video_coding/codecs/av1/libaom_av1_encoder.cc index 79a31d9baa..b47dab9834 100644 --- a/modules/video_coding/codecs/av1/libaom_av1_encoder.cc +++ b/modules/video_coding/codecs/av1/libaom_av1_encoder.cc @@ -105,6 +105,8 @@ class LibaomAv1Encoder final : public VideoEncoder { // Configures the encoder which buffers next frame updates and can reference. void SetSvcRefFrameConfig( const ScalableVideoController::LayerFrameConfig& layer_frame); + // If pixel format doesn't match, then reallocate. + void MaybeRewrapImgWithFormat(const aom_img_fmt_t fmt); std::unique_ptr svc_controller_; bool inited_; @@ -228,11 +230,10 @@ int LibaomAv1Encoder::InitEncode(const VideoCodec* codec_settings, cfg_.g_pass = AOM_RC_ONE_PASS; // One-pass rate control cfg_.g_lag_in_frames = kLagInFrames; // No look ahead when lag equals 0. - // Creating a wrapper to the image - setting image data to nullptr. Actual - // pointer will be set in encode. Setting align to 1, as it is meaningless - // (actual memory is not allocated). - frame_for_encode_ = - aom_img_alloc(nullptr, AOM_IMG_FMT_I420, cfg_.g_w, cfg_.g_h, 1); + if (frame_for_encode_ != nullptr) { + aom_img_free(frame_for_encode_); + frame_for_encode_ = nullptr; + } // Flag options: AOM_CODEC_USE_PSNR and AOM_CODEC_USE_HIGHBITDEPTH aom_codec_flags_t flags = 0; @@ -555,6 +556,21 @@ int32_t LibaomAv1Encoder::Release() { return WEBRTC_VIDEO_CODEC_OK; } +void LibaomAv1Encoder::MaybeRewrapImgWithFormat(const aom_img_fmt_t fmt) { + if (!frame_for_encode_) { + frame_for_encode_ = + aom_img_wrap(nullptr, fmt, cfg_.g_w, cfg_.g_h, 1, nullptr); + + } else if (frame_for_encode_->fmt != fmt) { + RTC_LOG(LS_INFO) << "Switching AV1 encoder pixel format to " + << (fmt == AOM_IMG_FMT_NV12 ? "NV12" : "I420"); + aom_img_free(frame_for_encode_); + frame_for_encode_ = + aom_img_wrap(nullptr, fmt, cfg_.g_w, cfg_.g_h, 1, nullptr); + } + // else no-op since the image is already in the right format. +} + int32_t LibaomAv1Encoder::Encode( const VideoFrame& frame, const std::vector* frame_types) { @@ -574,38 +590,74 @@ int32_t LibaomAv1Encoder::Encode( return WEBRTC_VIDEO_CODEC_ERROR; } + rtc::scoped_refptr buffer = frame.video_frame_buffer(); + absl::InlinedVector + supported_formats = {VideoFrameBuffer::Type::kI420, + VideoFrameBuffer::Type::kNV12}; + rtc::scoped_refptr mapped_buffer; + if (buffer->type() != VideoFrameBuffer::Type::kNative) { + // `buffer` is already mapped. + mapped_buffer = buffer; + } else { + // Attempt to map to one of the supported formats. + mapped_buffer = buffer->GetMappedFrameBuffer(supported_formats); + } + // Convert input frame to I420, if needed. - VideoFrame prepped_input_frame = frame; - if (prepped_input_frame.video_frame_buffer()->type() != - VideoFrameBuffer::Type::kI420 && - prepped_input_frame.video_frame_buffer()->type() != - VideoFrameBuffer::Type::kI420A) { + if (!mapped_buffer || + (absl::c_find(supported_formats, mapped_buffer->type()) == + supported_formats.end() && + mapped_buffer->type() != VideoFrameBuffer::Type::kI420A)) { rtc::scoped_refptr converted_buffer( - prepped_input_frame.video_frame_buffer()->ToI420()); + mapped_buffer->ToI420()); if (!converted_buffer) { RTC_LOG(LS_ERROR) << "Failed to convert " << VideoFrameBufferTypeToString( - prepped_input_frame.video_frame_buffer()->type()) + frame.video_frame_buffer()->type()) << " image to I420. Can't encode frame."; return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; } RTC_CHECK(converted_buffer->type() == VideoFrameBuffer::Type::kI420 || converted_buffer->type() == VideoFrameBuffer::Type::kI420A); - prepped_input_frame = VideoFrame(converted_buffer, frame.timestamp(), - frame.render_time_ms(), frame.rotation()); + + mapped_buffer = converted_buffer; } - // Set frame_for_encode_ data pointers and strides. - auto i420_buffer = prepped_input_frame.video_frame_buffer()->GetI420(); - frame_for_encode_->planes[AOM_PLANE_Y] = - const_cast(i420_buffer->DataY()); - frame_for_encode_->planes[AOM_PLANE_U] = - const_cast(i420_buffer->DataU()); - frame_for_encode_->planes[AOM_PLANE_V] = - const_cast(i420_buffer->DataV()); - frame_for_encode_->stride[AOM_PLANE_Y] = i420_buffer->StrideY(); - frame_for_encode_->stride[AOM_PLANE_U] = i420_buffer->StrideU(); - frame_for_encode_->stride[AOM_PLANE_V] = i420_buffer->StrideV(); + switch (mapped_buffer->type()) { + case VideoFrameBuffer::Type::kI420: + case VideoFrameBuffer::Type::kI420A: { + // Set frame_for_encode_ data pointers and strides. + MaybeRewrapImgWithFormat(AOM_IMG_FMT_I420); + auto i420_buffer = mapped_buffer->GetI420(); + RTC_DCHECK(i420_buffer); + frame_for_encode_->planes[AOM_PLANE_Y] = + const_cast(i420_buffer->DataY()); + frame_for_encode_->planes[AOM_PLANE_U] = + const_cast(i420_buffer->DataU()); + frame_for_encode_->planes[AOM_PLANE_V] = + const_cast(i420_buffer->DataV()); + frame_for_encode_->stride[AOM_PLANE_Y] = i420_buffer->StrideY(); + frame_for_encode_->stride[AOM_PLANE_U] = i420_buffer->StrideU(); + frame_for_encode_->stride[AOM_PLANE_V] = i420_buffer->StrideV(); + break; + } + case VideoFrameBuffer::Type::kNV12: { + MaybeRewrapImgWithFormat(AOM_IMG_FMT_NV12); + const NV12BufferInterface* nv12_buffer = mapped_buffer->GetNV12(); + RTC_DCHECK(nv12_buffer); + frame_for_encode_->planes[AOM_PLANE_Y] = + const_cast(nv12_buffer->DataY()); + frame_for_encode_->planes[AOM_PLANE_U] = + const_cast(nv12_buffer->DataUV()); + frame_for_encode_->planes[AOM_PLANE_V] = nullptr; + frame_for_encode_->stride[AOM_PLANE_Y] = nv12_buffer->StrideY(); + frame_for_encode_->stride[AOM_PLANE_U] = nv12_buffer->StrideUV(); + frame_for_encode_->stride[AOM_PLANE_V] = 0; + break; + } + default: + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } const uint32_t duration = kRtpTicksPerSecond / static_cast(encoder_settings_.maxFramerate); @@ -805,7 +857,8 @@ VideoEncoder::EncoderInfo LibaomAv1Encoder::GetEncoderInfo() const { info.has_trusted_rate_controller = true; info.is_hardware_accelerated = false; info.scaling_settings = VideoEncoder::ScalingSettings(kMinQindex, kMaxQindex); - info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420}; + info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420, + VideoFrameBuffer::Type::kNV12}; if (SvcEnabled()) { for (int sid = 0; sid < svc_params_->number_spatial_layers; ++sid) { info.fps_allocation[sid].resize(svc_params_->number_temporal_layers); diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index aea1988df9..b1ff12ed11 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -8694,7 +8694,7 @@ constexpr std::pair kVP8DisallowConversion = constexpr std::pair kVP9DisallowConversion = std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false); constexpr std::pair kAV1AllowConversion = - std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true); + std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false); constexpr std::pair kMultiplexDisallowConversion = std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false); #if defined(WEBRTC_USE_H264)