diff --git a/webrtc/common_video/include/video_frame_buffer.h b/webrtc/common_video/include/video_frame_buffer.h index fb867bc428..0b5d863f28 100644 --- a/webrtc/common_video/include/video_frame_buffer.h +++ b/webrtc/common_video/include/video_frame_buffer.h @@ -114,10 +114,6 @@ class I420Buffer : public VideoFrameBuffer { // Scale all of |src| to the size of |this| buffer, with no cropping. void ScaleFrom(const rtc::scoped_refptr& src); - // Create a new buffer with identical strides, and copy the pixel data. - static rtc::scoped_refptr CopyKeepStride( - const rtc::scoped_refptr& buffer); - // Returns a rotated versions of |src|. Native buffers are not // supported. The reason this function doesn't return an I420Buffer, // is that it returns |src| unchanged in case |rotation| is zero. diff --git a/webrtc/common_video/video_frame_buffer.cc b/webrtc/common_video/video_frame_buffer.cc index 1d209370a6..869eb4f87c 100644 --- a/webrtc/common_video/video_frame_buffer.cc +++ b/webrtc/common_video/video_frame_buffer.cc @@ -204,27 +204,6 @@ void I420Buffer::ScaleFrom(const rtc::scoped_refptr& src) { CropAndScaleFrom(src, 0, 0, src->width(), src->height()); } -// static -rtc::scoped_refptr I420Buffer::CopyKeepStride( - const rtc::scoped_refptr& source) { - int width = source->width(); - int height = source->height(); - int stride_y = source->StrideY(); - int stride_u = source->StrideU(); - int stride_v = source->StrideV(); - rtc::scoped_refptr target = - I420Buffer::Create(width, height, stride_y, stride_u, stride_v); - RTC_CHECK(libyuv::I420Copy(source->DataY(), stride_y, - source->DataU(), stride_u, - source->DataV(), stride_v, - target->MutableDataY(), stride_y, - target->MutableDataU(), stride_u, - target->MutableDataV(), stride_v, - width, height) == 0); - - return target; -} - // static rtc::scoped_refptr I420Buffer::Rotate( const rtc::scoped_refptr& src, diff --git a/webrtc/modules/video_processing/frame_preprocessor.cc b/webrtc/modules/video_processing/frame_preprocessor.cc index e86bbbb3bf..8d6d8bbaa6 100644 --- a/webrtc/modules/video_processing/frame_preprocessor.cc +++ b/webrtc/modules/video_processing/frame_preprocessor.cc @@ -19,7 +19,6 @@ VPMFramePreprocessor::VPMFramePreprocessor() spatial_resampler_ = new VPMSimpleSpatialResampler(); vd_ = new VPMVideoDecimator(); EnableDenoising(false); - denoised_frame_toggle_ = 0; } VPMFramePreprocessor::~VPMFramePreprocessor() { @@ -96,22 +95,10 @@ const VideoFrame* VPMFramePreprocessor::PreprocessFrame( const VideoFrame* current_frame = &frame; if (denoiser_) { - rtc::scoped_refptr* denoised_buffer = &denoised_buffer_[0]; - rtc::scoped_refptr* denoised_buffer_prev = &denoised_buffer_[1]; - // Swap the buffer to save one memcpy in DenoiseFrame. - if (denoised_frame_toggle_) { - denoised_buffer = &denoised_buffer_[1]; - denoised_buffer_prev = &denoised_buffer_[0]; - } - // Invert the flag. - denoised_frame_toggle_ ^= 1; - denoiser_->DenoiseFrame(current_frame->video_frame_buffer(), - denoised_buffer, - denoised_buffer_prev, true); - denoised_frame_ = VideoFrame(*denoised_buffer, - current_frame->timestamp(), - current_frame->render_time_ms(), - current_frame->rotation()); + denoised_frame_ = VideoFrame( + denoiser_->DenoiseFrame(current_frame->video_frame_buffer(), true), + current_frame->timestamp(), current_frame->render_time_ms(), + current_frame->rotation()); current_frame = &denoised_frame_; } diff --git a/webrtc/modules/video_processing/frame_preprocessor.h b/webrtc/modules/video_processing/frame_preprocessor.h index 47c17ec2e7..38875cdca7 100644 --- a/webrtc/modules/video_processing/frame_preprocessor.h +++ b/webrtc/modules/video_processing/frame_preprocessor.h @@ -61,13 +61,11 @@ class VPMFramePreprocessor { // we can compute new content metrics every |kSkipFrameCA| frames. enum { kSkipFrameCA = 2 }; - rtc::scoped_refptr denoised_buffer_[2]; VideoFrame denoised_frame_; VideoFrame resampled_frame_; VPMSpatialResampler* spatial_resampler_; VPMVideoDecimator* vd_; std::unique_ptr denoiser_; - uint8_t denoised_frame_toggle_; uint32_t frame_cnt_; }; diff --git a/webrtc/modules/video_processing/test/denoiser_test.cc b/webrtc/modules/video_processing/test/denoiser_test.cc index 325efbee29..b5e0352b94 100644 --- a/webrtc/modules/video_processing/test/denoiser_test.cc +++ b/webrtc/modules/video_processing/test/denoiser_test.cc @@ -12,7 +12,7 @@ #include -#include "webrtc/common_video/include/video_frame_buffer.h" +#include "webrtc/common_video/include/i420_buffer_pool.h" #include "webrtc/modules/video_processing/video_denoiser.h" #include "webrtc/test/gtest.h" #include "webrtc/test/frame_utils.h" @@ -135,16 +135,10 @@ TEST(VideoDenoiserTest, Denoiser) { ASSERT_TRUE(source_file != nullptr) << "Cannot open source file: " << video_file; - // Used in swap buffer. - int denoised_frame_toggle = 0; // Create pure C denoiser. VideoDenoiser denoiser_c(false); // Create SSE or NEON denoiser. VideoDenoiser denoiser_sse_neon(true); - rtc::scoped_refptr denoised_frame_c; - rtc::scoped_refptr denoised_frame_prev_c; - rtc::scoped_refptr denoised_frame_sse_neon; - rtc::scoped_refptr denoised_frame_prev_sse_neon; for (;;) { rtc::scoped_refptr video_frame_buffer( @@ -152,29 +146,14 @@ TEST(VideoDenoiserTest, Denoiser) { if (!video_frame_buffer) break; - rtc::scoped_refptr* p_denoised_c = &denoised_frame_c; - rtc::scoped_refptr* p_denoised_prev_c = &denoised_frame_prev_c; - rtc::scoped_refptr* p_denoised_sse_neon = - &denoised_frame_sse_neon; - rtc::scoped_refptr* p_denoised_prev_sse_neon = - &denoised_frame_prev_sse_neon; - // Swap the buffer to save one memcpy in DenoiseFrame. - if (denoised_frame_toggle) { - p_denoised_c = &denoised_frame_prev_c; - p_denoised_prev_c = &denoised_frame_c; - p_denoised_sse_neon = &denoised_frame_prev_sse_neon; - p_denoised_prev_sse_neon = &denoised_frame_sse_neon; - } - denoiser_c.DenoiseFrame(video_frame_buffer, - p_denoised_c, p_denoised_prev_c, - false); - denoiser_sse_neon.DenoiseFrame(video_frame_buffer, - p_denoised_sse_neon, - p_denoised_prev_sse_neon, false); - // Invert the flag. - denoised_frame_toggle ^= 1; + rtc::scoped_refptr denoised_frame_c( + denoiser_c.DenoiseFrame(video_frame_buffer, false)); + rtc::scoped_refptr denoised_frame_sse_neon( + denoiser_sse_neon.DenoiseFrame(video_frame_buffer, false)); + // Denoising results should be the same for C and SSE/NEON denoiser. - ASSERT_TRUE(test::FrameBufsEqual(*p_denoised_c, *p_denoised_sse_neon)); + ASSERT_TRUE( + test::FrameBufsEqual(denoised_frame_c, denoised_frame_sse_neon)); } ASSERT_NE(0, feof(source_file)) << "Error reading source file"; } diff --git a/webrtc/modules/video_processing/util/denoiser_filter.h b/webrtc/modules/video_processing/util/denoiser_filter.h index 1254a88d3c..fa1415b09a 100644 --- a/webrtc/modules/video_processing/util/denoiser_filter.h +++ b/webrtc/modules/video_processing/util/denoiser_filter.h @@ -42,7 +42,7 @@ class DenoiserFilter { const uint8_t* b, int b_stride, unsigned int* sse) = 0; - virtual DenoiserDecision MbDenoise(uint8_t* mc_running_avg_y, + virtual DenoiserDecision MbDenoise(const uint8_t* mc_running_avg_y, int mc_avg_y_stride, uint8_t* running_avg_y, int avg_y_stride, diff --git a/webrtc/modules/video_processing/util/denoiser_filter_c.cc b/webrtc/modules/video_processing/util/denoiser_filter_c.cc index 1b3c0b7098..8f4212193f 100644 --- a/webrtc/modules/video_processing/util/denoiser_filter_c.cc +++ b/webrtc/modules/video_processing/util/denoiser_filter_c.cc @@ -48,7 +48,7 @@ uint32_t DenoiserFilterC::Variance16x8(const uint8_t* a, return *sse - ((static_cast(sum) * sum) >> 7); } -DenoiserDecision DenoiserFilterC::MbDenoise(uint8_t* mc_running_avg_y, +DenoiserDecision DenoiserFilterC::MbDenoise(const uint8_t* mc_running_avg_y, int mc_avg_y_stride, uint8_t* running_avg_y, int avg_y_stride, diff --git a/webrtc/modules/video_processing/util/denoiser_filter_c.h b/webrtc/modules/video_processing/util/denoiser_filter_c.h index d8b6c5eb79..859c6f43e6 100644 --- a/webrtc/modules/video_processing/util/denoiser_filter_c.h +++ b/webrtc/modules/video_processing/util/denoiser_filter_c.h @@ -27,7 +27,7 @@ class DenoiserFilterC : public DenoiserFilter { const uint8_t* b, int b_stride, unsigned int* sse) override; - DenoiserDecision MbDenoise(uint8_t* mc_running_avg_y, + DenoiserDecision MbDenoise(const uint8_t* mc_running_avg_y, int mc_avg_y_stride, uint8_t* running_avg_y, int avg_y_stride, diff --git a/webrtc/modules/video_processing/util/denoiser_filter_neon.cc b/webrtc/modules/video_processing/util/denoiser_filter_neon.cc index 68c94cbdb7..b9e6b8c859 100644 --- a/webrtc/modules/video_processing/util/denoiser_filter_neon.cc +++ b/webrtc/modules/video_processing/util/denoiser_filter_neon.cc @@ -87,7 +87,7 @@ uint32_t DenoiserFilterNEON::Variance16x8(const uint8_t* a, return *sse - ((sum * sum) >> 7); } -DenoiserDecision DenoiserFilterNEON::MbDenoise(uint8_t* mc_running_avg_y, +DenoiserDecision DenoiserFilterNEON::MbDenoise(const uint8_t* mc_running_avg_y, int mc_running_avg_y_stride, uint8_t* running_avg_y, int running_avg_y_stride, diff --git a/webrtc/modules/video_processing/util/denoiser_filter_neon.h b/webrtc/modules/video_processing/util/denoiser_filter_neon.h index 55850bd1ea..076cb7902e 100644 --- a/webrtc/modules/video_processing/util/denoiser_filter_neon.h +++ b/webrtc/modules/video_processing/util/denoiser_filter_neon.h @@ -27,7 +27,7 @@ class DenoiserFilterNEON : public DenoiserFilter { const uint8_t* b, int b_stride, unsigned int* sse) override; - DenoiserDecision MbDenoise(uint8_t* mc_running_avg_y, + DenoiserDecision MbDenoise(const uint8_t* mc_running_avg_y, int mc_avg_y_stride, uint8_t* running_avg_y, int avg_y_stride, diff --git a/webrtc/modules/video_processing/util/denoiser_filter_sse2.cc b/webrtc/modules/video_processing/util/denoiser_filter_sse2.cc index 0545a97398..2e59e36d28 100644 --- a/webrtc/modules/video_processing/util/denoiser_filter_sse2.cc +++ b/webrtc/modules/video_processing/util/denoiser_filter_sse2.cc @@ -119,7 +119,7 @@ uint32_t DenoiserFilterSSE2::Variance16x8(const uint8_t* src, return *sse - ((sum * sum) >> 7); } -DenoiserDecision DenoiserFilterSSE2::MbDenoise(uint8_t* mc_running_avg_y, +DenoiserDecision DenoiserFilterSSE2::MbDenoise(const uint8_t* mc_running_avg_y, int mc_avg_y_stride, uint8_t* running_avg_y, int avg_y_stride, @@ -150,7 +150,7 @@ DenoiserDecision DenoiserFilterSSE2::MbDenoise(uint8_t* mc_running_avg_y, const __m128i v_sig = _mm_loadu_si128(reinterpret_cast(&sig[0])); const __m128i v_mc_running_avg_y = - _mm_loadu_si128(reinterpret_cast<__m128i*>(&mc_running_avg_y[0])); + _mm_loadu_si128(reinterpret_cast(&mc_running_avg_y[0])); __m128i v_running_avg_y; const __m128i pdiff = _mm_subs_epu8(v_mc_running_avg_y, v_sig); const __m128i ndiff = _mm_subs_epu8(v_sig, v_mc_running_avg_y); diff --git a/webrtc/modules/video_processing/util/denoiser_filter_sse2.h b/webrtc/modules/video_processing/util/denoiser_filter_sse2.h index 731344c809..5b2d957ad0 100644 --- a/webrtc/modules/video_processing/util/denoiser_filter_sse2.h +++ b/webrtc/modules/video_processing/util/denoiser_filter_sse2.h @@ -27,7 +27,7 @@ class DenoiserFilterSSE2 : public DenoiserFilter { const uint8_t* b, int b_stride, unsigned int* sse) override; - DenoiserDecision MbDenoise(uint8_t* mc_running_avg_y, + DenoiserDecision MbDenoise(const uint8_t* mc_running_avg_y, int mc_avg_y_stride, uint8_t* running_avg_y, int avg_y_stride, diff --git a/webrtc/modules/video_processing/video_denoiser.cc b/webrtc/modules/video_processing/video_denoiser.cc index 313a850b92..8c645f21cb 100644 --- a/webrtc/modules/video_processing/video_denoiser.cc +++ b/webrtc/modules/video_processing/video_denoiser.cc @@ -10,6 +10,7 @@ #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" #include "webrtc/modules/video_processing/video_denoiser.h" +#include "libyuv/planar_functions.h" namespace webrtc { @@ -30,37 +31,35 @@ static void ShowRect(const std::unique_ptr& filter, const std::unique_ptr& moving_edge_red, const std::unique_ptr& x_density, const std::unique_ptr& y_density, - const uint8_t* u_src, - const uint8_t* v_src, - uint8_t* u_dst, - uint8_t* v_dst, + const uint8_t* u_src, int stride_u_src, + const uint8_t* v_src, int stride_v_src, + uint8_t* u_dst, int stride_u_dst, + uint8_t* v_dst, int stride_v_dst, int mb_rows_, - int mb_cols_, - int stride_u_, - int stride_v_) { + int mb_cols_) { for (int mb_row = 0; mb_row < mb_rows_; ++mb_row) { for (int mb_col = 0; mb_col < mb_cols_; ++mb_col) { int mb_index = mb_row * mb_cols_ + mb_col; const uint8_t* mb_src_u = - u_src + (mb_row << 3) * stride_u_ + (mb_col << 3); + u_src + (mb_row << 3) * stride_u_src + (mb_col << 3); const uint8_t* mb_src_v = - v_src + (mb_row << 3) * stride_v_ + (mb_col << 3); - uint8_t* mb_dst_u = u_dst + (mb_row << 3) * stride_u_ + (mb_col << 3); - uint8_t* mb_dst_v = v_dst + (mb_row << 3) * stride_v_ + (mb_col << 3); + v_src + (mb_row << 3) * stride_v_src + (mb_col << 3); + uint8_t* mb_dst_u = u_dst + (mb_row << 3) * stride_u_dst + (mb_col << 3); + uint8_t* mb_dst_v = v_dst + (mb_row << 3) * stride_v_dst + (mb_col << 3); uint8_t uv_tmp[8 * 8]; memset(uv_tmp, 200, 8 * 8); if (d_status[mb_index] == 1) { // Paint to red. - CopyMem8x8(mb_src_u, stride_u_, mb_dst_u, stride_u_); - CopyMem8x8(uv_tmp, 8, mb_dst_v, stride_v_); + CopyMem8x8(mb_src_u, stride_u_src, mb_dst_u, stride_u_dst); + CopyMem8x8(uv_tmp, 8, mb_dst_v, stride_v_dst); } else if (moving_edge_red[mb_row * mb_cols_ + mb_col] && x_density[mb_col] * y_density[mb_row]) { // Paint to blue. - CopyMem8x8(uv_tmp, 8, mb_dst_u, stride_u_); - CopyMem8x8(mb_src_v, stride_v_, mb_dst_v, stride_v_); + CopyMem8x8(uv_tmp, 8, mb_dst_u, stride_u_dst); + CopyMem8x8(mb_src_v, stride_v_src, mb_dst_v, stride_v_dst); } else { - CopyMem8x8(mb_src_u, stride_u_, mb_dst_u, stride_u_); - CopyMem8x8(mb_src_v, stride_v_, mb_dst_v, stride_v_); + CopyMem8x8(mb_src_u, stride_u_src, mb_dst_u, stride_u_dst); + CopyMem8x8(mb_src_v, stride_v_src, mb_dst_v, stride_v_dst); } } } @@ -73,23 +72,11 @@ VideoDenoiser::VideoDenoiser(bool runtime_cpu_detection) filter_(DenoiserFilter::Create(runtime_cpu_detection, &cpu_type_)), ne_(new NoiseEstimation()) {} -void VideoDenoiser::DenoiserReset( - const rtc::scoped_refptr& frame, - rtc::scoped_refptr* denoised_frame, - rtc::scoped_refptr* denoised_frame_prev) { +void VideoDenoiser::DenoiserReset(rtc::scoped_refptr frame) { width_ = frame->width(); height_ = frame->height(); mb_cols_ = width_ >> 4; mb_rows_ = height_ >> 4; - stride_y_ = frame->StrideY(); - stride_u_ = frame->StrideU(); - stride_v_ = frame->StrideV(); - - // Allocate an empty buffer for denoised_frame_prev. - *denoised_frame_prev = I420Buffer::Create( - width_, height_, stride_y_, stride_u_, stride_v_); - // Allocate and initialize denoised_frame with key frame. - *denoised_frame = I420Buffer::CopyKeepStride(frame); // Init noise estimator and allocate buffers. ne_->Init(width_, height_, cpu_type_); @@ -176,14 +163,16 @@ bool VideoDenoiser::IsTrailingBlock(const std::unique_ptr& d_status, return ret; } -void VideoDenoiser::CopySrcOnMOB(const uint8_t* y_src, uint8_t* y_dst) { +void VideoDenoiser::CopySrcOnMOB(const uint8_t* y_src, + int stride_src, + uint8_t* y_dst, + int stride_dst) { // Loop over to copy src block if the block is marked as moving object block // or if the block may cause trailing artifacts. for (int mb_row = 0; mb_row < mb_rows_; ++mb_row) { const int mb_index_base = mb_row * mb_cols_; - const int offset_base = (mb_row << 4) * stride_y_; - const uint8_t* mb_src_base = y_src + offset_base; - uint8_t* mb_dst_base = y_dst + offset_base; + const uint8_t* mb_src_base = y_src + (mb_row << 4) * stride_src; + uint8_t* mb_dst_base = y_dst + (mb_row << 4) * stride_dst; for (int mb_col = 0; mb_col < mb_cols_; ++mb_col) { const int mb_index = mb_index_base + mb_col; const uint32_t offset_col = mb_col << 4; @@ -196,49 +185,55 @@ void VideoDenoiser::CopySrcOnMOB(const uint8_t* y_src, uint8_t* y_dst) { (x_density_[mb_col] * y_density_[mb_row] && moving_object_[mb_row * mb_cols_ + mb_col])) { // Copy y source. - filter_->CopyMem16x16(mb_src, stride_y_, mb_dst, stride_y_); + filter_->CopyMem16x16(mb_src, stride_src, mb_dst, stride_dst); } } } } -void VideoDenoiser::CopyLumaOnMargin(const uint8_t* y_src, uint8_t* y_dst) { - if ((mb_rows_ << 4) != height_) { - const uint8_t* margin_y_src = y_src + (mb_rows_ << 4) * stride_y_; - uint8_t* margin_y_dst = y_dst + (mb_rows_ << 4) * stride_y_; - memcpy(margin_y_dst, margin_y_src, (height_ - (mb_rows_ << 4)) * stride_y_); +void VideoDenoiser::CopyLumaOnMargin(const uint8_t* y_src, + int stride_src, + uint8_t* y_dst, + int stride_dst) { + int height_margin = height_ - (mb_rows_ << 4); + if (height_margin > 0) { + const uint8_t* margin_y_src = y_src + (mb_rows_ << 4) * stride_src; + uint8_t* margin_y_dst = y_dst + (mb_rows_ << 4) * stride_dst; + libyuv::CopyPlane(margin_y_src, stride_src, margin_y_dst, stride_dst, + width_, height_margin); } - if ((mb_cols_ << 4) != width_) { + int width_margin = width_ - (mb_cols_ << 4); + if (width_margin > 0) { const uint8_t* margin_y_src = y_src + (mb_cols_ << 4); uint8_t* margin_y_dst = y_dst + (mb_cols_ << 4); - for (int i = 0; i < height_; ++i) { - for (int j = mb_cols_ << 4; j < width_; ++j) { - margin_y_dst[i * stride_y_ + j] = margin_y_src[i * stride_y_ + j]; - } - } + libyuv::CopyPlane(margin_y_src, stride_src, margin_y_dst, stride_dst, + width_ - (mb_cols_ << 4), mb_rows_ << 4); } } -void VideoDenoiser::DenoiseFrame( - const rtc::scoped_refptr& frame, - rtc::scoped_refptr* denoised_frame, - rtc::scoped_refptr* denoised_frame_prev, +rtc::scoped_refptr VideoDenoiser::DenoiseFrame( + rtc::scoped_refptr frame, bool noise_estimation_enabled) { // If previous width and height are different from current frame's, need to // reallocate the buffers and no denoising for the current frame. - if (width_ != frame->width() || height_ != frame->height()) { - DenoiserReset(frame, denoised_frame, denoised_frame_prev); - return; + if (!prev_buffer_ || width_ != frame->width() || height_ != frame->height()) { + DenoiserReset(frame); + prev_buffer_ = frame; + return frame; } // Set buffer pointers. const uint8_t* y_src = frame->DataY(); - const uint8_t* u_src = frame->DataU(); - const uint8_t* v_src = frame->DataV(); - uint8_t* y_dst = (*denoised_frame)->MutableDataY(); - uint8_t* u_dst = (*denoised_frame)->MutableDataU(); - uint8_t* v_dst = (*denoised_frame)->MutableDataV(); - uint8_t* y_dst_prev = (*denoised_frame_prev)->MutableDataY(); + int stride_y_src = frame->StrideY(); + rtc::scoped_refptr dst = + buffer_pool_.CreateBuffer(width_, height_); + + uint8_t* y_dst = dst->MutableDataY(); + int stride_y_dst = dst->StrideY(); + + const uint8_t* y_dst_prev = prev_buffer_->DataY(); + int stride_prev = prev_buffer_->StrideY(); + memset(x_density_.get(), 0, mb_cols_); memset(y_density_.get(), 0, mb_rows_); memset(moving_object_.get(), 1, mb_cols_ * mb_rows_); @@ -249,10 +244,9 @@ void VideoDenoiser::DenoiseFrame( // factors for moving object detection. for (int mb_row = 0; mb_row < mb_rows_; ++mb_row) { const int mb_index_base = mb_row * mb_cols_; - const int offset_base = (mb_row << 4) * stride_y_; - const uint8_t* mb_src_base = y_src + offset_base; - uint8_t* mb_dst_base = y_dst + offset_base; - uint8_t* mb_dst_prev_base = y_dst_prev + offset_base; + const uint8_t* mb_src_base = y_src + (mb_row << 4) * stride_y_src; + uint8_t* mb_dst_base = y_dst + (mb_row << 4) * stride_y_dst; + const uint8_t* mb_dst_prev_base = y_dst_prev + (mb_row << 4) * stride_prev; for (int mb_col = 0; mb_col < mb_cols_; ++mb_col) { const int mb_index = mb_index_base + mb_col; const bool ne_enable = (mb_index % NOISE_SUBSAMPLE_INTERVAL == 0); @@ -261,22 +255,22 @@ void VideoDenoiser::DenoiseFrame( const uint32_t offset_col = mb_col << 4; const uint8_t* mb_src = mb_src_base + offset_col; uint8_t* mb_dst = mb_dst_base + offset_col; - uint8_t* mb_dst_prev = mb_dst_prev_base + offset_col; + const uint8_t* mb_dst_prev = mb_dst_prev_base + offset_col; // TODO(jackychen): Need SSE2/NEON opt. int luma = 0; if (ne_enable) { for (int i = 4; i < 12; ++i) { for (int j = 4; j < 12; ++j) { - luma += mb_src[i * stride_y_ + j]; + luma += mb_src[i * stride_y_src + j]; } } } // Get the filtered block and filter_decision. mb_filter_decision_[mb_index] = - filter_->MbDenoise(mb_dst_prev, stride_y_, mb_dst, stride_y_, mb_src, - stride_y_, 0, noise_level); + filter_->MbDenoise(mb_dst_prev, stride_prev, mb_dst, stride_y_dst, + mb_src, stride_y_src, 0, noise_level); // If filter decision is FILTER_BLOCK, no need to check moving edge. // It is unlikely for a moving edge block to be filtered in current @@ -286,8 +280,8 @@ void VideoDenoiser::DenoiseFrame( if (ne_enable) { // The variance used in noise estimation is based on the src block in // time t (mb_src) and filtered block in time t-1 (mb_dist_prev). - uint32_t noise_var = filter_->Variance16x8(mb_dst_prev, stride_y_, - mb_src, stride_y_, &sse_t); + uint32_t noise_var = filter_->Variance16x8( + mb_dst_prev, stride_y_dst, mb_src, stride_y_src, &sse_t); ne_->GetNoise(mb_index, noise_var, luma); } moving_edge_[mb_index] = 0; // Not a moving edge block. @@ -295,8 +289,8 @@ void VideoDenoiser::DenoiseFrame( uint32_t sse_t = 0; // The variance used in MOD is based on the filtered blocks in time // T (mb_dst) and T-1 (mb_dst_prev). - uint32_t noise_var = filter_->Variance16x8(mb_dst_prev, stride_y_, - mb_dst, stride_y_, &sse_t); + uint32_t noise_var = filter_->Variance16x8( + mb_dst_prev, stride_prev, mb_dst, stride_y_dst, &sse_t); if (noise_var > thr_var_adp) { // Moving edge checking. if (ne_enable) { ne_->ResetConsecLowVar(mb_index); @@ -310,7 +304,7 @@ void VideoDenoiser::DenoiseFrame( // The variance used in noise estimation is based on the src block // in time t (mb_src) and filtered block in time t-1 (mb_dist_prev). uint32_t noise_var = filter_->Variance16x8( - mb_dst_prev, stride_y_, mb_src, stride_y_, &sse_t); + mb_dst_prev, stride_prev, mb_src, stride_y_src, &sse_t); ne_->GetNoise(mb_index, noise_var, luma); } } @@ -320,23 +314,31 @@ void VideoDenoiser::DenoiseFrame( ReduceFalseDetection(moving_edge_, &moving_object_, noise_level); - CopySrcOnMOB(y_src, y_dst); + CopySrcOnMOB(y_src, stride_y_src, y_dst, stride_y_dst); // When frame width/height not divisible by 16, copy the margin to // denoised_frame. if ((mb_rows_ << 4) != height_ || (mb_cols_ << 4) != width_) - CopyLumaOnMargin(y_src, y_dst); + CopyLumaOnMargin(y_src, stride_y_src, y_dst, stride_y_dst); - // TODO(jackychen): Need SSE2/NEON opt. // Copy u/v planes. - memcpy(u_dst, u_src, (height_ >> 1) * stride_u_); - memcpy(v_dst, v_src, (height_ >> 1) * stride_v_); + libyuv::CopyPlane(frame->DataU(), frame->StrideU(), + dst->MutableDataU(), dst->StrideU(), + (width_ + 1) >> 1, (height_ + 1) >> 1); + libyuv::CopyPlane(frame->DataV(), frame->StrideV(), + dst->MutableDataV(), dst->StrideV(), + (width_ + 1) >> 1, (height_ + 1) >> 1); #if DISPLAY || DISPLAYNEON // Show rectangular region - ShowRect(filter_, moving_edge_, moving_object_, x_density_, y_density_, u_src, - v_src, u_dst, v_dst, mb_rows_, mb_cols_, stride_u_, stride_v_); + ShowRect(filter_, moving_edge_, moving_object_, x_density_, y_density_, + frame->DataU(), frame->StrideU(), frame->DataV(), frame->StrideV(), + dst->MutableDataU(), dst->StrideU(), + dst->MutableDataV(), dst->StrideV(), + mb_rows_, mb_cols_); #endif + prev_buffer_ = dst; + return dst; } } // namespace webrtc diff --git a/webrtc/modules/video_processing/video_denoiser.h b/webrtc/modules/video_processing/video_denoiser.h index 5293a9902b..e67bd59e68 100644 --- a/webrtc/modules/video_processing/video_denoiser.h +++ b/webrtc/modules/video_processing/video_denoiser.h @@ -13,6 +13,7 @@ #include +#include "webrtc/common_video/include/i420_buffer_pool.h" #include "webrtc/modules/video_processing/util/denoiser_filter.h" #include "webrtc/modules/video_processing/util/noise_estimation.h" #include "webrtc/modules/video_processing/util/skin_detection.h" @@ -23,21 +24,12 @@ class VideoDenoiser { public: explicit VideoDenoiser(bool runtime_cpu_detection); - // TODO(nisse): Let the denoised_frame and denoised_frame_prev be - // member variables referencing two I420Buffer, and return a refptr - // to the current one. When we also move the double-buffering logic - // from the caller. - void DenoiseFrame(const rtc::scoped_refptr& frame, - // Buffers are allocated/replaced when dimensions - // change. - rtc::scoped_refptr* denoised_frame, - rtc::scoped_refptr* denoised_frame_prev, - bool noise_estimation_enabled); + rtc::scoped_refptr DenoiseFrame( + rtc::scoped_refptr frame, + bool noise_estimation_enabled); private: - void DenoiserReset(const rtc::scoped_refptr& frame, - rtc::scoped_refptr* denoised_frame, - rtc::scoped_refptr* denoised_frame_prev); + void DenoiserReset(rtc::scoped_refptr frame); // Check the mb position, return 1: close to the frame center (between 1/8 // and 7/8 of width/height), 3: close to the border (out of 1/16 and 15/16 @@ -56,18 +48,21 @@ class VideoDenoiser { int mb_col); // Copy input blocks to dst buffer on moving object blocks (MOB). - void CopySrcOnMOB(const uint8_t* y_src, uint8_t* y_dst); + void CopySrcOnMOB(const uint8_t* y_src, + int stride_src, + uint8_t* y_dst, + int stride_dst); // Copy luma margin blocks when frame width/height not divisible by 16. - void CopyLumaOnMargin(const uint8_t* y_src, uint8_t* y_dst); + void CopyLumaOnMargin(const uint8_t* y_src, + int stride_src, + uint8_t* y_dst, + int stride_dst); int width_; int height_; int mb_rows_; int mb_cols_; - int stride_y_; - int stride_u_; - int stride_v_; CpuType cpu_type_; std::unique_ptr filter_; std::unique_ptr ne_; @@ -80,6 +75,8 @@ class VideoDenoiser { std::unique_ptr y_density_; // Save the return values by MbDenoise for each block. std::unique_ptr mb_filter_decision_; + I420BufferPool buffer_pool_; + rtc::scoped_refptr prev_buffer_; }; } // namespace webrtc