From 5054f544575b1a0471b241266c6fc8c2ccf93af0 Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Thu, 31 Jan 2019 14:35:11 +0100 Subject: [PATCH] Partial frame capture API part 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement test utility for extracting changed part of video frames. Bug: webrtc:10152 Change-Id: Iead052d2a18384aaa828cd7821be961b8614568e Reviewed-on: https://webrtc-review.googlesource.com/c/120407 Commit-Queue: Ilya Nikolaevskiy Reviewed-by: Erik Språng Cr-Commit-Position: refs/heads/master@{#26496} --- test/BUILD.gn | 2 + test/frame_change_extractor.cc | 158 +++++++++++++++++++++++++++++++++ test/frame_change_extractor.h | 50 +++++++++++ 3 files changed, 210 insertions(+) create mode 100644 test/frame_change_extractor.cc create mode 100644 test/frame_change_extractor.h diff --git a/test/BUILD.gn b/test/BUILD.gn index 5981af2ea5..fa0f4c71ae 100644 --- a/test/BUILD.gn +++ b/test/BUILD.gn @@ -41,6 +41,8 @@ rtc_source_set("video_test_common") { sources = [ "fake_texture_frame.cc", "fake_texture_frame.h", + "frame_change_extractor.cc", + "frame_change_extractor.h", "frame_generator.cc", "frame_generator.h", "frame_generator_capturer.cc", diff --git a/test/frame_change_extractor.cc b/test/frame_change_extractor.cc new file mode 100644 index 0000000000..3a5196361d --- /dev/null +++ b/test/frame_change_extractor.cc @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "test/frame_change_extractor.h" + +#include + +#include "api/video/i420_buffer.h" +#include "rtc_base/checks.h" + +namespace webrtc { + +FrameChangeExtractor::FrameChangeExtractor() = default; +FrameChangeExtractor::~FrameChangeExtractor() = default; + +void FrameChangeExtractor::SetSource( + rtc::VideoSourceInterface* video_source) { + rtc::CritScope lock(&source_crit_); + + rtc::VideoSourceInterface* old_source = nullptr; + old_source = source_; + source_ = video_source; + if (old_source != video_source && old_source != nullptr) { + old_source->RemoveSink(this); + } + if (video_source) { + source_->AddOrUpdateSink(this, rtc::VideoSinkWants()); + } +} + +void FrameChangeExtractor::OnFrame(const VideoFrame& frame) { + // Currently only supports I420 buffers. + RTC_CHECK(frame.video_frame_buffer()->type() == + VideoFrameBuffer::Type::kI420); + + rtc::CritScope lock(&sink_crit_); + + if (!sink_) + return; + + webrtc::I420BufferInterface* current_buffer = + frame.video_frame_buffer()->ToI420(); + int min_row, min_col, max_row, max_col; + if (!last_frame_buffer_ || + current_buffer->width() != last_frame_buffer_->width() || + current_buffer->height() != last_frame_buffer_->height()) { + // New resolution - fully new picture. + min_row = min_col = 0; + max_row = current_buffer->height() - 1; + max_col = current_buffer->width() - 1; + } else { + min_row = frame.width(); + min_col = frame.height(); + max_row = 0; + max_col = 0; + // Detect changed rect. + for (int row = 0; row < frame.height(); ++row) { + for (int col = 0; col < frame.width(); ++col) { + int uv_row = row / 2; + int uv_col = col / 2; + if (current_buffer + ->DataU()[uv_row * current_buffer->StrideU() + uv_col] != + last_frame_buffer_ + ->DataU()[uv_row * last_frame_buffer_->StrideU() + + uv_col] || + current_buffer + ->DataV()[uv_row * current_buffer->StrideV() + uv_col] != + last_frame_buffer_ + ->DataV()[uv_row * last_frame_buffer_->StrideV() + + uv_col] || + current_buffer->DataY()[row * current_buffer->StrideY() + col] != + last_frame_buffer_ + ->DataY()[row * last_frame_buffer_->StrideY() + col]) { + min_row = std::min(min_row, row); + max_row = std::max(max_row, row); + min_col = std::min(min_col, col); + max_col = std::max(max_col, col); + } + } + } + } + if (max_row < min_row || max_col < min_col) { + min_col = min_row = 0; + max_col = max_row = -1; + } + // Expand changed rect to accommodate subsampled UV plane. + if (min_row % 2) + --min_row; + if (min_col % 2) + --min_row; + if (max_row < frame.width() && max_row % 2 == 0) + ++max_row; + if (max_col < frame.height() && max_col % 2 == 0) + ++max_col; + VideoFrame::PartialFrameDescription part_desc; + part_desc.offset_x = min_col; + part_desc.offset_y = min_row; + int changed_width = max_col - min_col + 1; + int changed_height = max_row - min_row + 1; + last_frame_buffer_ = current_buffer; + VideoFrame copy = frame; + if (changed_width > 0 && changed_height > 0) { + rtc::scoped_refptr changed_buffer = + I420Buffer::Create(changed_width, changed_height); + changed_buffer->CropAndScaleFrom(*current_buffer, part_desc.offset_x, + part_desc.offset_y, changed_width, + changed_height); + copy.set_video_frame_buffer(changed_buffer); + } else { + copy.set_video_frame_buffer(nullptr); + } + if (changed_width < frame.width() || changed_height < frame.height()) { + copy.set_partial_frame_description(part_desc); + } + copy.set_cache_buffer_for_partial_updates(true); + sink_->OnFrame(copy); +} + +void FrameChangeExtractor::AddOrUpdateSink( + rtc::VideoSinkInterface* sink, + const rtc::VideoSinkWants& wants) { + RTC_CHECK(wants.partial_frames); + { + rtc::CritScope lock(&source_crit_); + if (source_) { + source_->AddOrUpdateSink(this, wants); + } + } + { + rtc::CritScope lock(&sink_crit_); + // Several sinks are unsupported. + RTC_CHECK(sink_ == nullptr || sink_ == sink); + sink_ = sink; + } +} + +void FrameChangeExtractor::RemoveSink( + rtc::VideoSinkInterface* sink) { + { + rtc::CritScope lock(&source_crit_); + if (source_) { + source_->RemoveSink(this); + } + } + { + rtc::CritScope lock(&sink_crit_); + sink_ = nullptr; + } +} + +} // namespace webrtc diff --git a/test/frame_change_extractor.h b/test/frame_change_extractor.h new file mode 100644 index 0000000000..ea24da6b29 --- /dev/null +++ b/test/frame_change_extractor.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef TEST_FRAME_CHANGE_EXTRACTOR_H_ +#define TEST_FRAME_CHANGE_EXTRACTOR_H_ + +#include "api/video/video_frame.h" +#include "api/video/video_sink_interface.h" +#include "api/video/video_source_interface.h" +#include "rtc_base/critical_section.h" + +namespace webrtc { + +// This class is used to test partial video frame interface. +// It receives a full frame, detects changed region, and passes forward only +// the changed part with correct PartialDescription struct set. +// Currently only works with I420 buffers. +class FrameChangeExtractor : public rtc::VideoSinkInterface, + public rtc::VideoSourceInterface { + public: + FrameChangeExtractor(); + ~FrameChangeExtractor(); + void SetSource(rtc::VideoSourceInterface* video_source); + + // Implements rtc::VideoSinkInterface. + void OnFrame(const VideoFrame& frame) override; + + // Implements rtc::VideoSourceInterface. + void AddOrUpdateSink(rtc::VideoSinkInterface* sink, + const rtc::VideoSinkWants& wants) override; + void RemoveSink(rtc::VideoSinkInterface* sink) override; + + private: + rtc::CriticalSection source_crit_; + rtc::VideoSourceInterface* source_ RTC_GUARDED_BY(source_crit_); + rtc::CriticalSection sink_crit_; + rtc::VideoSinkInterface* sink_ RTC_GUARDED_BY(sink_crit_); + rtc::scoped_refptr last_frame_buffer_ + RTC_GUARDED_BY(sink_crit_); +}; + +} // namespace webrtc +#endif // TEST_FRAME_CHANGE_EXTRACTOR_H_