zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement automatic animation detection in VideoStreamEncoder

Browse Source 
If WebRTC-AutomaticAnimationDetectionScreenshare experiment is enabled,
content type is screenshare and degradation preference is BALANCED,
then input resolution is restricted if update_rect of the incoming frames
is the same for considerable amount of time and is big enough.

This entails treating BALANCED degradation preference for screenshare as
MAINTAIN_RESOLUTION in adaptation logic.

Bug: webrtc:11058
Change-Id: I903dddf53fcbd7c8eac6c5b1447225b15fd8fe5f
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/161097
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30002}
This commit is contained in:
Ilya Nikolaevskiy
2019-12-03 16:54:17 +01:00
committed by Commit Bot
parent 32565f684b
commit 648b9d77c7
4 changed files with 258 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
test/test_video_capturer.cc
View File

@ -39,15 +39,24 @@ void TestVideoCapturer::OnFrame(const VideoFrame& original_frame) {
if (out_height != frame.height() || out_width != frame.width()) {
// Video adapter has requested a down-scale. Allocate a new buffer and
// return scaled version.
// For simplicity, only scale here without cropping.
rtc::scoped_refptr<I420Buffer> scaled_buffer =
I420Buffer::Create(out_width, out_height);
scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420());
broadcaster_.OnFrame(VideoFrame::Builder()
.set_video_frame_buffer(scaled_buffer)
.set_rotation(kVideoRotation_0)
.set_timestamp_us(frame.timestamp_us())
.set_id(frame.id())
.build());
VideoFrame::Builder new_frame_builder =
VideoFrame::Builder()
.set_video_frame_buffer(scaled_buffer)
.set_rotation(kVideoRotation_0)
.set_timestamp_us(frame.timestamp_us())
.set_id(frame.id());
if (frame.has_update_rect()) {
VideoFrame::UpdateRect new_rect = frame.update_rect().ScaleWithFrame(
frame.width(), frame.height(), 0, 0, frame.width(), frame.height(),
out_width, out_height);
new_frame_builder.set_update_rect(new_rect);
}
broadcaster_.OnFrame(new_frame_builder.build());
} else {
// No adaptations needed, just return the frame as is.
broadcaster_.OnFrame(frame);

143
video/video_stream_encoder.cc
View File

@ -68,6 +68,9 @@ const size_t kDefaultPayloadSize = 1440;
const int64_t kParameterUpdateIntervalMs = 1000;
// Animation is capped to 720p.
constexpr int kMaxAnimationPixels = 1280 * 720;
uint32_t abs_diff(uint32_t a, uint32_t b) {
return (a < b) ? b - a : a - b;
}
@ -219,7 +222,8 @@ class VideoStreamEncoder::VideoSourceProxy {
: video_stream_encoder_(video_stream_encoder),
degradation_preference_(DegradationPreference::DISABLED),
source_(nullptr),
max_framerate_(std::numeric_limits<int>::max()) {}
max_framerate_(std::numeric_limits<int>::max()),
max_pixels_(std::numeric_limits<int>::max()) {}
void SetSource(rtc::VideoSourceInterface<VideoFrame>* source,
const DegradationPreference& degradation_preference) {
@ -407,6 +411,22 @@ class VideoStreamEncoder::VideoSourceProxy {
return true;
}
// Used in automatic animation detection for screenshare.
bool RestrictPixels(int max_pixels) {
// Called on the encoder task queue.
rtc::CritScope lock(&crit_);
if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) {
// This can happen since |degradation_preference_| is set on libjingle's
// worker thread but the adaptation is done on the encoder task queue.
return false;
}
max_pixels_ = max_pixels;
RTC_LOG(LS_INFO) << "Applying max pixel restriction: " << max_pixels;
source_->AddOrUpdateSink(video_stream_encoder_,
GetActiveSinkWantsInternal());
return true;
}
private:
rtc::VideoSinkWants GetActiveSinkWantsInternal()
RTC_EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
@ -430,6 +450,9 @@ class VideoStreamEncoder::VideoSourceProxy {
}
// Limit to configured max framerate.
wants.max_framerate_fps = std::min(max_framerate_, wants.max_framerate_fps);
// Limit resolution due to automatic animation detection for screenshare.
wants.max_pixel_count = std::min(max_pixels_, wants.max_pixel_count);
return wants;
}
@ -440,6 +463,7 @@ class VideoStreamEncoder::VideoSourceProxy {
DegradationPreference degradation_preference_ RTC_GUARDED_BY(&crit_);
rtc::VideoSourceInterface<VideoFrame>* source_ RTC_GUARDED_BY(&crit_);
int max_framerate_ RTC_GUARDED_BY(&crit_);
int max_pixels_ RTC_GUARDED_BY(&crit_);
RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy);
};
@ -519,6 +543,9 @@ VideoStreamEncoder::VideoStreamEncoder(
pending_frame_post_time_us_(0),
accumulated_update_rect_{0, 0, 0, 0},
accumulated_update_rect_is_valid_(true),
animation_start_time_(Timestamp::PlusInfinity()),
cap_resolution_due_to_video_content_(false),
expect_resize_state_(ExpectResizeState::kNoResize),
bitrate_observer_(nullptr),
fec_controller_override_(nullptr),
force_disable_frame_dropper_(false),
@ -529,6 +556,8 @@ VideoStreamEncoder::VideoStreamEncoder(
experiment_groups_(GetExperimentGroups()),
next_frame_id_(0),
encoder_switch_experiment_(ParseEncoderSwitchFieldTrial()),
automatic_animation_detection_experiment_(
ParseAutomatincAnimationDetectionFieldTrial()),
encoder_switch_requested_(false),
encoder_queue_(task_queue_factory->CreateTaskQueue(
"EncoderQueue",
@ -1114,6 +1143,7 @@ void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) {
const int posted_frames_waiting_for_encode =
posted_frames_waiting_for_encode_.fetch_sub(1);
RTC_DCHECK_GT(posted_frames_waiting_for_encode, 0);
CheckForAnimatedContent(incoming_frame, post_time_us);
if (posted_frames_waiting_for_encode == 1) {
MaybeEncodeVideoFrame(incoming_frame, post_time_us);
} else {
@ -1951,7 +1981,7 @@ bool VideoStreamEncoder::AdaptDown(AdaptReason reason) {
bool did_adapt = true;
switch (degradation_preference_) {
switch (EffectiveDegradataionPreference()) {
case DegradationPreference::BALANCED:
break;
case DegradationPreference::MAINTAIN_FRAMERATE:
@ -1980,7 +2010,7 @@ bool VideoStreamEncoder::AdaptDown(AdaptReason reason) {
return true;
}
switch (degradation_preference_) {
switch (EffectiveDegradataionPreference()) {
case DegradationPreference::BALANCED: {
// Try scale down framerate, if lower.
int fps = balanced_settings_.MinFps(encoder_config_.codec_type,
@ -2057,7 +2087,8 @@ void VideoStreamEncoder::AdaptUp(AdaptReason reason) {
last_adaptation_request_ &&
last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
if (degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE) {
if (EffectiveDegradataionPreference() ==
DegradationPreference::MAINTAIN_FRAMERATE) {
if (adapt_up_requested &&
adaptation_request.input_pixel_count_ <=
last_adaptation_request_->input_pixel_count_) {
@ -2067,7 +2098,7 @@ void VideoStreamEncoder::AdaptUp(AdaptReason reason) {
}
}
switch (degradation_preference_) {
switch (EffectiveDegradataionPreference()) {
case DegradationPreference::BALANCED: {
// Check if quality should be increased based on bitrate.
if (reason == kQuality &&
@ -2494,4 +2525,106 @@ VideoStreamEncoder::ParseEncoderSwitchFieldTrial() const {
return result;
}
VideoStreamEncoder::AutomaticAnimationDetectionExperiment
VideoStreamEncoder::ParseAutomatincAnimationDetectionFieldTrial() const {
AutomaticAnimationDetectionExperiment result;
result.Parser()->Parse(webrtc::field_trial::FindFullName(
"WebRTC-AutomaticAnimationDetectionScreenshare"));
if (!result.enabled) {
RTC_LOG(LS_INFO) << "Automatic animation detection experiment is disabled.";
return result;
}
RTC_LOG(LS_INFO) << "Automatic animation detection experiment settings:"
<< " min_duration_ms=" << result.min_duration_ms
<< " min_area_ration=" << result.min_area_ratio
<< " min_fps=" << result.min_fps;
return result;
}
void VideoStreamEncoder::CheckForAnimatedContent(
const VideoFrame& frame,
int64_t time_when_posted_in_us) {
if (!automatic_animation_detection_experiment_.enabled ||
encoder_config_.content_type !=
VideoEncoderConfig::ContentType::kScreen ||
degradation_preference_ != DegradationPreference::BALANCED) {
return;
}
if (expect_resize_state_ == ExpectResizeState::kResize && last_frame_info_ &&
last_frame_info_->width != frame.width() &&
last_frame_info_->height != frame.height()) {
// On applying resolution cap there will be one frame with no/different
// update, which should be skipped.
// It can be delayed by several frames.
expect_resize_state_ = ExpectResizeState::kFirstFrameAfterResize;
return;
}
if (expect_resize_state_ == ExpectResizeState::kFirstFrameAfterResize) {
// The first frame after resize should have new, scaled update_rect.
if (frame.has_update_rect()) {
last_update_rect_ = frame.update_rect();
} else {
last_update_rect_ = absl::nullopt;
}
expect_resize_state_ = ExpectResizeState::kNoResize;
}
bool should_cap_resolution = false;
if (!frame.has_update_rect()) {
last_update_rect_ = absl::nullopt;
animation_start_time_ = Timestamp::PlusInfinity();
} else if ((!last_update_rect_ ||
frame.update_rect() != *last_update_rect_)) {
last_update_rect_ = frame.update_rect();
animation_start_time_ = Timestamp::us(time_when_posted_in_us);
} else {
TimeDelta animation_duration =
Timestamp::us(time_when_posted_in_us) - animation_start_time_;
float area_ratio = static_cast<float>(last_update_rect_->width *
last_update_rect_->height) /
(frame.width() * frame.height());
if (animation_duration.ms() >=
automatic_animation_detection_experiment_.min_duration_ms &&
area_ratio >=
automatic_animation_detection_experiment_.min_area_ratio &&
encoder_stats_observer_->GetInputFrameRate() >=
automatic_animation_detection_experiment_.min_fps) {
should_cap_resolution = true;
}
}
if (cap_resolution_due_to_video_content_ != should_cap_resolution) {
expect_resize_state_ = should_cap_resolution ? ExpectResizeState::kResize
: ExpectResizeState::kNoResize;
cap_resolution_due_to_video_content_ = should_cap_resolution;
if (should_cap_resolution) {
RTC_LOG(LS_INFO) << "Applying resolution cap due to animation detection.";
} else {
RTC_LOG(LS_INFO) << "Removing resolution cap due to no consistent "
"animation detection.";
}
source_proxy_->RestrictPixels(should_cap_resolution
? kMaxAnimationPixels
: std::numeric_limits<int>::max());
}
}
DegradationPreference VideoStreamEncoder::EffectiveDegradataionPreference()
const {
// Balanced mode for screenshare works via automatic animation detection:
// Resolution is capped for fullscreen animated content.
// Adapatation is done only via framerate downgrade.
// Thus effective degradation preference is MAINTAIN_RESOLUTION.
return (encoder_config_.content_type ==
VideoEncoderConfig::ContentType::kScreen &&
degradation_preference_ == DegradationPreference::BALANCED)
? DegradationPreference::MAINTAIN_RESOLUTION
: degradation_preference_;
}
} // namespace webrtc

41
video/video_stream_encoder.h
View File

@ -234,6 +234,14 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
bool HasInternalSource() const RTC_RUN_ON(&encoder_queue_);
void ReleaseEncoder() RTC_RUN_ON(&encoder_queue_);
void CheckForAnimatedContent(const VideoFrame& frame,
int64_t time_when_posted_in_ms)
RTC_RUN_ON(&encoder_queue_);
// Calculates degradation preference used in adaptation down or up.
DegradationPreference EffectiveDegradataionPreference() const
RTC_RUN_ON(&encoder_queue_);
rtc::Event shutdown_event_;
const uint32_t number_of_cores_;
@ -344,6 +352,19 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
RTC_GUARDED_BY(&encoder_queue_);
bool accumulated_update_rect_is_valid_ RTC_GUARDED_BY(&encoder_queue_);
// Used for automatic content type detection.
absl::optional<VideoFrame::UpdateRect> last_update_rect_
RTC_GUARDED_BY(&encoder_queue_);
Timestamp animation_start_time_ RTC_GUARDED_BY(&encoder_queue_);
bool cap_resolution_due_to_video_content_ RTC_GUARDED_BY(&encoder_queue_);
// Used to correctly ignore changes in update_rect introduced by
// resize triggered by animation detection.
enum class ExpectResizeState {
kNoResize, // Normal operation.
kResize, // Resize was triggered by the animation detection.
kFirstFrameAfterResize // Resize observed.
} expect_resize_state_ RTC_GUARDED_BY(&encoder_queue_);
VideoBitrateAllocationObserver* bitrate_observer_
RTC_GUARDED_BY(&encoder_queue_);
FecControllerOverride* fec_controller_override_
@ -428,6 +449,26 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
EncoderSwitchExperiment encoder_switch_experiment_
RTC_GUARDED_BY(&encoder_queue_);
struct AutomaticAnimationDetectionExperiment {
bool enabled = false;
int min_duration_ms = 2000;
double min_area_ratio = 0.8;
int min_fps = 10;
std::unique_ptr<StructParametersParser> Parser() {
return StructParametersParser::Create(
"enabled", &enabled, //
"min_duration_ms", &min_duration_ms, //
"min_area_ratio", &min_area_ratio, //
"min_fps", &min_fps);
}
};
AutomaticAnimationDetectionExperiment
ParseAutomatincAnimationDetectionFieldTrial() const;
AutomaticAnimationDetectionExperiment
automatic_animation_detection_experiment_ RTC_GUARDED_BY(&encoder_queue_);
// An encoder switch is only requested once, this variable is used to keep
// track of whether a request has been made or not.
bool encoder_switch_requested_ RTC_GUARDED_BY(&encoder_queue_);

64
video/video_stream_encoder_unittest.cc
View File

@ -301,6 +301,13 @@ class AdaptingFrameForwarder : public test::FrameForwarder {
.set_rotation(kVideoRotation_0)
.build();
adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
if (video_frame.has_update_rect()) {
adapted_frame.set_update_rect(
video_frame.update_rect().ScaleWithFrame(
video_frame.width(), video_frame.height(), 0, 0,
video_frame.width(), video_frame.height(), out_width,
out_height));
}
test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
last_width_.emplace(adapted_frame.width());
last_height_.emplace(adapted_frame.height());
@ -5201,4 +5208,61 @@ TEST_F(VideoStreamEncoderTest,
video_stream_encoder_->Stop();
}
TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
test::ScopedFieldTrials field_trials(
"WebRTC-AutomaticAnimationDetectionScreenshare/"
"enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
const int kFramerateFps = 30;
const int kWidth = 1920;
const int kHeight = 1080;
const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
// Works on screenshare mode.
ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
// We rely on the automatic resolution adaptation, but we handle framerate
// adaptation manually by mocking the stats proxy.
video_source_.set_adaptation_enabled(true);
// BALANCED degradation preference is required for this feature.
video_stream_encoder_->OnBitrateUpdated(
DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
DataRate::bps(kTargetBitrateBps), 0, 0);
video_stream_encoder_->SetSource(&video_source_,
webrtc::DegradationPreference::BALANCED);
VerifyNoLimitation(video_source_.sink_wants());
VideoFrame frame = CreateFrame(1, kWidth, kHeight);
frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
// Pass enough frames with the full update to trigger animation detection.
for (int i = 0; i < kNumFrames; ++i) {
int64_t timestamp_ms =
fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
frame.set_ntp_time_ms(timestamp_ms);
frame.set_timestamp_us(timestamp_ms * 1000);
video_source_.IncomingCapturedFrame(frame);
WaitForEncodedFrame(timestamp_ms);
}
// Resolution should be limited.
rtc::VideoSinkWants expected;
expected.max_framerate_fps = kFramerateFps;
expected.max_pixel_count = 1280 * 720 + 1;
VerifyFpsEqResolutionLt(video_source_.sink_wants(), expected);
// Pass one frame with no known update.
// Resolution cap should be removed immediately.
int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
frame.set_ntp_time_ms(timestamp_ms);
frame.set_timestamp_us(timestamp_ms * 1000);
frame.clear_update_rect();
video_source_.IncomingCapturedFrame(frame);
WaitForEncodedFrame(timestamp_ms);
// Resolution should be unlimited now.
VerifyFpsEqResolutionMax(video_source_.sink_wants(), kFramerateFps);
video_stream_encoder_->Stop();
}
} // namespace webrtc