Use bitrate limits provided by encoder.
- Use minimum start bitrate to drop frame and adapt resolution in the beginning of call. - Use minimum bitrate to decide whether or not resolution should be increased based on quality in MAINTAIN_FRAMERATE and BALANCED modes. In BALANCED mode bitrate limits provided by the corresponding field trial are prioritized over the limits provided by encoder. Bug: webrtc:10853 Change-Id: I8257eb64565bcafa6ae9887a1af18e90f8400cac Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/156302 Commit-Queue: Sergey Silkin <ssilkin@webrtc.org> Reviewed-by: Åsa Persson <asapersson@webrtc.org> Cr-Commit-Position: refs/heads/master@{#29461}
This commit is contained in:

committed by
Commit Bot

parent
5ab79e62f6
commit
41c650bea2
@ -320,6 +320,16 @@ class VideoStreamEncoder::VideoSourceProxy {
|
|||||||
return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1;
|
return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int GetHigherResolutionThan(int pixel_count) const {
|
||||||
|
// On step down we request at most 3/5 the pixel count of the previous
|
||||||
|
// resolution, so in order to take "one step up" we request a resolution
|
||||||
|
// as close as possible to 5/3 of the current resolution. The actual pixel
|
||||||
|
// count selected depends on the capabilities of the source. In order to
|
||||||
|
// not take a too large step up, we cap the requested pixel count to be at
|
||||||
|
// most four time the current number of pixels.
|
||||||
|
return (pixel_count * 5) / 3;
|
||||||
|
}
|
||||||
|
|
||||||
bool RequestHigherResolutionThan(int pixel_count) {
|
bool RequestHigherResolutionThan(int pixel_count) {
|
||||||
// Called on the encoder task queue.
|
// Called on the encoder task queue.
|
||||||
rtc::CritScope lock(&crit_);
|
rtc::CritScope lock(&crit_);
|
||||||
@ -340,13 +350,7 @@ class VideoStreamEncoder::VideoSourceProxy {
|
|||||||
// Remove any constraints.
|
// Remove any constraints.
|
||||||
sink_wants_.target_pixel_count.reset();
|
sink_wants_.target_pixel_count.reset();
|
||||||
} else {
|
} else {
|
||||||
// On step down we request at most 3/5 the pixel count of the previous
|
sink_wants_.target_pixel_count = GetHigherResolutionThan(pixel_count);
|
||||||
// resolution, so in order to take "one step up" we request a resolution
|
|
||||||
// as close as possible to 5/3 of the current resolution. The actual pixel
|
|
||||||
// count selected depends on the capabilities of the source. In order to
|
|
||||||
// not take a too large step up, we cap the requested pixel count to be at
|
|
||||||
// most four time the current number of pixels.
|
|
||||||
sink_wants_.target_pixel_count = (pixel_count * 5) / 3;
|
|
||||||
}
|
}
|
||||||
RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: "
|
RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: "
|
||||||
<< max_pixels_wanted;
|
<< max_pixels_wanted;
|
||||||
@ -1861,14 +1865,25 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate,
|
|||||||
}
|
}
|
||||||
|
|
||||||
bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const {
|
bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const {
|
||||||
if (initial_framedrop_ < kMaxInitialFramedrop &&
|
if (initial_framedrop_ >= kMaxInitialFramedrop ||
|
||||||
encoder_start_bitrate_bps_ > 0) {
|
encoder_start_bitrate_bps_ == 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
absl::optional<VideoEncoder::ResolutionBitrateLimits> encoder_bitrate_limits =
|
||||||
|
GetEncoderBitrateLimits(encoder_->GetEncoderInfo(), pixel_count);
|
||||||
|
|
||||||
|
if (encoder_bitrate_limits.has_value()) {
|
||||||
|
// Use bitrate limits provided by encoder.
|
||||||
|
return encoder_start_bitrate_bps_ <
|
||||||
|
static_cast<uint32_t>(encoder_bitrate_limits->min_start_bitrate_bps);
|
||||||
|
}
|
||||||
|
|
||||||
if (encoder_start_bitrate_bps_ < 300000 /* qvga */) {
|
if (encoder_start_bitrate_bps_ < 300000 /* qvga */) {
|
||||||
return pixel_count > 320 * 240;
|
return pixel_count > 320 * 240;
|
||||||
} else if (encoder_start_bitrate_bps_ < 500000 /* vga */) {
|
} else if (encoder_start_bitrate_bps_ < 500000 /* vga */) {
|
||||||
return pixel_count > 640 * 480;
|
return pixel_count > 640 * 480;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2032,6 +2047,14 @@ void VideoStreamEncoder::AdaptUp(AdaptReason reason) {
|
|||||||
RTC_FALLTHROUGH();
|
RTC_FALLTHROUGH();
|
||||||
}
|
}
|
||||||
case DegradationPreference::MAINTAIN_FRAMERATE: {
|
case DegradationPreference::MAINTAIN_FRAMERATE: {
|
||||||
|
// Check if resolution should be increased based on bitrate and
|
||||||
|
// limits specified by encoder capabilities.
|
||||||
|
if (reason == kQuality &&
|
||||||
|
!CanAdaptUpResolution(last_frame_info_->pixel_count(),
|
||||||
|
encoder_start_bitrate_bps_)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Scale up resolution.
|
// Scale up resolution.
|
||||||
int pixel_count = adaptation_request.input_pixel_count_;
|
int pixel_count = adaptation_request.input_pixel_count_;
|
||||||
if (adapt_counter.ResolutionCount() == 1) {
|
if (adapt_counter.ResolutionCount() == 1) {
|
||||||
@ -2073,6 +2096,19 @@ void VideoStreamEncoder::AdaptUp(AdaptReason reason) {
|
|||||||
RTC_LOG(LS_INFO) << adapt_counter.ToString();
|
RTC_LOG(LS_INFO) << adapt_counter.ToString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool VideoStreamEncoder::CanAdaptUpResolution(int pixels,
|
||||||
|
uint32_t bitrate_bps) const {
|
||||||
|
absl::optional<VideoEncoder::ResolutionBitrateLimits> bitrate_limits =
|
||||||
|
GetEncoderBitrateLimits(encoder_info_,
|
||||||
|
source_proxy_->GetHigherResolutionThan(pixels));
|
||||||
|
if (!bitrate_limits.has_value() || bitrate_bps == 0) {
|
||||||
|
return true; // No limit configured or bitrate provided.
|
||||||
|
}
|
||||||
|
RTC_DCHECK_GE(bitrate_limits->frame_size_pixels, pixels);
|
||||||
|
return bitrate_bps >=
|
||||||
|
static_cast<uint32_t>(bitrate_limits->min_start_bitrate_bps);
|
||||||
|
}
|
||||||
|
|
||||||
// TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged.
|
// TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged.
|
||||||
void VideoStreamEncoder::UpdateAdaptationStats(AdaptReason reason) {
|
void VideoStreamEncoder::UpdateAdaptationStats(AdaptReason reason) {
|
||||||
switch (reason) {
|
switch (reason) {
|
||||||
|
@ -223,6 +223,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
|
|||||||
void UpdateAdaptationStats(AdaptReason reason) RTC_RUN_ON(&encoder_queue_);
|
void UpdateAdaptationStats(AdaptReason reason) RTC_RUN_ON(&encoder_queue_);
|
||||||
VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts(
|
VideoStreamEncoderObserver::AdaptationSteps GetActiveCounts(
|
||||||
AdaptReason reason) RTC_RUN_ON(&encoder_queue_);
|
AdaptReason reason) RTC_RUN_ON(&encoder_queue_);
|
||||||
|
bool CanAdaptUpResolution(int pixels, uint32_t bitrate_bps) const
|
||||||
|
RTC_RUN_ON(&encoder_queue_);
|
||||||
void RunPostEncode(EncodedImage encoded_image,
|
void RunPostEncode(EncodedImage encoded_image,
|
||||||
int64_t time_sent_us,
|
int64_t time_sent_us,
|
||||||
int temporal_index);
|
int temporal_index);
|
||||||
|
@ -67,6 +67,10 @@ const int kMaxInitialFramedrop = 4;
|
|||||||
const int kDefaultFramerate = 30;
|
const int kDefaultFramerate = 30;
|
||||||
const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
|
const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
|
||||||
const int64_t kProcessIntervalMs = 1000;
|
const int64_t kProcessIntervalMs = 1000;
|
||||||
|
const VideoEncoder::ResolutionBitrateLimits
|
||||||
|
kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
|
||||||
|
const VideoEncoder::ResolutionBitrateLimits
|
||||||
|
kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
|
||||||
|
|
||||||
uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
|
uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
|
||||||
0x00, 0x00, 0x03, 0x03, 0xF4,
|
0x00, 0x00, 0x03, 0x03, 0xF4,
|
||||||
@ -2668,6 +2672,87 @@ TEST_F(VideoStreamEncoderTest,
|
|||||||
video_stream_encoder_->Stop();
|
video_stream_encoder_->Stop();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
|
||||||
|
fake_encoder_.SetResolutionBitrateLimits(
|
||||||
|
{kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
|
||||||
|
|
||||||
|
video_stream_encoder_->OnBitrateUpdated(
|
||||||
|
DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps),
|
||||||
|
DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps),
|
||||||
|
DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0, 0);
|
||||||
|
|
||||||
|
// Enable MAINTAIN_FRAMERATE preference, no initial limitation.
|
||||||
|
AdaptingFrameForwarder source;
|
||||||
|
source.set_adaptation_enabled(true);
|
||||||
|
video_stream_encoder_->SetSource(
|
||||||
|
&source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
|
||||||
|
|
||||||
|
// Insert 720p frame.
|
||||||
|
int64_t timestamp_ms = kFrameIntervalMs;
|
||||||
|
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
|
||||||
|
WaitForEncodedFrame(1280, 720);
|
||||||
|
|
||||||
|
// Reduce bitrate and trigger adapt down.
|
||||||
|
video_stream_encoder_->OnBitrateUpdated(
|
||||||
|
DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps),
|
||||||
|
DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps),
|
||||||
|
DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0, 0);
|
||||||
|
video_stream_encoder_->TriggerQualityLow();
|
||||||
|
|
||||||
|
// Insert 720p frame. It should be downscaled and encoded.
|
||||||
|
timestamp_ms += kFrameIntervalMs;
|
||||||
|
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
|
||||||
|
WaitForEncodedFrame(960, 540);
|
||||||
|
|
||||||
|
// Trigger adapt up. Higher resolution should not be requested duo to lack
|
||||||
|
// of bitrate.
|
||||||
|
video_stream_encoder_->TriggerQualityHigh();
|
||||||
|
VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
|
||||||
|
|
||||||
|
// Increase bitrate.
|
||||||
|
video_stream_encoder_->OnBitrateUpdated(
|
||||||
|
DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps),
|
||||||
|
DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps),
|
||||||
|
DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0, 0);
|
||||||
|
|
||||||
|
// Trigger adapt up. Higher resolution should be requested.
|
||||||
|
video_stream_encoder_->TriggerQualityHigh();
|
||||||
|
VerifyFpsMaxResolutionMax(source.sink_wants());
|
||||||
|
|
||||||
|
video_stream_encoder_->Stop();
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
|
||||||
|
fake_encoder_.SetResolutionBitrateLimits(
|
||||||
|
{kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
|
||||||
|
|
||||||
|
// Set bitrate equal to min bitrate of 540p.
|
||||||
|
video_stream_encoder_->OnBitrateUpdated(
|
||||||
|
DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps),
|
||||||
|
DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps),
|
||||||
|
DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0, 0);
|
||||||
|
|
||||||
|
// Enable MAINTAIN_FRAMERATE preference, no initial limitation.
|
||||||
|
AdaptingFrameForwarder source;
|
||||||
|
source.set_adaptation_enabled(true);
|
||||||
|
video_stream_encoder_->SetSource(
|
||||||
|
&source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
|
||||||
|
|
||||||
|
// Insert 720p frame. It should be dropped and lower resolution should be
|
||||||
|
// requested.
|
||||||
|
int64_t timestamp_ms = kFrameIntervalMs;
|
||||||
|
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
|
||||||
|
ExpectDroppedFrame();
|
||||||
|
VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
|
||||||
|
|
||||||
|
// Insert 720p frame. It should be downscaled and encoded.
|
||||||
|
timestamp_ms += kFrameIntervalMs;
|
||||||
|
source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
|
||||||
|
WaitForEncodedFrame(960, 540);
|
||||||
|
|
||||||
|
video_stream_encoder_->Stop();
|
||||||
|
}
|
||||||
|
|
||||||
class BalancedDegradationTest : public VideoStreamEncoderTest {
|
class BalancedDegradationTest : public VideoStreamEncoderTest {
|
||||||
protected:
|
protected:
|
||||||
void SetupTest() {
|
void SetupTest() {
|
||||||
|
Reference in New Issue
Block a user