zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove QualityScaler framerate reduction.

Browse Source 
Framerate-reduction code is disabled on all platforms, and this code
adds complexity. It's necessary to react fast, especially on mobile
platforms or other bad network conditions and framerate reduction adds
another step between HD and QVGA.

BUG=webrtc:5678, webrtc:5830
R=jackychen@webrtc.org, mflodman@webrtc.org

Review URL: https://codereview.webrtc.org/1885893002 .

Cr-Commit-Position: refs/heads/master@{#12503}
This commit is contained in:
Peter Boström
2016-04-26 13:37:10 +02:00
parent d486dc1bd7
commit 3c6eac2860
19 changed files with 15 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
webrtc/api/java/jni/androidmediaencoder_jni.cc
View File

@ -121,8 +121,6 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder,
void OnDroppedFrame() override;
int GetTargetFramerate() override;
bool SupportsNativeHandle() const override { return egl_context_ != nullptr; }
const char* ImplementationName() const override;
@ -397,7 +395,7 @@ int32_t MediaCodecVideoEncoder::InitEncode(
// 63].
const int kLowQpThreshold = 29;
const int kBadQpThreshold = 100;
quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold, false,
quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold,
codec_settings->startBitrate, codec_settings->width,
codec_settings->height,
codec_settings->maxFramerate);
@ -405,7 +403,7 @@ int32_t MediaCodecVideoEncoder::InitEncode(
// H264 QP is in the range [0, 51].
const int kLowQpThreshold = 24;
const int kBadQpThreshold = 39;
quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold, false,
quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold,
codec_settings->startBitrate, codec_settings->width,
codec_settings->height,
codec_settings->maxFramerate);
@ -1176,10 +1174,6 @@ void MediaCodecVideoEncoder::OnDroppedFrame() {
quality_scaler_.ReportDroppedFrame();
}
int MediaCodecVideoEncoder::GetTargetFramerate() {
return scale_ ? quality_scaler_.GetTargetFramerate() : -1;
}
const char* MediaCodecVideoEncoder::ImplementationName() const {
return "MediaCodec";
}

4
webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.cc
View File

@ -494,10 +494,6 @@ void SimulcastEncoderAdapter::OnDroppedFrame() {
streaminfos_[0].encoder->OnDroppedFrame();
}
int SimulcastEncoderAdapter::GetTargetFramerate() {
return streaminfos_[0].encoder->GetTargetFramerate();
}
bool SimulcastEncoderAdapter::SupportsNativeHandle() const {
// We should not be calling this method before streaminfos_ are configured.
RTC_DCHECK(!streaminfos_.empty());

1
webrtc/modules/video_coding/codecs/vp8/simulcast_encoder_adapter.h
View File

@ -58,7 +58,6 @@ class SimulcastEncoderAdapter : public VP8Encoder {
void OnDroppedFrame() override;
int GetTargetFramerate() override;
bool SupportsNativeHandle() const override;
const char* ImplementationName() const override;

6
webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
View File

@ -598,15 +598,13 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst,
}
rps_.Init();
// Disable both high-QP limits and framedropping. Both are handled by libvpx
// internally.
// QP thresholds are chosen to be high enough to be hit in practice when
// quality is good, but also low enough to not cause a flip-flop behavior
// (e.g. going up in resolution shouldn't give so bad quality that we should
// go back down).
const int kLowQpThreshold = 29;
const int kDisabledBadQpThreshold = 100;
quality_scaler_.Init(kLowQpThreshold, kDisabledBadQpThreshold, false,
const int kBadQpThreshold = 100;
quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold,
codec_.startBitrate, codec_.width, codec_.height,
codec_.maxFramerate);

4
webrtc/modules/video_coding/generic_encoder.cc
View File

@ -138,10 +138,6 @@ bool VCMGenericEncoder::SupportsNativeHandle() const {
return encoder_->SupportsNativeHandle();
}
int VCMGenericEncoder::GetTargetFramerate() {
return encoder_->GetTargetFramerate();
}
VCMEncodedFrameCallback::VCMEncodedFrameCallback(
EncodedImageCallback* post_encode_callback)
: send_callback_(),

1
webrtc/modules/video_coding/generic_encoder.h
View File

@ -83,7 +83,6 @@ class VCMGenericEncoder {
bool InternalSource() const;
void OnDroppedFrame();
bool SupportsNativeHandle() const;
int GetTargetFramerate();
private:
VideoEncoder* const encoder_;

2
webrtc/modules/video_coding/include/video_coding_defines.h
View File

@ -195,8 +195,6 @@ class VCMQMSettingsCallback {
const uint32_t width,
const uint32_t height) = 0;
virtual void SetTargetFramerate(int frame_rate) = 0;
protected:
virtual ~VCMQMSettingsCallback() {}
};

30
webrtc/modules/video_coding/utility/quality_scaler.cc
View File

@ -30,12 +30,10 @@ static const int kQvgaBitrateThresholdKbps = 250;
static const int kQvgaNumPixels = 400 * 300; // 320x240
} // namespace
QualityScaler::QualityScaler()
: low_qp_threshold_(-1), framerate_down_(false) {}
QualityScaler::QualityScaler() : low_qp_threshold_(-1) {}
void QualityScaler::Init(int low_qp_threshold,
int high_qp_threshold,
bool use_framerate_reduction,
int initial_bitrate_kbps,
int width,
int height,
@ -43,7 +41,6 @@ void QualityScaler::Init(int low_qp_threshold,
ClearSamples();
low_qp_threshold_ = low_qp_threshold;
high_qp_threshold_ = high_qp_threshold;
use_framerate_reduction_ = use_framerate_reduction;
downscale_shift_ = 0;
// Use a faster window for upscaling initially (but be more graceful later).
// This enables faster initial rampups without risking strong up-down
@ -65,7 +62,6 @@ void QualityScaler::Init(int low_qp_threshold,
}
UpdateTargetResolution(init_width, init_height);
ReportFramerate(fps);
target_framerate_ = -1;
}
// Report framerate(fps) to estimate # of samples.
@ -94,35 +90,15 @@ void QualityScaler::OnEncodeFrame(const VideoFrame& frame) {
int avg_drop = 0;
int avg_qp = 0;
// When encoder consistently overshoots, framerate reduction and spatial
// resizing will be triggered to get a smoother video.
if ((framedrop_percent_.GetAverage(num_samples_downscale_, &avg_drop) &&
avg_drop >= kFramedropPercentThreshold) ||
(average_qp_downscale_.GetAverage(num_samples_downscale_, &avg_qp) &&
avg_qp > high_qp_threshold_)) {
// Reducing frame rate before spatial resolution change.
// Reduce frame rate only when it is above a certain number.
// Only one reduction is allowed for now.
// TODO(jackychen): Allow more than one framerate reduction.
if (use_framerate_reduction_ && !framerate_down_ && framerate_ >= 20) {
target_framerate_ = framerate_ / 2;
framerate_down_ = true;
// If frame rate has been updated, clear the buffer. We don't want
// spatial resolution to change right after frame rate change.
ClearSamples();
} else {
AdjustScale(false);
}
} else if (average_qp_upscale_.GetAverage(num_samples_upscale_, &avg_qp) &&
avg_qp <= low_qp_threshold_) {
if (use_framerate_reduction_ && framerate_down_) {
target_framerate_ = -1;
framerate_down_ = false;
ClearSamples();
} else {
AdjustScale(true);
}
}
UpdateTargetResolution(frame.width(), frame.height());
}
@ -130,10 +106,6 @@ QualityScaler::Resolution QualityScaler::GetScaledResolution() const {
return res_;
}
int QualityScaler::GetTargetFramerate() const {
return target_framerate_;
}
const VideoFrame& QualityScaler::GetScaledFrame(const VideoFrame& frame) {
Resolution res = GetScaledResolution();
if (res.width == frame.width())

6
webrtc/modules/video_coding/utility/quality_scaler.h
View File

@ -25,7 +25,6 @@ class QualityScaler {
QualityScaler();
void Init(int low_qp_threshold,
int high_qp_threshold,
bool use_framerate_reduction,
int initial_bitrate_kbps,
int width,
int height,
@ -33,11 +32,9 @@ class QualityScaler {
void ReportFramerate(int framerate);
void ReportQP(int qp);
void ReportDroppedFrame();
void Reset(int framerate, int bitrate, int width, int height);
void OnEncodeFrame(const VideoFrame& frame);
Resolution GetScaledResolution() const;
const VideoFrame& GetScaledFrame(const VideoFrame& frame);
int GetTargetFramerate() const;
int downscale_shift() const { return downscale_shift_; }
private:
@ -56,15 +53,12 @@ class QualityScaler {
MovingAverage<int> average_qp_downscale_;
int framerate_;
int target_framerate_;
int low_qp_threshold_;
int high_qp_threshold_;
MovingAverage<int> framedrop_percent_;
Resolution res_;
int downscale_shift_;
int framerate_down_;
bool use_framerate_reduction_;
};
} // namespace webrtc

138
webrtc/modules/video_coding/utility/quality_scaler_unittest.cc
View File

@ -34,14 +34,6 @@ static const int kMinDownscaleDimension = 140;
} // namespace
class QualityScalerTest : public ::testing::Test {
public:
// Temporal and spatial resolution.
struct Resolution {
int framerate;
int width;
int height;
};
protected:
enum ScaleDirection {
kKeepScaleAtHighQp,
@ -49,12 +41,11 @@ class QualityScalerTest : public ::testing::Test {
kScaleDownAboveHighQp,
kScaleUp
};
enum BadQualityMetric { kDropFrame, kReportLowQP };
QualityScalerTest() {
input_frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, kHalfWidth,
kHalfWidth);
qs_.Init(kLowQpThreshold, kHighQp, false, 0, 0, 0, kFramerate);
qs_.Init(kLowQpThreshold, kHighQp, 0, 0, 0, kFramerate);
qs_.OnEncodeFrame(input_frame_);
}
@ -101,16 +92,6 @@ class QualityScalerTest : public ::testing::Test {
void DoesNotDownscaleFrameDimensions(int width, int height);
Resolution TriggerResolutionChange(BadQualityMetric dropframe_lowqp,
int num_second,
int initial_framerate);
void VerifyQualityAdaptation(int initial_framerate,
int seconds_downscale,
int seconds_upscale,
bool expect_spatial_resize,
bool expect_framerate_reduction);
void DownscaleEndsAt(int input_width,
int input_height,
int end_width,
@ -268,105 +249,6 @@ TEST_F(QualityScalerTest, DoesNotDownscaleFrom1Px) {
DoesNotDownscaleFrameDimensions(1, 1);
}
QualityScalerTest::Resolution QualityScalerTest::TriggerResolutionChange(
BadQualityMetric dropframe_lowqp,
int num_second,
int initial_framerate) {
QualityScalerTest::Resolution res;
res.framerate = initial_framerate;
qs_.OnEncodeFrame(input_frame_);
res.width = qs_.GetScaledResolution().width;
res.height = qs_.GetScaledResolution().height;
for (int i = 0; i < kFramerate * num_second; ++i) {
switch (dropframe_lowqp) {
case kReportLowQP:
qs_.ReportQP(kLowQp);
break;
case kDropFrame:
qs_.ReportDroppedFrame();
break;
}
qs_.OnEncodeFrame(input_frame_);
// Simulate the case when SetRates is called right after reducing
// framerate.
qs_.ReportFramerate(initial_framerate);
res.framerate = qs_.GetTargetFramerate();
if (res.framerate != -1)
qs_.ReportFramerate(res.framerate);
res.width = qs_.GetScaledResolution().width;
res.height = qs_.GetScaledResolution().height;
}
return res;
}
void QualityScalerTest::VerifyQualityAdaptation(
int initial_framerate,
int seconds_downscale,
int seconds_upscale,
bool expect_spatial_resize,
bool expect_framerate_reduction) {
qs_.Init(kLowQpThreshold, kDisabledBadQpThreshold, true, 0, 0, 0,
initial_framerate);
qs_.OnEncodeFrame(input_frame_);
int init_width = qs_.GetScaledResolution().width;
int init_height = qs_.GetScaledResolution().height;
// Test reducing framerate by dropping frame continuously.
QualityScalerTest::Resolution res =
TriggerResolutionChange(kDropFrame, seconds_downscale, initial_framerate);
if (expect_framerate_reduction) {
EXPECT_LT(res.framerate, initial_framerate);
} else {
// No framerate reduction, video decimator should be disabled.
EXPECT_EQ(-1, res.framerate);
}
if (expect_spatial_resize) {
EXPECT_LT(res.width, init_width);
EXPECT_LT(res.height, init_height);
} else {
EXPECT_EQ(init_width, res.width);
EXPECT_EQ(init_height, res.height);
}
// The "seconds * 1.5" is to ensure spatial resolution to recover.
// For example, in 6 seconds test, framerate reduction happens in the first
// 3 seconds from 30fps to 15fps and causes the buffer size to be half of the
// original one. Then it will take only 45 samples to downscale (twice in 90
// samples). So to recover the resolution changes, we need more than 10
// seconds (i.e, seconds_upscale * 1.5). This is because the framerate
// increases before spatial size recovers, so it will take 150 samples to
// recover spatial size (300 for twice).
res = TriggerResolutionChange(kReportLowQP, seconds_upscale * 1.5,
initial_framerate);
EXPECT_EQ(-1, res.framerate);
EXPECT_EQ(init_width, res.width);
EXPECT_EQ(init_height, res.height);
}
// In 3 seconds test, only framerate adjusting should happen and 5 second
// upscaling duration, only a framerate adjusting should happen.
TEST_F(QualityScalerTest, ChangeFramerateOnly) {
VerifyQualityAdaptation(kFramerate, kMeasureSecondsDownscale,
kMeasureSecondsUpscale, false, true);
}
// In 6 seconds test, framerate adjusting and scaling are both
// triggered, it shows that scaling would happen after framerate
// adjusting.
TEST_F(QualityScalerTest, ChangeFramerateAndSpatialSize) {
VerifyQualityAdaptation(kFramerate, kMeasureSecondsDownscale * 2,
kMeasureSecondsUpscale * 2, true, true);
}
// When starting from a low framerate, only spatial size will be changed.
TEST_F(QualityScalerTest, ChangeSpatialSizeOnly) {
qs_.ReportFramerate(kFramerate >> 1);
VerifyQualityAdaptation(kFramerate >> 1, kMeasureSecondsDownscale * 2,
kMeasureSecondsUpscale * 2, true, false);
}
TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsWidth) {
DoesNotDownscaleFrameDimensions(
2 * kMinDownscaleDimension - 1, 1000);
@ -383,7 +265,7 @@ TEST_F(QualityScalerTest, DownscaleToVgaOnLowInitialBitrate) {
static const int kInitialBitrateKbps = 300;
input_frame_.CreateEmptyFrame(kWidth720p, kHeight720p, kWidth720p,
kWidth720p / 2, kWidth720p / 2);
qs_.Init(kLowQpThreshold, kDisabledBadQpThreshold, true, kInitialBitrateKbps,
qs_.Init(kLowQpThreshold, kDisabledBadQpThreshold, kInitialBitrateKbps,
kWidth720p, kHeight720p, kFramerate);
qs_.OnEncodeFrame(input_frame_);
int init_width = qs_.GetScaledResolution().width;
@ -398,7 +280,7 @@ TEST_F(QualityScalerTest, DownscaleToQvgaOnLowerInitialBitrate) {
static const int kInitialBitrateKbps = 200;
input_frame_.CreateEmptyFrame(kWidth720p, kHeight720p, kWidth720p,
kWidth720p / 2, kWidth720p / 2);
qs_.Init(kLowQpThreshold, kDisabledBadQpThreshold, true, kInitialBitrateKbps,
qs_.Init(kLowQpThreshold, kDisabledBadQpThreshold, kInitialBitrateKbps,
kWidth720p, kHeight720p, kFramerate);
qs_.OnEncodeFrame(input_frame_);
int init_width = qs_.GetScaledResolution().width;
@ -408,11 +290,9 @@ TEST_F(QualityScalerTest, DownscaleToQvgaOnLowerInitialBitrate) {
}
TEST_F(QualityScalerTest, DownscaleAfterMeasuredSecondsThenSlowerBackUp) {
QualityScalerTest::Resolution initial_res;
qs_.Init(kLowQpThreshold, kHighQp, false, 0, kWidth, kHeight, kFramerate);
qs_.Init(kLowQpThreshold, kHighQp, 0, kWidth, kHeight, kFramerate);
qs_.OnEncodeFrame(input_frame_);
initial_res.width = qs_.GetScaledResolution().width;
initial_res.height = qs_.GetScaledResolution().height;
QualityScaler::Resolution initial_res = qs_.GetScaledResolution();
// Should not downscale if less than kMeasureSecondsDownscale seconds passed.
for (int i = 0; i < kFramerate * kMeasureSecondsDownscale - 1; ++i) {
@ -447,12 +327,10 @@ TEST_F(QualityScalerTest, DownscaleAfterMeasuredSecondsThenSlowerBackUp) {
}
TEST_F(QualityScalerTest, UpscaleQuicklyInitiallyAfterMeasuredSeconds) {
QualityScalerTest::Resolution initial_res;
qs_.Init(kLowQpThreshold, kHighQp, false, kLowInitialBitrateKbps, kWidth,
kHeight, kFramerate);
qs_.Init(kLowQpThreshold, kHighQp, kLowInitialBitrateKbps, kWidth, kHeight,
kFramerate);
qs_.OnEncodeFrame(input_frame_);
initial_res.width = qs_.GetScaledResolution().width;
initial_res.height = qs_.GetScaledResolution().height;
QualityScaler::Resolution initial_res = qs_.GetScaledResolution();
// Should not upscale if less than kMeasureSecondsFastUpscale seconds passed.
for (int i = 0; i < kFramerate * kMeasureSecondsFastUpscale - 1; ++i) {

2
webrtc/modules/video_coding/video_sender.cc
View File

@ -339,8 +339,6 @@ int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame,
next_frame_types_[i] = kVideoFrameDelta;
}
}
if (qm_settings_callback_)
qm_settings_callback_->SetTargetFramerate(_encoder->GetTargetFramerate());
return VCM_OK;
}

9
webrtc/modules/video_processing/frame_preprocessor.cc
View File

@ -71,15 +71,6 @@ int32_t VPMFramePreprocessor::SetTargetResolution(uint32_t width,
return VPM_OK;
}
void VPMFramePreprocessor::SetTargetFramerate(int frame_rate) {
if (frame_rate == -1) {
vd_->EnableTemporalDecimation(false);
} else {
vd_->EnableTemporalDecimation(true);
vd_->SetTargetFramerate(frame_rate);
}
}
void VPMFramePreprocessor::UpdateIncomingframe_rate() {
vd_->UpdateIncomingframe_rate();
}

3
webrtc/modules/video_processing/frame_preprocessor.h
View File

@ -46,9 +46,6 @@ class VPMFramePreprocessor {
uint32_t height,
uint32_t frame_rate);
// Set target frame rate.
void SetTargetFramerate(int frame_rate);
// Update incoming frame rate/dimension.
void UpdateIncomingframe_rate();

2
webrtc/modules/video_processing/include/video_processing.h
View File

@ -79,8 +79,6 @@ class VideoProcessing {
uint32_t height,
uint32_t frame_rate) = 0;
virtual void SetTargetFramerate(int frame_rate) = 0;
virtual uint32_t GetDecimatedFrameRate() = 0;
virtual uint32_t GetDecimatedWidth() const = 0;
virtual uint32_t GetDecimatedHeight() const = 0;

5
webrtc/modules/video_processing/video_processing_impl.cc
View File

@ -135,11 +135,6 @@ int32_t VideoProcessingImpl::SetTargetResolution(uint32_t width,
return frame_pre_processor_.SetTargetResolution(width, height, frame_rate);
}
void VideoProcessingImpl::SetTargetFramerate(int frame_rate) {
rtc::CritScope cs(&mutex_);
frame_pre_processor_.SetTargetFramerate(frame_rate);
}
uint32_t VideoProcessingImpl::GetDecimatedFrameRate() {
rtc::CritScope cs(&mutex_);
return frame_pre_processor_.GetDecimatedFrameRate();

1
webrtc/modules/video_processing/video_processing_impl.h
View File

@ -35,7 +35,6 @@ class VideoProcessingImpl : public VideoProcessing {
int32_t SetTargetResolution(uint32_t width,
uint32_t height,
uint32_t frame_rate) override;
void SetTargetFramerate(int frame_rate) override;
uint32_t GetDecimatedFrameRate() override;
uint32_t GetDecimatedWidth() const override;
uint32_t GetDecimatedHeight() const override;

6
webrtc/video/video_encoder.cc
View File

@ -191,10 +191,4 @@ const char* VideoEncoderSoftwareFallbackWrapper::ImplementationName() const {
return encoder_->ImplementationName();
}
int VideoEncoderSoftwareFallbackWrapper::GetTargetFramerate() {
if (fallback_encoder_)
return fallback_encoder_->GetTargetFramerate();
return encoder_->GetTargetFramerate();
}
} // namespace webrtc

7
webrtc/video/vie_encoder.cc
View File

@ -71,9 +71,6 @@ class QMVideoSettingsCallback : public VCMQMSettingsCallback {
const uint32_t width,
const uint32_t height);
// Update target frame rate.
void SetTargetFramerate(int frame_rate);
private:
VideoProcessing* vp_;
};
@ -535,8 +532,4 @@ int32_t QMVideoSettingsCallback::SetVideoQMSettings(
return vp_->SetTargetResolution(width, height, frame_rate);
}
void QMVideoSettingsCallback::SetTargetFramerate(int frame_rate) {
vp_->SetTargetFramerate(frame_rate);
}
} // namespace webrtc

2
webrtc/video_encoder.h
View File

@ -125,7 +125,6 @@ class VideoEncoder {
virtual int32_t SetPeriodicKeyFrames(bool enable) { return -1; }
virtual void OnDroppedFrame() {}
virtual int GetTargetFramerate() { return -1; }
virtual bool SupportsNativeHandle() const { return false; }
virtual const char* ImplementationName() const { return "unknown"; }
};
@ -153,7 +152,6 @@ class VideoEncoderSoftwareFallbackWrapper : public VideoEncoder {
int32_t SetRates(uint32_t bitrate, uint32_t framerate) override;
void OnDroppedFrame() override;
int GetTargetFramerate() override;
bool SupportsNativeHandle() const override;
const char* ImplementationName() const override;