zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove remaining quality-analysis (QM).

Browse Source 
This was never turned on, contains a lot of complexity and somehow
manages triggering a bug in a downstream project.

BUG=webrtc:5066
R=marpan@webrtc.org
TBR=mflodman@webrtc.org

Review URL: https://codereview.webrtc.org/1917323002 .

Cr-Commit-Position: refs/heads/master@{#12692}
This commit is contained in:
Peter Boström
2016-05-12 03:01:31 +02:00
parent 919288f6ba
commit ad6fc5a05c
34 changed files with 30 additions and 3991 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
webrtc/modules/include/module_common_types.h
View File

@ -494,25 +494,6 @@ class CallStatsObserver {
virtual ~CallStatsObserver() {} virtual ~CallStatsObserver() {}
}; };
struct VideoContentMetrics {
VideoContentMetrics()
: motion_magnitude(0.0f),
spatial_pred_err(0.0f),
spatial_pred_err_h(0.0f),
spatial_pred_err_v(0.0f) {}
void Reset() {
motion_magnitude = 0.0f;
spatial_pred_err = 0.0f;
spatial_pred_err_h = 0.0f;
spatial_pred_err_v = 0.0f;
}
float motion_magnitude;
float spatial_pred_err;
float spatial_pred_err_h;
float spatial_pred_err_v;
};
/* This class holds up to 60 ms of super-wideband (32 kHz) stereo audio. It /* This class holds up to 60 ms of super-wideband (32 kHz) stereo audio. It
* allows for adding and subtracting frames while keeping track of the resulting * allows for adding and subtracting frames while keeping track of the resulting
* states. * states.

2
webrtc/modules/modules.gyp
View File

@ -380,13 +380,11 @@
'video_coding/video_coding_robustness_unittest.cc', 'video_coding/video_coding_robustness_unittest.cc',
'video_coding/video_receiver_unittest.cc', 'video_coding/video_receiver_unittest.cc',
'video_coding/video_sender_unittest.cc', 'video_coding/video_sender_unittest.cc',
'video_coding/qm_select_unittest.cc',
'video_coding/test/stream_generator.cc', 'video_coding/test/stream_generator.cc',
'video_coding/test/stream_generator.h', 'video_coding/test/stream_generator.h',
'video_coding/utility/frame_dropper_unittest.cc', 'video_coding/utility/frame_dropper_unittest.cc',
'video_coding/utility/ivf_file_writer_unittest.cc', 'video_coding/utility/ivf_file_writer_unittest.cc',
'video_coding/utility/quality_scaler_unittest.cc', 'video_coding/utility/quality_scaler_unittest.cc',
'video_processing/test/content_metrics_test.cc',
'video_processing/test/denoiser_test.cc', 'video_processing/test/denoiser_test.cc',
'video_processing/test/video_processing_unittest.cc', 'video_processing/test/video_processing_unittest.cc',
'video_processing/test/video_processing_unittest.h', 'video_processing/test/video_processing_unittest.h',

5
webrtc/modules/video_coding/BUILD.gn
View File

@ -14,8 +14,6 @@ source_set("video_coding") {
"codec_database.h", "codec_database.h",
"codec_timer.cc", "codec_timer.cc",
"codec_timer.h", "codec_timer.h",
"content_metrics_processing.cc",
"content_metrics_processing.h",
"decoding_state.cc", "decoding_state.cc",
"decoding_state.h", "decoding_state.h",
"encoded_frame.cc", "encoded_frame.cc",
@ -54,9 +52,6 @@ source_set("video_coding") {
"packet_buffer.h", "packet_buffer.h",
"percentile_filter.cc", "percentile_filter.cc",
"percentile_filter.h", "percentile_filter.h",
"qm_select.cc",
"qm_select.h",
"qm_select_data.h",
"receiver.cc", "receiver.cc",
"receiver.h", "receiver.h",
"rtt_filter.cc", "rtt_filter.cc",

126
webrtc/modules/video_coding/content_metrics_processing.cc
View File

@ -1,126 +0,0 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/modules/video_coding/content_metrics_processing.h"
#include <math.h>
#include "webrtc/modules/include/module_common_types.h"
#include "webrtc/modules/video_coding/include/video_coding_defines.h"
namespace webrtc {
//////////////////////////////////
/// VCMContentMetricsProcessing //
//////////////////////////////////
VCMContentMetricsProcessing::VCMContentMetricsProcessing()
: recursive_avg_factor_(1 / 150.0f), // matched to 30fps.
frame_cnt_uniform_avg_(0),
avg_motion_level_(0.0f),
avg_spatial_level_(0.0f) {
recursive_avg_ = new VideoContentMetrics();
uniform_avg_ = new VideoContentMetrics();
}
VCMContentMetricsProcessing::~VCMContentMetricsProcessing() {
delete recursive_avg_;
delete uniform_avg_;
}
int VCMContentMetricsProcessing::Reset() {
recursive_avg_->Reset();
uniform_avg_->Reset();
frame_cnt_uniform_avg_ = 0;
avg_motion_level_ = 0.0f;
avg_spatial_level_ = 0.0f;
return VCM_OK;
}
void VCMContentMetricsProcessing::UpdateFrameRate(uint32_t frameRate) {
if (frameRate == 0)
frameRate = 1;
// Update factor for recursive averaging.
recursive_avg_factor_ = static_cast<float>(1000.0f) /
static_cast<float>(frameRate * kQmMinIntervalMs);
}
VideoContentMetrics* VCMContentMetricsProcessing::LongTermAvgData() {
return recursive_avg_;
}
VideoContentMetrics* VCMContentMetricsProcessing::ShortTermAvgData() {
if (frame_cnt_uniform_avg_ == 0) {
return NULL;
}
// Two metrics are used: motion and spatial level.
uniform_avg_->motion_magnitude =
avg_motion_level_ / static_cast<float>(frame_cnt_uniform_avg_);
uniform_avg_->spatial_pred_err =
avg_spatial_level_ / static_cast<float>(frame_cnt_uniform_avg_);
return uniform_avg_;
}
void VCMContentMetricsProcessing::ResetShortTermAvgData() {
// Reset.
avg_motion_level_ = 0.0f;
avg_spatial_level_ = 0.0f;
frame_cnt_uniform_avg_ = 0;
}
int VCMContentMetricsProcessing::UpdateContentData(
const VideoContentMetrics* contentMetrics) {
if (contentMetrics == NULL) {
return VCM_OK;
}
return ProcessContent(contentMetrics);
}
int VCMContentMetricsProcessing::ProcessContent(
const VideoContentMetrics* contentMetrics) {
// Update the recursive averaged metrics: average is over longer window
// of time: over QmMinIntervalMs ms.
UpdateRecursiveAvg(contentMetrics);
// Update the uniform averaged metrics: average is over shorter window
// of time: based on ~RTCP reports.
UpdateUniformAvg(contentMetrics);
return VCM_OK;
}
void VCMContentMetricsProcessing::UpdateUniformAvg(
const VideoContentMetrics* contentMetrics) {
// Update frame counter.
frame_cnt_uniform_avg_ += 1;
// Update averaged metrics: motion and spatial level are used.
avg_motion_level_ += contentMetrics->motion_magnitude;
avg_spatial_level_ += contentMetrics->spatial_pred_err;
return;
}
void VCMContentMetricsProcessing::UpdateRecursiveAvg(
const VideoContentMetrics* contentMetrics) {
// Spatial metrics: 2x2, 1x2(H), 2x1(V).
recursive_avg_->spatial_pred_err =
(1 - recursive_avg_factor_) * recursive_avg_->spatial_pred_err +
recursive_avg_factor_ * contentMetrics->spatial_pred_err;
recursive_avg_->spatial_pred_err_h =
(1 - recursive_avg_factor_) * recursive_avg_->spatial_pred_err_h +
recursive_avg_factor_ * contentMetrics->spatial_pred_err_h;
recursive_avg_->spatial_pred_err_v =
(1 - recursive_avg_factor_) * recursive_avg_->spatial_pred_err_v +
recursive_avg_factor_ * contentMetrics->spatial_pred_err_v;
// Motion metric: Derived from NFD (normalized frame difference).
recursive_avg_->motion_magnitude =
(1 - recursive_avg_factor_) * recursive_avg_->motion_magnitude +
recursive_avg_factor_ * contentMetrics->motion_magnitude;
}
} // namespace webrtc

72
webrtc/modules/video_coding/content_metrics_processing.h
View File

@ -1,72 +0,0 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_VIDEO_CODING_CONTENT_METRICS_PROCESSING_H_
#define WEBRTC_MODULES_VIDEO_CODING_CONTENT_METRICS_PROCESSING_H_
#include "webrtc/typedefs.h"
namespace webrtc {
struct VideoContentMetrics;
// QM interval time (in ms)
enum { kQmMinIntervalMs = 10000 };
// Flag for NFD metric vs motion metric
enum { kNfdMetric = 1 };
/**********************************/
/* Content Metrics Processing */
/**********************************/
class VCMContentMetricsProcessing {
public:
VCMContentMetricsProcessing();
~VCMContentMetricsProcessing();
// Update class with latest metrics.
int UpdateContentData(const VideoContentMetrics* contentMetrics);
// Reset the short-term averaged content data.
void ResetShortTermAvgData();
// Initialize.
int Reset();
// Inform class of current frame rate.
void UpdateFrameRate(uint32_t frameRate);
// Returns the long-term averaged content data: recursive average over longer
// time scale.
VideoContentMetrics* LongTermAvgData();
// Returns the short-term averaged content data: uniform average over
// shorter time scalE.
VideoContentMetrics* ShortTermAvgData();
private:
// Compute working average.
int ProcessContent(const VideoContentMetrics* contentMetrics);
// Update the recursive averaged metrics: longer time average (~5/10 secs).
void UpdateRecursiveAvg(const VideoContentMetrics* contentMetrics);
// Update the uniform averaged metrics: shorter time average (~RTCP report).
void UpdateUniformAvg(const VideoContentMetrics* contentMetrics);
VideoContentMetrics* recursive_avg_;
VideoContentMetrics* uniform_avg_;
float recursive_avg_factor_;
uint32_t frame_cnt_uniform_avg_;
float avg_motion_level_;
float avg_spatial_level_;
};
} // namespace webrtc
#endif // WEBRTC_MODULES_VIDEO_CODING_CONTENT_METRICS_PROCESSING_H_

5
webrtc/modules/video_coding/include/video_coding.h
View File

@ -31,6 +31,10 @@ namespace webrtc {
class Clock; class Clock;
class EncodedImageCallback; class EncodedImageCallback;
// TODO(pbos): Remove VCMQMSettingsCallback completely. This might be done by
// removing the VCM and use VideoSender/VideoReceiver as a public interface
// directly.
class VCMQMSettingsCallback;
class VideoEncoder; class VideoEncoder;
class VideoDecoder; class VideoDecoder;
struct CodecSpecificInfo; struct CodecSpecificInfo;
@ -223,7 +227,6 @@ class VideoCodingModule : public Module {
// < 0, on error. // < 0, on error.
virtual int32_t AddVideoFrame( virtual int32_t AddVideoFrame(
const VideoFrame& videoFrame, const VideoFrame& videoFrame,
const VideoContentMetrics* contentMetrics = NULL,
const CodecSpecificInfo* codecSpecificInfo = NULL) = 0; const CodecSpecificInfo* codecSpecificInfo = NULL) = 0;
// Next frame encoded should be an intra frame (keyframe). // Next frame encoded should be an intra frame (keyframe).

12
webrtc/modules/video_coding/include/video_coding_defines.h
View File

@ -176,18 +176,6 @@ class KeyFrameRequestSender {
virtual ~KeyFrameRequestSender() {} virtual ~KeyFrameRequestSender() {}
}; };
// Callback used to inform the user of the the desired resolution
// as subscribed by Media Optimization (Quality Modes)
class VCMQMSettingsCallback {
public:
virtual int32_t SetVideoQMSettings(const uint32_t frameRate,
const uint32_t width,
const uint32_t height) = 0;
protected:
virtual ~VCMQMSettingsCallback() {}
};
} // namespace webrtc } // namespace webrtc
#endif // WEBRTC_MODULES_VIDEO_CODING_INCLUDE_VIDEO_CODING_DEFINES_H_ #endif // WEBRTC_MODULES_VIDEO_CODING_INCLUDE_VIDEO_CODING_DEFINES_H_

1
webrtc/modules/video_coding/media_opt_util.h
View File

@ -18,7 +18,6 @@
#include "webrtc/base/exp_filter.h" #include "webrtc/base/exp_filter.h"
#include "webrtc/modules/video_coding/internal_defines.h" #include "webrtc/modules/video_coding/internal_defines.h"
#include "webrtc/modules/video_coding/qm_select.h"
#include "webrtc/system_wrappers/include/trace.h" #include "webrtc/system_wrappers/include/trace.h"
#include "webrtc/typedefs.h" #include "webrtc/typedefs.h"

150
webrtc/modules/video_coding/media_optimization.cc
View File

@ -11,8 +11,6 @@
#include "webrtc/modules/video_coding/media_optimization.h" #include "webrtc/modules/video_coding/media_optimization.h"
#include "webrtc/base/logging.h" #include "webrtc/base/logging.h"
#include "webrtc/modules/video_coding/content_metrics_processing.h"
#include "webrtc/modules/video_coding/qm_select.h"
#include "webrtc/modules/video_coding/utility/frame_dropper.h" #include "webrtc/modules/video_coding/utility/frame_dropper.h"
#include "webrtc/system_wrappers/include/clock.h" #include "webrtc/system_wrappers/include/clock.h"
@ -81,16 +79,11 @@ MediaOptimization::MediaOptimization(Clock* clock)
max_payload_size_(1460), max_payload_size_(1460),
video_target_bitrate_(0), video_target_bitrate_(0),
incoming_frame_rate_(0), incoming_frame_rate_(0),
enable_qm_(false),
encoded_frame_samples_(), encoded_frame_samples_(),
avg_sent_bit_rate_bps_(0), avg_sent_bit_rate_bps_(0),
avg_sent_framerate_(0), avg_sent_framerate_(0),
key_frame_cnt_(0), key_frame_cnt_(0),
delta_frame_cnt_(0), delta_frame_cnt_(0),
content_(new VCMContentMetricsProcessing()),
qm_resolution_(new VCMQmResolution()),
last_qm_update_time_(0),
last_change_time_(0),
num_layers_(0), num_layers_(0),
suspension_enabled_(false), suspension_enabled_(false),
video_suspended_(false), video_suspended_(false),
@ -113,8 +106,6 @@ void MediaOptimization::Reset() {
frame_dropper_->Reset(); frame_dropper_->Reset();
loss_prot_logic_->Reset(clock_->TimeInMilliseconds()); loss_prot_logic_->Reset(clock_->TimeInMilliseconds());
frame_dropper_->SetRates(0, 0); frame_dropper_->SetRates(0, 0);
content_->Reset();
qm_resolution_->Reset();
loss_prot_logic_->UpdateFrameRate(incoming_frame_rate_); loss_prot_logic_->UpdateFrameRate(incoming_frame_rate_);
loss_prot_logic_->Reset(clock_->TimeInMilliseconds()); loss_prot_logic_->Reset(clock_->TimeInMilliseconds());
send_statistics_zero_encode_ = 0; send_statistics_zero_encode_ = 0;
@ -124,8 +115,6 @@ void MediaOptimization::Reset() {
user_frame_rate_ = 0; user_frame_rate_ = 0;
key_frame_cnt_ = 0; key_frame_cnt_ = 0;
delta_frame_cnt_ = 0; delta_frame_cnt_ = 0;
last_qm_update_time_ = 0;
last_change_time_ = 0;
encoded_frame_samples_.clear(); encoded_frame_samples_.clear();
avg_sent_bit_rate_bps_ = 0; avg_sent_bit_rate_bps_ = 0;
num_layers_ = 1; num_layers_ = 1;
@ -153,12 +142,7 @@ void MediaOptimization::SetEncodingDataInternal(VideoCodecType send_codec_type,
int num_layers, int num_layers,
int32_t mtu) { int32_t mtu) {
// Everything codec specific should be reset here since this means the codec // Everything codec specific should be reset here since this means the codec
// has changed. If native dimension values have changed, then either user // has changed.
// initiated change, or QM initiated change. Will be able to determine only
// after the processing of the first frame.
last_change_time_ = clock_->TimeInMilliseconds();
content_->Reset();
content_->UpdateFrameRate(frame_rate);
max_bit_rate_ = max_bit_rate; max_bit_rate_ = max_bit_rate;
send_codec_type_ = send_codec_type; send_codec_type_ = send_codec_type;
@ -175,16 +159,13 @@ void MediaOptimization::SetEncodingDataInternal(VideoCodecType send_codec_type,
codec_height_ = height; codec_height_ = height;
num_layers_ = (num_layers <= 1) ? 1 : num_layers; // Can also be zero. num_layers_ = (num_layers <= 1) ? 1 : num_layers; // Can also be zero.
max_payload_size_ = mtu; max_payload_size_ = mtu;
qm_resolution_->Initialize(target_bitrate_kbps, user_frame_rate_,
codec_width_, codec_height_, num_layers_);
} }
uint32_t MediaOptimization::SetTargetRates( uint32_t MediaOptimization::SetTargetRates(
uint32_t target_bitrate, uint32_t target_bitrate,
uint8_t fraction_lost, uint8_t fraction_lost,
int64_t round_trip_time_ms, int64_t round_trip_time_ms,
VCMProtectionCallback* protection_callback, VCMProtectionCallback* protection_callback) {
VCMQMSettingsCallback* qmsettings_callback) {
CriticalSectionScoped lock(crit_sect_.get()); CriticalSectionScoped lock(crit_sect_.get());
VCMProtectionMethod* selected_method = loss_prot_logic_->SelectedMethod(); VCMProtectionMethod* selected_method = loss_prot_logic_->SelectedMethod();
float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f; float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
@ -220,7 +201,6 @@ uint32_t MediaOptimization::SetTargetRates(
float protection_overhead_rate = 0.0f; float protection_overhead_rate = 0.0f;
// Update protection settings, when applicable. // Update protection settings, when applicable.
float sent_video_rate_kbps = 0.0f;
if (loss_prot_logic_->SelectedType() != kNone) { if (loss_prot_logic_->SelectedType() != kNone) {
// Update method will compute the robustness settings for the given // Update method will compute the robustness settings for the given
// protection method and the overhead cost // protection method and the overhead cost
@ -255,7 +235,6 @@ uint32_t MediaOptimization::SetTargetRates(
// Get the effective packet loss for encoder ER when applicable. Should be // Get the effective packet loss for encoder ER when applicable. Should be
// passed to encoder via fraction_lost. // passed to encoder via fraction_lost.
packet_loss_enc = selected_method->RequiredPacketLossER(); packet_loss_enc = selected_method->RequiredPacketLossER();
sent_video_rate_kbps = static_cast<float>(sent_video_rate_bps) / 1000.0f;
} }
// Source coding rate: total rate - protection overhead. // Source coding rate: total rate - protection overhead.
@ -271,19 +250,6 @@ uint32_t MediaOptimization::SetTargetRates(
static_cast<float>(video_target_bitrate_) / 1000.0f; static_cast<float>(video_target_bitrate_) / 1000.0f;
frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_); frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_);
if (enable_qm_ && qmsettings_callback) {
// Update QM with rates.
qm_resolution_->UpdateRates(target_video_bitrate_kbps, sent_video_rate_kbps,
incoming_frame_rate_, fraction_lost_);
// Check for QM selection.
bool select_qm = CheckStatusForQMchange();
if (select_qm) {
SelectQuality(qmsettings_callback);
}
// Reset the short-term averaged content data.
content_->ResetShortTermAvgData();
}
CheckSuspendConditions(); CheckSuspendConditions();
return video_target_bitrate_; return video_target_bitrate_;
@ -357,11 +323,6 @@ int32_t MediaOptimization::UpdateWithEncodedData(
loss_prot_logic_->UpdatePacketsPerFrameKey( loss_prot_logic_->UpdatePacketsPerFrameKey(
min_packets_per_frame, clock_->TimeInMilliseconds()); min_packets_per_frame, clock_->TimeInMilliseconds());
} }
if (enable_qm_) {
// Update quality select with encoded length.
qm_resolution_->UpdateEncodedSize(encoded_length);
}
} }
if (!delta_frame && encoded_length > 0) { if (!delta_frame && encoded_length > 0) {
loss_prot_logic_->UpdateKeyFrameSize(static_cast<float>(encoded_length)); loss_prot_logic_->UpdateKeyFrameSize(static_cast<float>(encoded_length));
@ -378,11 +339,6 @@ int32_t MediaOptimization::UpdateWithEncodedData(
return VCM_OK; return VCM_OK;
} }
void MediaOptimization::EnableQM(bool enable) {
CriticalSectionScoped lock(crit_sect_.get());
enable_qm_ = enable;
}
void MediaOptimization::EnableFrameDropper(bool enable) { void MediaOptimization::EnableFrameDropper(bool enable) {
CriticalSectionScoped lock(crit_sect_.get()); CriticalSectionScoped lock(crit_sect_.get());
frame_dropper_->Enable(enable); frame_dropper_->Enable(enable);
@ -414,19 +370,6 @@ bool MediaOptimization::DropFrame() {
return frame_dropper_->DropFrame(); return frame_dropper_->DropFrame();
} }
void MediaOptimization::UpdateContentData(
const VideoContentMetrics* content_metrics) {
CriticalSectionScoped lock(crit_sect_.get());
// Updating content metrics.
if (content_metrics == NULL) {
// Disable QM if metrics are NULL.
enable_qm_ = false;
qm_resolution_->Reset();
} else {
content_->UpdateContentData(content_metrics);
}
}
void MediaOptimization::UpdateIncomingFrameRate() { void MediaOptimization::UpdateIncomingFrameRate() {
int64_t now = clock_->TimeInMilliseconds(); int64_t now = clock_->TimeInMilliseconds();
if (incoming_frame_times_[0] == 0) { if (incoming_frame_times_[0] == 0) {
@ -441,36 +384,6 @@ void MediaOptimization::UpdateIncomingFrameRate() {
ProcessIncomingFrameRate(now); ProcessIncomingFrameRate(now);
} }
int32_t MediaOptimization::SelectQuality(
VCMQMSettingsCallback* video_qmsettings_callback) {
// Reset quantities for QM select.
qm_resolution_->ResetQM();
// Update QM will long-term averaged content metrics.
qm_resolution_->UpdateContent(content_->LongTermAvgData());
// Select quality mode.
VCMResolutionScale* qm = NULL;
int32_t ret = qm_resolution_->SelectResolution(&qm);
if (ret < 0) {
return ret;
}
// Check for updates to spatial/temporal modes.
QMUpdate(qm, video_qmsettings_callback);
// Reset all the rate and related frame counters quantities.
qm_resolution_->ResetRates();
// Reset counters.
last_qm_update_time_ = clock_->TimeInMilliseconds();
// Reset content metrics.
content_->Reset();
return VCM_OK;
}
void MediaOptimization::PurgeOldFrameSamples(int64_t now_ms) { void MediaOptimization::PurgeOldFrameSamples(int64_t now_ms) {
while (!encoded_frame_samples_.empty()) { while (!encoded_frame_samples_.empty()) {
if (now_ms - encoded_frame_samples_.front().time_complete_ms > if (now_ms - encoded_frame_samples_.front().time_complete_ms >
@ -517,65 +430,6 @@ void MediaOptimization::UpdateSentFramerate() {
} }
} }
bool MediaOptimization::QMUpdate(
VCMResolutionScale* qm,
VCMQMSettingsCallback* video_qmsettings_callback) {
// Check for no change.
if (!qm->change_resolution_spatial && !qm->change_resolution_temporal) {
return false;
}
// Check for change in frame rate.
if (qm->change_resolution_temporal) {
incoming_frame_rate_ = qm->frame_rate;
// Reset frame rate estimate.
memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
}
// Check for change in frame size.
if (qm->change_resolution_spatial) {
codec_width_ = qm->codec_width;
codec_height_ = qm->codec_height;
}
LOG(LS_INFO) << "Media optimizer requests the video resolution to be changed "
"to "
<< qm->codec_width << "x" << qm->codec_height << "@"
<< qm->frame_rate;
// Update VPM with new target frame rate and frame size.
// Note: use |qm->frame_rate| instead of |_incoming_frame_rate| for updating
// target frame rate in VPM frame dropper. The quantity |_incoming_frame_rate|
// will vary/fluctuate, and since we don't want to change the state of the
// VPM frame dropper, unless a temporal action was selected, we use the
// quantity |qm->frame_rate| for updating.
video_qmsettings_callback->SetVideoQMSettings(qm->frame_rate, codec_width_,
codec_height_);
content_->UpdateFrameRate(qm->frame_rate);
qm_resolution_->UpdateCodecParameters(qm->frame_rate, codec_width_,
codec_height_);
return true;
}
// Check timing constraints and look for significant change in:
// (1) scene content,
// (2) target bit rate.
bool MediaOptimization::CheckStatusForQMchange() {
bool status = true;
// Check that we do not call QMSelect too often, and that we waited some time
// (to sample the metrics) from the event last_change_time
// last_change_time is the time where user changed the size/rate/frame rate
// (via SetEncodingData).
int64_t now = clock_->TimeInMilliseconds();
if ((now - last_qm_update_time_) < kQmMinIntervalMs ||
(now - last_change_time_) < kQmMinIntervalMs) {
status = false;
}
return status;
}
// Allowing VCM to keep track of incoming frame rate. // Allowing VCM to keep track of incoming frame rate.
void MediaOptimization::ProcessIncomingFrameRate(int64_t now) { void MediaOptimization::ProcessIncomingFrameRate(int64_t now) {
int32_t num = 0; int32_t num = 0;

25
webrtc/modules/video_coding/media_optimization.h
View File

@ -17,7 +17,6 @@
#include "webrtc/modules/include/module_common_types.h" #include "webrtc/modules/include/module_common_types.h"
#include "webrtc/modules/video_coding/include/video_coding.h" #include "webrtc/modules/video_coding/include/video_coding.h"
#include "webrtc/modules/video_coding/media_opt_util.h" #include "webrtc/modules/video_coding/media_opt_util.h"
#include "webrtc/modules/video_coding/qm_select.h"
#include "webrtc/system_wrappers/include/critical_section_wrapper.h" #include "webrtc/system_wrappers/include/critical_section_wrapper.h"
namespace webrtc { namespace webrtc {
@ -59,11 +58,9 @@ class MediaOptimization {
uint32_t SetTargetRates(uint32_t target_bitrate, uint32_t SetTargetRates(uint32_t target_bitrate,
uint8_t fraction_lost, uint8_t fraction_lost,
int64_t round_trip_time_ms, int64_t round_trip_time_ms,
VCMProtectionCallback* protection_callback, VCMProtectionCallback* protection_callback);
VCMQMSettingsCallback* qmsettings_callback);
void SetProtectionMethod(VCMProtectionMethodEnum method); void SetProtectionMethod(VCMProtectionMethodEnum method);
void EnableQM(bool enable);
void EnableFrameDropper(bool enable); void EnableFrameDropper(bool enable);
// Lets the sender suspend video when the rate drops below // Lets the sender suspend video when the rate drops below
@ -74,8 +71,6 @@ class MediaOptimization {
bool DropFrame(); bool DropFrame();
void UpdateContentData(const VideoContentMetrics* content_metrics);
// Informs Media Optimization of encoded output. // Informs Media Optimization of encoded output.
int32_t UpdateWithEncodedData(const EncodedImage& encoded_image); int32_t UpdateWithEncodedData(const EncodedImage& encoded_image);
@ -98,19 +93,6 @@ class MediaOptimization {
void UpdateSentBitrate(int64_t now_ms) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); void UpdateSentBitrate(int64_t now_ms) EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
void UpdateSentFramerate() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); void UpdateSentFramerate() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
// Computes new Quality Mode.
int32_t SelectQuality(VCMQMSettingsCallback* qmsettings_callback)
EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
// Verifies if QM settings differ from default, i.e. if an update is required.
// Computes actual values, as will be sent to the encoder.
bool QMUpdate(VCMResolutionScale* qm,
VCMQMSettingsCallback* qmsettings_callback)
EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
// Checks if we should make a QM change. Return true if yes, false otherwise.
bool CheckStatusForQMchange() EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
void ProcessIncomingFrameRate(int64_t now) void ProcessIncomingFrameRate(int64_t now)
EXCLUSIVE_LOCKS_REQUIRED(crit_sect_); EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
@ -152,16 +134,11 @@ class MediaOptimization {
int video_target_bitrate_ GUARDED_BY(crit_sect_); int video_target_bitrate_ GUARDED_BY(crit_sect_);
float incoming_frame_rate_ GUARDED_BY(crit_sect_); float incoming_frame_rate_ GUARDED_BY(crit_sect_);
int64_t incoming_frame_times_[kFrameCountHistorySize] GUARDED_BY(crit_sect_); int64_t incoming_frame_times_[kFrameCountHistorySize] GUARDED_BY(crit_sect_);
bool enable_qm_ GUARDED_BY(crit_sect_);
std::list<EncodedFrameSample> encoded_frame_samples_ GUARDED_BY(crit_sect_); std::list<EncodedFrameSample> encoded_frame_samples_ GUARDED_BY(crit_sect_);
uint32_t avg_sent_bit_rate_bps_ GUARDED_BY(crit_sect_); uint32_t avg_sent_bit_rate_bps_ GUARDED_BY(crit_sect_);
uint32_t avg_sent_framerate_ GUARDED_BY(crit_sect_); uint32_t avg_sent_framerate_ GUARDED_BY(crit_sect_);
uint32_t key_frame_cnt_ GUARDED_BY(crit_sect_); uint32_t key_frame_cnt_ GUARDED_BY(crit_sect_);
uint32_t delta_frame_cnt_ GUARDED_BY(crit_sect_); uint32_t delta_frame_cnt_ GUARDED_BY(crit_sect_);
std::unique_ptr<VCMContentMetricsProcessing> content_ GUARDED_BY(crit_sect_);
std::unique_ptr<VCMQmResolution> qm_resolution_ GUARDED_BY(crit_sect_);
int64_t last_qm_update_time_ GUARDED_BY(crit_sect_);
int64_t last_change_time_ GUARDED_BY(crit_sect_); // Content/user triggered.
int num_layers_ GUARDED_BY(crit_sect_); int num_layers_ GUARDED_BY(crit_sect_);
bool suspension_enabled_ GUARDED_BY(crit_sect_); bool suspension_enabled_ GUARDED_BY(crit_sect_);
bool video_suspended_ GUARDED_BY(crit_sect_); bool video_suspended_ GUARDED_BY(crit_sect_);

12
webrtc/modules/video_coding/media_optimization_unittest.cc
View File

@ -66,7 +66,7 @@ TEST_F(TestMediaOptimization, VerifyMuting) {
media_opt_.SetTargetRates(target_bitrate_kbps * 1000, media_opt_.SetTargetRates(target_bitrate_kbps * 1000,
0, // Lossrate. 0, // Lossrate.
100, // RTT in ms. 100, // RTT in ms.
nullptr, nullptr); nullptr);
media_opt_.EnableFrameDropper(true); media_opt_.EnableFrameDropper(true);
for (int time = 0; time < 2000; time += frame_time_ms_) { for (int time = 0; time < 2000; time += frame_time_ms_) {
ASSERT_NO_FATAL_FAILURE(AddFrameAndAdvanceTime(target_bitrate_kbps, false)); ASSERT_NO_FATAL_FAILURE(AddFrameAndAdvanceTime(target_bitrate_kbps, false));
@ -76,7 +76,7 @@ TEST_F(TestMediaOptimization, VerifyMuting) {
media_opt_.SetTargetRates(kThresholdBps - 1000, media_opt_.SetTargetRates(kThresholdBps - 1000,
0, // Lossrate. 0, // Lossrate.
100, // RTT in ms. 100, // RTT in ms.
nullptr, nullptr); nullptr);
// Expect the muter to engage immediately and stay muted. // Expect the muter to engage immediately and stay muted.
// Test during 2 seconds. // Test during 2 seconds.
for (int time = 0; time < 2000; time += frame_time_ms_) { for (int time = 0; time < 2000; time += frame_time_ms_) {
@ -89,7 +89,7 @@ TEST_F(TestMediaOptimization, VerifyMuting) {
media_opt_.SetTargetRates(kThresholdBps + 1000, media_opt_.SetTargetRates(kThresholdBps + 1000,
0, // Lossrate. 0, // Lossrate.
100, // RTT in ms. 100, // RTT in ms.
nullptr, nullptr); nullptr);
// Expect the muter to stay muted. // Expect the muter to stay muted.
// Test during 2 seconds. // Test during 2 seconds.
for (int time = 0; time < 2000; time += frame_time_ms_) { for (int time = 0; time < 2000; time += frame_time_ms_) {
@ -101,7 +101,7 @@ TEST_F(TestMediaOptimization, VerifyMuting) {
media_opt_.SetTargetRates(kThresholdBps + kWindowBps + 1000, media_opt_.SetTargetRates(kThresholdBps + kWindowBps + 1000,
0, // Lossrate. 0, // Lossrate.
100, // RTT in ms. 100, // RTT in ms.
nullptr, nullptr); nullptr);
// Expect the muter to disengage immediately. // Expect the muter to disengage immediately.
// Test during 2 seconds. // Test during 2 seconds.
for (int time = 0; time < 2000; time += frame_time_ms_) { for (int time = 0; time < 2000; time += frame_time_ms_) {
@ -138,7 +138,7 @@ TEST_F(TestMediaOptimization, ProtectsUsingFecBitrateAboveCodecMax) {
// Using 10% of codec bitrate for FEC, should still be able to use all of it. // Using 10% of codec bitrate for FEC, should still be able to use all of it.
protection_callback.fec_rate_bps_ = kCodecBitrateBps / 10; protection_callback.fec_rate_bps_ = kCodecBitrateBps / 10;
uint32_t target_bitrate = media_opt_.SetTargetRates( uint32_t target_bitrate = media_opt_.SetTargetRates(
kMaxBitrateBps, 0, 0, &protection_callback, nullptr); kMaxBitrateBps, 0, 0, &protection_callback);
EXPECT_EQ(kCodecBitrateBps, static_cast<int>(target_bitrate)); EXPECT_EQ(kCodecBitrateBps, static_cast<int>(target_bitrate));
@ -146,7 +146,7 @@ TEST_F(TestMediaOptimization, ProtectsUsingFecBitrateAboveCodecMax) {
// both equally, but only be half of max (since that ceiling should be hit). // both equally, but only be half of max (since that ceiling should be hit).
protection_callback.fec_rate_bps_ = kCodecBitrateBps; protection_callback.fec_rate_bps_ = kCodecBitrateBps;
target_bitrate = media_opt_.SetTargetRates(kMaxBitrateBps, 128, 100, target_bitrate = media_opt_.SetTargetRates(kMaxBitrateBps, 128, 100,
&protection_callback, nullptr); &protection_callback);
EXPECT_EQ(kMaxBitrateBps / 2, static_cast<int>(target_bitrate)); EXPECT_EQ(kMaxBitrateBps / 2, static_cast<int>(target_bitrate));
} }

901
webrtc/modules/video_coding/qm_select.cc
View File

@ -1,901 +0,0 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/modules/video_coding/qm_select.h"
#include <math.h>
#include "webrtc/modules/include/module_common_types.h"
#include "webrtc/modules/video_coding/include/video_coding_defines.h"
#include "webrtc/modules/video_coding/internal_defines.h"
#include "webrtc/modules/video_coding/qm_select_data.h"
#include "webrtc/system_wrappers/include/trace.h"
namespace webrtc {
// QM-METHOD class
VCMQmMethod::VCMQmMethod()
: content_metrics_(NULL),
width_(0),
height_(0),
user_frame_rate_(0.0f),
native_width_(0),
native_height_(0),
native_frame_rate_(0.0f),
image_type_(kVGA),
framerate_level_(kFrameRateHigh),
init_(false) {
ResetQM();
}
VCMQmMethod::~VCMQmMethod() {}
void VCMQmMethod::ResetQM() {
aspect_ratio_ = 1.0f;
motion_.Reset();
spatial_.Reset();
content_class_ = 0;
}
uint8_t VCMQmMethod::ComputeContentClass() {
ComputeMotionNFD();
ComputeSpatial();
return content_class_ = 3 * motion_.level + spatial_.level;
}
void VCMQmMethod::UpdateContent(const VideoContentMetrics* contentMetrics) {
content_metrics_ = contentMetrics;
}
void VCMQmMethod::ComputeMotionNFD() {
if (content_metrics_) {
motion_.value = content_metrics_->motion_magnitude;
}
// Determine motion level.
if (motion_.value < kLowMotionNfd) {
motion_.level = kLow;
} else if (motion_.value > kHighMotionNfd) {
motion_.level = kHigh;
} else {
motion_.level = kDefault;
}
}
void VCMQmMethod::ComputeSpatial() {
float spatial_err = 0.0;
float spatial_err_h = 0.0;
float spatial_err_v = 0.0;
if (content_metrics_) {
spatial_err = content_metrics_->spatial_pred_err;
spatial_err_h = content_metrics_->spatial_pred_err_h;
spatial_err_v = content_metrics_->spatial_pred_err_v;
}
// Spatial measure: take average of 3 prediction errors.
spatial_.value = (spatial_err + spatial_err_h + spatial_err_v) / 3.0f;
// Reduce thresholds for large scenes/higher pixel correlation.
float scale2 = image_type_ > kVGA ? kScaleTexture : 1.0;
if (spatial_.value > scale2 * kHighTexture) {
spatial_.level = kHigh;
} else if (spatial_.value < scale2 * kLowTexture) {
spatial_.level = kLow;
} else {
spatial_.level = kDefault;
}
}
ImageType VCMQmMethod::GetImageType(uint16_t width, uint16_t height) {
// Get the image type for the encoder frame size.
uint32_t image_size = width * height;
if (image_size == kSizeOfImageType[kQCIF]) {
return kQCIF;
} else if (image_size == kSizeOfImageType[kHCIF]) {
return kHCIF;
} else if (image_size == kSizeOfImageType[kQVGA]) {
return kQVGA;
} else if (image_size == kSizeOfImageType[kCIF]) {
return kCIF;
} else if (image_size == kSizeOfImageType[kHVGA]) {
return kHVGA;
} else if (image_size == kSizeOfImageType[kVGA]) {
return kVGA;
} else if (image_size == kSizeOfImageType[kQFULLHD]) {
return kQFULLHD;
} else if (image_size == kSizeOfImageType[kWHD]) {
return kWHD;
} else if (image_size == kSizeOfImageType[kFULLHD]) {
return kFULLHD;
} else {
// No exact match, find closet one.
return FindClosestImageType(width, height);
}
}
ImageType VCMQmMethod::FindClosestImageType(uint16_t width, uint16_t height) {
float size = static_cast<float>(width * height);
float min = size;
int isel = 0;
for (int i = 0; i < kNumImageTypes; ++i) {
float dist = fabs(size - kSizeOfImageType[i]);
if (dist < min) {
min = dist;
isel = i;
}
}
return static_cast<ImageType>(isel);
}
FrameRateLevelClass VCMQmMethod::FrameRateLevel(float avg_framerate) {
if (avg_framerate <= kLowFrameRate) {
return kFrameRateLow;
} else if (avg_framerate <= kMiddleFrameRate) {
return kFrameRateMiddle1;
} else if (avg_framerate <= kHighFrameRate) {
return kFrameRateMiddle2;
} else {
return kFrameRateHigh;
}
}
// RESOLUTION CLASS
VCMQmResolution::VCMQmResolution() : qm_(new VCMResolutionScale()) {
Reset();
}
VCMQmResolution::~VCMQmResolution() {
delete qm_;
}
void VCMQmResolution::ResetRates() {
sum_target_rate_ = 0.0f;
sum_incoming_framerate_ = 0.0f;
sum_rate_MM_ = 0.0f;
sum_rate_MM_sgn_ = 0.0f;
sum_packet_loss_ = 0.0f;
buffer_level_ = kInitBufferLevel * target_bitrate_;
frame_cnt_ = 0;
frame_cnt_delta_ = 0;
low_buffer_cnt_ = 0;
update_rate_cnt_ = 0;
}
void VCMQmResolution::ResetDownSamplingState() {
state_dec_factor_spatial_ = 1.0;
state_dec_factor_temporal_ = 1.0;
for (int i = 0; i < kDownActionHistorySize; i++) {
down_action_history_[i].spatial = kNoChangeSpatial;
down_action_history_[i].temporal = kNoChangeTemporal;
}
}
void VCMQmResolution::Reset() {
target_bitrate_ = 0.0f;
incoming_framerate_ = 0.0f;
buffer_level_ = 0.0f;
per_frame_bandwidth_ = 0.0f;
avg_target_rate_ = 0.0f;
avg_incoming_framerate_ = 0.0f;
avg_ratio_buffer_low_ = 0.0f;
avg_rate_mismatch_ = 0.0f;
avg_rate_mismatch_sgn_ = 0.0f;
avg_packet_loss_ = 0.0f;
encoder_state_ = kStableEncoding;
num_layers_ = 1;
ResetRates();
ResetDownSamplingState();
ResetQM();
}
EncoderState VCMQmResolution::GetEncoderState() {
return encoder_state_;
}
// Initialize state after re-initializing the encoder,
// i.e., after SetEncodingData() in mediaOpt.
int VCMQmResolution::Initialize(float bitrate,
float user_framerate,
uint16_t width,
uint16_t height,
int num_layers) {
if (user_framerate == 0.0f || width == 0 || height == 0) {
return VCM_PARAMETER_ERROR;
}
Reset();
target_bitrate_ = bitrate;
incoming_framerate_ = user_framerate;
UpdateCodecParameters(user_framerate, width, height);
native_width_ = width;
native_height_ = height;
native_frame_rate_ = user_framerate;
num_layers_ = num_layers;
// Initial buffer level.
buffer_level_ = kInitBufferLevel * target_bitrate_;
// Per-frame bandwidth.
per_frame_bandwidth_ = target_bitrate_ / user_framerate;
init_ = true;
return VCM_OK;
}
void VCMQmResolution::UpdateCodecParameters(float frame_rate,
uint16_t width,
uint16_t height) {
width_ = width;
height_ = height;
// |user_frame_rate| is the target frame rate for VPM frame dropper.
user_frame_rate_ = frame_rate;
image_type_ = GetImageType(width, height);
}
// Update rate data after every encoded frame.
void VCMQmResolution::UpdateEncodedSize(size_t encoded_size) {
frame_cnt_++;
// Convert to Kbps.
float encoded_size_kbits = 8.0f * static_cast<float>(encoded_size) / 1000.0f;
// Update the buffer level:
// Note this is not the actual encoder buffer level.
// |buffer_level_| is reset to an initial value after SelectResolution is
// called, and does not account for frame dropping by encoder or VCM.
buffer_level_ += per_frame_bandwidth_ - encoded_size_kbits;
// Counter for occurrences of low buffer level:
// low/negative values means encoder is likely dropping frames.
if (buffer_level_ <= kPercBufferThr * kInitBufferLevel * target_bitrate_) {
low_buffer_cnt_++;
}
}
// Update various quantities after SetTargetRates in MediaOpt.
void VCMQmResolution::UpdateRates(float target_bitrate,
float encoder_sent_rate,
float incoming_framerate,
uint8_t packet_loss) {
// Sum the target bitrate: this is the encoder rate from previous update
// (~1sec), i.e, before the update for next ~1sec.
sum_target_rate_ += target_bitrate_;
update_rate_cnt_++;
// Sum the received (from RTCP reports) packet loss rates.
sum_packet_loss_ += static_cast<float>(packet_loss / 255.0);
// Sum the sequence rate mismatch:
// Mismatch here is based on the difference between the target rate
// used (in previous ~1sec) and the average actual encoding rate measured
// at previous ~1sec.
float diff = target_bitrate_ - encoder_sent_rate;
if (target_bitrate_ > 0.0)
sum_rate_MM_ += fabs(diff) / target_bitrate_;
int sgnDiff = diff > 0 ? 1 : (diff < 0 ? -1 : 0);
// To check for consistent under(+)/over_shooting(-) of target rate.
sum_rate_MM_sgn_ += sgnDiff;
// Update with the current new target and frame rate:
// these values are ones the encoder will use for the current/next ~1sec.
target_bitrate_ = target_bitrate;
incoming_framerate_ = incoming_framerate;
sum_incoming_framerate_ += incoming_framerate_;
// Update the per_frame_bandwidth:
// this is the per_frame_bw for the current/next ~1sec.
per_frame_bandwidth_ = 0.0f;
if (incoming_framerate_ > 0.0f) {
per_frame_bandwidth_ = target_bitrate_ / incoming_framerate_;
}
}
// Select the resolution factors: frame size and frame rate change (qm scales).
// Selection is for going down in resolution, or for going back up
// (if a previous down-sampling action was taken).
// In the current version the following constraints are imposed:
// 1) We only allow for one action, either down or up, at a given time.
// 2) The possible down-sampling actions are: spatial by 1/2x1/2, 3/4x3/4;
// temporal/frame rate reduction by 1/2 and 2/3.
// 3) The action for going back up is the reverse of last (spatial or temporal)
// down-sampling action. The list of down-sampling actions from the
// Initialize() state are kept in |down_action_history_|.
// 4) The total amount of down-sampling (spatial and/or temporal) from the
// Initialize() state (native resolution) is limited by various factors.
int VCMQmResolution::SelectResolution(VCMResolutionScale** qm) {
if (!init_) {
return VCM_UNINITIALIZED;
}
if (content_metrics_ == NULL) {
Reset();
*qm = qm_;
return VCM_OK;
}
// Check conditions on down-sampling state.
assert(state_dec_factor_spatial_ >= 1.0f);
assert(state_dec_factor_temporal_ >= 1.0f);
assert(state_dec_factor_spatial_ <= kMaxSpatialDown);
assert(state_dec_factor_temporal_ <= kMaxTempDown);
assert(state_dec_factor_temporal_ * state_dec_factor_spatial_ <=
kMaxTotalDown);
// Compute content class for selection.
content_class_ = ComputeContentClass();
// Compute various rate quantities for selection.
ComputeRatesForSelection();
// Get the encoder state.
ComputeEncoderState();
// Default settings: no action.
SetDefaultAction();
*qm = qm_;
// Check for going back up in resolution, if we have had some down-sampling
// relative to native state in Initialize().
if (down_action_history_[0].spatial != kNoChangeSpatial ||
down_action_history_[0].temporal != kNoChangeTemporal) {
if (GoingUpResolution()) {
*qm = qm_;
return VCM_OK;
}
}
// Check for going down in resolution.
if (GoingDownResolution()) {
*qm = qm_;
return VCM_OK;
}
return VCM_OK;
}
void VCMQmResolution::SetDefaultAction() {
qm_->codec_width = width_;
qm_->codec_height = height_;
qm_->frame_rate = user_frame_rate_;
qm_->change_resolution_spatial = false;
qm_->change_resolution_temporal = false;
qm_->spatial_width_fact = 1.0f;
qm_->spatial_height_fact = 1.0f;
qm_->temporal_fact = 1.0f;
action_.spatial = kNoChangeSpatial;
action_.temporal = kNoChangeTemporal;
}
void VCMQmResolution::ComputeRatesForSelection() {
avg_target_rate_ = 0.0f;
avg_incoming_framerate_ = 0.0f;
avg_ratio_buffer_low_ = 0.0f;
avg_rate_mismatch_ = 0.0f;
avg_rate_mismatch_sgn_ = 0.0f;
avg_packet_loss_ = 0.0f;
if (frame_cnt_ > 0) {
avg_ratio_buffer_low_ =
static_cast<float>(low_buffer_cnt_) / static_cast<float>(frame_cnt_);
}
if (update_rate_cnt_ > 0) {
avg_rate_mismatch_ =
static_cast<float>(sum_rate_MM_) / static_cast<float>(update_rate_cnt_);
avg_rate_mismatch_sgn_ = static_cast<float>(sum_rate_MM_sgn_) /
static_cast<float>(update_rate_cnt_);
avg_target_rate_ = static_cast<float>(sum_target_rate_) /
static_cast<float>(update_rate_cnt_);
avg_incoming_framerate_ = static_cast<float>(sum_incoming_framerate_) /
static_cast<float>(update_rate_cnt_);
avg_packet_loss_ = static_cast<float>(sum_packet_loss_) /
static_cast<float>(update_rate_cnt_);
}
// For selection we may want to weight some quantities more heavily
// with the current (i.e., next ~1sec) rate values.
avg_target_rate_ =
kWeightRate * avg_target_rate_ + (1.0 - kWeightRate) * target_bitrate_;
avg_incoming_framerate_ = kWeightRate * avg_incoming_framerate_ +
(1.0 - kWeightRate) * incoming_framerate_;
// Use base layer frame rate for temporal layers: this will favor spatial.
assert(num_layers_ > 0);
framerate_level_ = FrameRateLevel(avg_incoming_framerate_ /
static_cast<float>(1 << (num_layers_ - 1)));
}
void VCMQmResolution::ComputeEncoderState() {
// Default.
encoder_state_ = kStableEncoding;
// Assign stressed state if:
// 1) occurrences of low buffer levels is high, or
// 2) rate mis-match is high, and consistent over-shooting by encoder.
if ((avg_ratio_buffer_low_ > kMaxBufferLow) ||
((avg_rate_mismatch_ > kMaxRateMisMatch) &&
(avg_rate_mismatch_sgn_ < -kRateOverShoot))) {
encoder_state_ = kStressedEncoding;
}
// Assign easy state if:
// 1) rate mis-match is high, and
// 2) consistent under-shooting by encoder.
if ((avg_rate_mismatch_ > kMaxRateMisMatch) &&
(avg_rate_mismatch_sgn_ > kRateUnderShoot)) {
encoder_state_ = kEasyEncoding;
}
}
bool VCMQmResolution::GoingUpResolution() {
// For going up, we check for undoing the previous down-sampling action.
float fac_width = kFactorWidthSpatial[down_action_history_[0].spatial];
float fac_height = kFactorHeightSpatial[down_action_history_[0].spatial];
float fac_temp = kFactorTemporal[down_action_history_[0].temporal];
// For going up spatially, we allow for going up by 3/4x3/4 at each stage.
// So if the last spatial action was 1/2x1/2 it would be undone in 2 stages.
// Modify the fac_width/height for this case.
if (down_action_history_[0].spatial == kOneQuarterSpatialUniform) {
fac_width = kFactorWidthSpatial[kOneQuarterSpatialUniform] /
kFactorWidthSpatial[kOneHalfSpatialUniform];
fac_height = kFactorHeightSpatial[kOneQuarterSpatialUniform] /
kFactorHeightSpatial[kOneHalfSpatialUniform];
}
// Check if we should go up both spatially and temporally.
if (down_action_history_[0].spatial != kNoChangeSpatial &&
down_action_history_[0].temporal != kNoChangeTemporal) {
if (ConditionForGoingUp(fac_width, fac_height, fac_temp,
kTransRateScaleUpSpatialTemp)) {
action_.spatial = down_action_history_[0].spatial;
action_.temporal = down_action_history_[0].temporal;
UpdateDownsamplingState(kUpResolution);
return true;
}
}
// Check if we should go up either spatially or temporally.
bool selected_up_spatial = false;
bool selected_up_temporal = false;
if (down_action_history_[0].spatial != kNoChangeSpatial) {
selected_up_spatial = ConditionForGoingUp(fac_width, fac_height, 1.0f,
kTransRateScaleUpSpatial);
}
if (down_action_history_[0].temporal != kNoChangeTemporal) {
selected_up_temporal =
ConditionForGoingUp(1.0f, 1.0f, fac_temp, kTransRateScaleUpTemp);
}
if (selected_up_spatial && !selected_up_temporal) {
action_.spatial = down_action_history_[0].spatial;
action_.temporal = kNoChangeTemporal;
UpdateDownsamplingState(kUpResolution);
return true;
} else if (!selected_up_spatial && selected_up_temporal) {
action_.spatial = kNoChangeSpatial;
action_.temporal = down_action_history_[0].temporal;
UpdateDownsamplingState(kUpResolution);
return true;
} else if (selected_up_spatial && selected_up_temporal) {
PickSpatialOrTemporal();
UpdateDownsamplingState(kUpResolution);
return true;
}
return false;
}
bool VCMQmResolution::ConditionForGoingUp(float fac_width,
float fac_height,
float fac_temp,
float scale_fac) {
float estimated_transition_rate_up =
GetTransitionRate(fac_width, fac_height, fac_temp, scale_fac);
// Go back up if:
// 1) target rate is above threshold and current encoder state is stable, or
// 2) encoder state is easy (encoder is significantly under-shooting target).
if (((avg_target_rate_ > estimated_transition_rate_up) &&
(encoder_state_ == kStableEncoding)) ||
(encoder_state_ == kEasyEncoding)) {
return true;
} else {
return false;
}
}
bool VCMQmResolution::GoingDownResolution() {
float estimated_transition_rate_down =
GetTransitionRate(1.0f, 1.0f, 1.0f, 1.0f);
float max_rate = kFrameRateFac[framerate_level_] * kMaxRateQm[image_type_];
// Resolution reduction if:
// (1) target rate is below transition rate, or
// (2) encoder is in stressed state and target rate below a max threshold.
if ((avg_target_rate_ < estimated_transition_rate_down) ||
(encoder_state_ == kStressedEncoding && avg_target_rate_ < max_rate)) {
// Get the down-sampling action: based on content class, and how low
// average target rate is relative to transition rate.
uint8_t spatial_fact =
kSpatialAction[content_class_ +
9 * RateClass(estimated_transition_rate_down)];
uint8_t temp_fact =
kTemporalAction[content_class_ +
9 * RateClass(estimated_transition_rate_down)];
switch (spatial_fact) {
case 4: {
action_.spatial = kOneQuarterSpatialUniform;
break;
}
case 2: {
action_.spatial = kOneHalfSpatialUniform;
break;
}
case 1: {
action_.spatial = kNoChangeSpatial;
break;
}
default: { assert(false); }
}
switch (temp_fact) {
case 3: {
action_.temporal = kTwoThirdsTemporal;
break;
}
case 2: {
action_.temporal = kOneHalfTemporal;
break;
}
case 1: {
action_.temporal = kNoChangeTemporal;
break;
}
default: { assert(false); }
}
// Only allow for one action (spatial or temporal) at a given time.
assert(action_.temporal == kNoChangeTemporal ||
action_.spatial == kNoChangeSpatial);
// Adjust cases not captured in tables, mainly based on frame rate, and
// also check for odd frame sizes.
AdjustAction();
// Update down-sampling state.
if (action_.spatial != kNoChangeSpatial ||
action_.temporal != kNoChangeTemporal) {
UpdateDownsamplingState(kDownResolution);
return true;
}
}
return false;
}
float VCMQmResolution::GetTransitionRate(float fac_width,
float fac_height,
float fac_temp,
float scale_fac) {
ImageType image_type =
GetImageType(static_cast<uint16_t>(fac_width * width_),
static_cast<uint16_t>(fac_height * height_));
FrameRateLevelClass framerate_level =
FrameRateLevel(fac_temp * avg_incoming_framerate_);
// If we are checking for going up temporally, and this is the last
// temporal action, then use native frame rate.
if (down_action_history_[1].temporal == kNoChangeTemporal &&
fac_temp > 1.0f) {
framerate_level = FrameRateLevel(native_frame_rate_);
}
// The maximum allowed rate below which down-sampling is allowed:
// Nominal values based on image format (frame size and frame rate).
float max_rate = kFrameRateFac[framerate_level] * kMaxRateQm[image_type];
uint8_t image_class = image_type > kVGA ? 1 : 0;
uint8_t table_index = image_class * 9 + content_class_;
// Scale factor for down-sampling transition threshold:
// factor based on the content class and the image size.
float scaleTransRate = kScaleTransRateQm[table_index];
// Threshold bitrate for resolution action.
return static_cast<float>(scale_fac * scaleTransRate * max_rate);
}
void VCMQmResolution::UpdateDownsamplingState(UpDownAction up_down) {
if (up_down == kUpResolution) {
qm_->spatial_width_fact = 1.0f / kFactorWidthSpatial[action_.spatial];
qm_->spatial_height_fact = 1.0f / kFactorHeightSpatial[action_.spatial];
// If last spatial action was 1/2x1/2, we undo it in two steps, so the
// spatial scale factor in this first step is modified as (4.0/3.0 / 2.0).
if (action_.spatial == kOneQuarterSpatialUniform) {
qm_->spatial_width_fact = 1.0f *
kFactorWidthSpatial[kOneHalfSpatialUniform] /
kFactorWidthSpatial[kOneQuarterSpatialUniform];
qm_->spatial_height_fact =
1.0f * kFactorHeightSpatial[kOneHalfSpatialUniform] /
kFactorHeightSpatial[kOneQuarterSpatialUniform];
}
qm_->temporal_fact = 1.0f / kFactorTemporal[action_.temporal];
RemoveLastDownAction();
} else if (up_down == kDownResolution) {
ConstrainAmountOfDownSampling();
ConvertSpatialFractionalToWhole();
qm_->spatial_width_fact = kFactorWidthSpatial[action_.spatial];
qm_->spatial_height_fact = kFactorHeightSpatial[action_.spatial];
qm_->temporal_fact = kFactorTemporal[action_.temporal];
InsertLatestDownAction();
} else {
// This function should only be called if either the Up or Down action
// has been selected.
assert(false);
}
UpdateCodecResolution();
state_dec_factor_spatial_ = state_dec_factor_spatial_ *
qm_->spatial_width_fact *
qm_->spatial_height_fact;
state_dec_factor_temporal_ = state_dec_factor_temporal_ * qm_->temporal_fact;
}
void VCMQmResolution::UpdateCodecResolution() {
if (action_.spatial != kNoChangeSpatial) {
qm_->change_resolution_spatial = true;
qm_->codec_width =
static_cast<uint16_t>(width_ / qm_->spatial_width_fact + 0.5f);
qm_->codec_height =
static_cast<uint16_t>(height_ / qm_->spatial_height_fact + 0.5f);
// Size should not exceed native sizes.
assert(qm_->codec_width <= native_width_);
assert(qm_->codec_height <= native_height_);
// New sizes should be multiple of 2, otherwise spatial should not have
// been selected.
assert(qm_->codec_width % 2 == 0);
assert(qm_->codec_height % 2 == 0);
}
if (action_.temporal != kNoChangeTemporal) {
qm_->change_resolution_temporal = true;
// Update the frame rate based on the average incoming frame rate.
qm_->frame_rate = avg_incoming_framerate_ / qm_->temporal_fact + 0.5f;
if (down_action_history_[0].temporal == 0) {
// When we undo the last temporal-down action, make sure we go back up
// to the native frame rate. Since the incoming frame rate may
// fluctuate over time, |avg_incoming_framerate_| scaled back up may
// be smaller than |native_frame rate_|.
qm_->frame_rate = native_frame_rate_;
}
}
}
uint8_t VCMQmResolution::RateClass(float transition_rate) {
return avg_target_rate_ < (kFacLowRate * transition_rate)
? 0
: (avg_target_rate_ >= transition_rate ? 2 : 1);
}
// TODO(marpan): Would be better to capture these frame rate adjustments by
// extending the table data (qm_select_data.h).
void VCMQmResolution::AdjustAction() {
// If the spatial level is default state (neither low or high), motion level
// is not high, and spatial action was selected, switch to 2/3 frame rate
// reduction if the average incoming frame rate is high.
if (spatial_.level == kDefault && motion_.level != kHigh &&
action_.spatial != kNoChangeSpatial &&
framerate_level_ == kFrameRateHigh) {
action_.spatial = kNoChangeSpatial;
action_.temporal = kTwoThirdsTemporal;
}
// If both motion and spatial level are low, and temporal down action was
// selected, switch to spatial 3/4x3/4 if the frame rate is not above the
// lower middle level (|kFrameRateMiddle1|).
if (motion_.level == kLow && spatial_.level == kLow &&
framerate_level_ <= kFrameRateMiddle1 &&
action_.temporal != kNoChangeTemporal) {
action_.spatial = kOneHalfSpatialUniform;
action_.temporal = kNoChangeTemporal;
}
// If spatial action is selected, and there has been too much spatial
// reduction already (i.e., 1/4), then switch to temporal action if the
// average frame rate is not low.
if (action_.spatial != kNoChangeSpatial &&
down_action_history_[0].spatial == kOneQuarterSpatialUniform &&
framerate_level_ != kFrameRateLow) {
action_.spatial = kNoChangeSpatial;
action_.temporal = kTwoThirdsTemporal;
}
// Never use temporal action if number of temporal layers is above 2.
if (num_layers_ > 2) {
if (action_.temporal != kNoChangeTemporal) {
action_.spatial = kOneHalfSpatialUniform;
}
action_.temporal = kNoChangeTemporal;
}
// If spatial action was selected, we need to make sure the frame sizes
// are multiples of two. Otherwise switch to 2/3 temporal.
if (action_.spatial != kNoChangeSpatial && !EvenFrameSize()) {
action_.spatial = kNoChangeSpatial;
// Only one action (spatial or temporal) is allowed at a given time, so need
// to check whether temporal action is currently selected.
action_.temporal = kTwoThirdsTemporal;
}
}
void VCMQmResolution::ConvertSpatialFractionalToWhole() {
// If 3/4 spatial is selected, check if there has been another 3/4,
// and if so, combine them into 1/2. 1/2 scaling is more efficient than 9/16.
// Note we define 3/4x3/4 spatial as kOneHalfSpatialUniform.
if (action_.spatial == kOneHalfSpatialUniform) {
bool found = false;
int isel = kDownActionHistorySize;
for (int i = 0; i < kDownActionHistorySize; ++i) {
if (down_action_history_[i].spatial == kOneHalfSpatialUniform) {
isel = i;
found = true;
break;
}
}
if (found) {
action_.spatial = kOneQuarterSpatialUniform;
state_dec_factor_spatial_ =
state_dec_factor_spatial_ /
(kFactorWidthSpatial[kOneHalfSpatialUniform] *
kFactorHeightSpatial[kOneHalfSpatialUniform]);
// Check if switching to 1/2x1/2 (=1/4) spatial is allowed.
ConstrainAmountOfDownSampling();
if (action_.spatial == kNoChangeSpatial) {
// Not allowed. Go back to 3/4x3/4 spatial.
action_.spatial = kOneHalfSpatialUniform;
state_dec_factor_spatial_ =
state_dec_factor_spatial_ *
kFactorWidthSpatial[kOneHalfSpatialUniform] *
kFactorHeightSpatial[kOneHalfSpatialUniform];
} else {
// Switching is allowed. Remove 3/4x3/4 from the history, and update
// the frame size.
for (int i = isel; i < kDownActionHistorySize - 1; ++i) {
down_action_history_[i].spatial = down_action_history_[i + 1].spatial;
}
width_ = width_ * kFactorWidthSpatial[kOneHalfSpatialUniform];
height_ = height_ * kFactorHeightSpatial[kOneHalfSpatialUniform];
}
}
}
}
// Returns false if the new frame sizes, under the current spatial action,
// are not multiples of two.
bool VCMQmResolution::EvenFrameSize() {
if (action_.spatial == kOneHalfSpatialUniform) {
if ((width_ * 3 / 4) % 2 != 0 || (height_ * 3 / 4) % 2 != 0) {
return false;
}
} else if (action_.spatial == kOneQuarterSpatialUniform) {
if ((width_ * 1 / 2) % 2 != 0 || (height_ * 1 / 2) % 2 != 0) {
return false;
}
}
return true;
}
void VCMQmResolution::InsertLatestDownAction() {
if (action_.spatial != kNoChangeSpatial) {
for (int i = kDownActionHistorySize - 1; i > 0; --i) {
down_action_history_[i].spatial = down_action_history_[i - 1].spatial;
}
down_action_history_[0].spatial = action_.spatial;
}
if (action_.temporal != kNoChangeTemporal) {
for (int i = kDownActionHistorySize - 1; i > 0; --i) {
down_action_history_[i].temporal = down_action_history_[i - 1].temporal;
}
down_action_history_[0].temporal = action_.temporal;
}
}
void VCMQmResolution::RemoveLastDownAction() {
if (action_.spatial != kNoChangeSpatial) {
// If the last spatial action was 1/2x1/2 we replace it with 3/4x3/4.
if (action_.spatial == kOneQuarterSpatialUniform) {
down_action_history_[0].spatial = kOneHalfSpatialUniform;
} else {
for (int i = 0; i < kDownActionHistorySize - 1; ++i) {
down_action_history_[i].spatial = down_action_history_[i + 1].spatial;
}
down_action_history_[kDownActionHistorySize - 1].spatial =
kNoChangeSpatial;
}
}
if (action_.temporal != kNoChangeTemporal) {
for (int i = 0; i < kDownActionHistorySize - 1; ++i) {
down_action_history_[i].temporal = down_action_history_[i + 1].temporal;
}
down_action_history_[kDownActionHistorySize - 1].temporal =
kNoChangeTemporal;
}
}
void VCMQmResolution::ConstrainAmountOfDownSampling() {
// Sanity checks on down-sampling selection:
// override the settings for too small image size and/or frame rate.
// Also check the limit on current down-sampling states.
float spatial_width_fact = kFactorWidthSpatial[action_.spatial];
float spatial_height_fact = kFactorHeightSpatial[action_.spatial];
float temporal_fact = kFactorTemporal[action_.temporal];
float new_dec_factor_spatial =
state_dec_factor_spatial_ * spatial_width_fact * spatial_height_fact;
float new_dec_factor_temp = state_dec_factor_temporal_ * temporal_fact;
// No spatial sampling if current frame size is too small, or if the
// amount of spatial down-sampling is above maximum spatial down-action.
if ((width_ * height_) <= kMinImageSize ||
new_dec_factor_spatial > kMaxSpatialDown) {
action_.spatial = kNoChangeSpatial;
new_dec_factor_spatial = state_dec_factor_spatial_;
}
// No frame rate reduction if average frame rate is below some point, or if
// the amount of temporal down-sampling is above maximum temporal down-action.
if (avg_incoming_framerate_ <= kMinFrameRate ||
new_dec_factor_temp > kMaxTempDown) {
action_.temporal = kNoChangeTemporal;
new_dec_factor_temp = state_dec_factor_temporal_;
}
// Check if the total (spatial-temporal) down-action is above maximum allowed,
// if so, disallow the current selected down-action.
if (new_dec_factor_spatial * new_dec_factor_temp > kMaxTotalDown) {
if (action_.spatial != kNoChangeSpatial) {
action_.spatial = kNoChangeSpatial;
} else if (action_.temporal != kNoChangeTemporal) {
action_.temporal = kNoChangeTemporal;
} else {
// We only allow for one action (spatial or temporal) at a given time, so
// either spatial or temporal action is selected when this function is
// called. If the selected action is disallowed from one of the above
// 2 prior conditions (on spatial & temporal max down-action), then this
// condition "total down-action > |kMaxTotalDown|" would not be entered.
assert(false);
}
}
}
void VCMQmResolution::PickSpatialOrTemporal() {
// Pick the one that has had the most down-sampling thus far.
if (state_dec_factor_spatial_ > state_dec_factor_temporal_) {
action_.spatial = down_action_history_[0].spatial;
action_.temporal = kNoChangeTemporal;
} else {
action_.spatial = kNoChangeSpatial;
action_.temporal = down_action_history_[0].temporal;
}
}
// TODO(marpan): Update when we allow for directional spatial down-sampling.
void VCMQmResolution::SelectSpatialDirectionMode(float transition_rate) {
// Default is 4/3x4/3
// For bit rates well below transitional rate, we select 2x2.
if (avg_target_rate_ < transition_rate * kRateRedSpatial2X2) {
qm_->spatial_width_fact = 2.0f;
qm_->spatial_height_fact = 2.0f;
}
// Otherwise check prediction errors and aspect ratio.
float spatial_err = 0.0f;
float spatial_err_h = 0.0f;
float spatial_err_v = 0.0f;
if (content_metrics_) {
spatial_err = content_metrics_->spatial_pred_err;
spatial_err_h = content_metrics_->spatial_pred_err_h;
spatial_err_v = content_metrics_->spatial_pred_err_v;
}
// Favor 1x2 if aspect_ratio is 16:9.
if (aspect_ratio_ >= 16.0f / 9.0f) {
// Check if 1x2 has lowest prediction error.
if (spatial_err_h < spatial_err && spatial_err_h < spatial_err_v) {
qm_->spatial_width_fact = 2.0f;
qm_->spatial_height_fact = 1.0f;
}
}
// Check for 4/3x4/3 selection: favor 2x2 over 1x2 and 2x1.
if (spatial_err < spatial_err_h * (1.0f + kSpatialErr2x2VsHoriz) &&
spatial_err < spatial_err_v * (1.0f + kSpatialErr2X2VsVert)) {
qm_->spatial_width_fact = 4.0f / 3.0f;
qm_->spatial_height_fact = 4.0f / 3.0f;
}
// Check for 2x1 selection.
if (spatial_err_v < spatial_err_h * (1.0f - kSpatialErrVertVsHoriz) &&
spatial_err_v < spatial_err * (1.0f - kSpatialErr2X2VsVert)) {
qm_->spatial_width_fact = 1.0f;
qm_->spatial_height_fact = 2.0f;
}
}
} // namespace webrtc

326
webrtc/modules/video_coding/qm_select.h
View File

@ -1,326 +0,0 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_H_
#define WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_H_
#include "webrtc/common_types.h"
#include "webrtc/typedefs.h"
/******************************************************/
/* Quality Modes: Resolution and Robustness settings */
/******************************************************/
namespace webrtc {
struct VideoContentMetrics;
struct VCMResolutionScale {
VCMResolutionScale()
: codec_width(640),
codec_height(480),
frame_rate(30.0f),
spatial_width_fact(1.0f),
spatial_height_fact(1.0f),
temporal_fact(1.0f),
change_resolution_spatial(false),
change_resolution_temporal(false) {}
uint16_t codec_width;
uint16_t codec_height;
float frame_rate;
float spatial_width_fact;
float spatial_height_fact;
float temporal_fact;
bool change_resolution_spatial;
bool change_resolution_temporal;
};
enum ImageType {
kQCIF = 0, // 176x144
kHCIF, // 264x216 = half(~3/4x3/4) CIF.
kQVGA, // 320x240 = quarter VGA.
kCIF, // 352x288
kHVGA, // 480x360 = half(~3/4x3/4) VGA.
kVGA, // 640x480
kQFULLHD, // 960x540 = quarter FULLHD, and half(~3/4x3/4) WHD.
kWHD, // 1280x720
kFULLHD, // 1920x1080
kNumImageTypes
};
const uint32_t kSizeOfImageType[kNumImageTypes] = {
25344, 57024, 76800, 101376, 172800, 307200, 518400, 921600, 2073600};
enum FrameRateLevelClass {
kFrameRateLow,
kFrameRateMiddle1,
kFrameRateMiddle2,
kFrameRateHigh
};
enum ContentLevelClass { kLow, kHigh, kDefault };
struct VCMContFeature {
VCMContFeature() : value(0.0f), level(kDefault) {}
void Reset() {
value = 0.0f;
level = kDefault;
}
float value;
ContentLevelClass level;
};
enum UpDownAction { kUpResolution, kDownResolution };
enum SpatialAction {
kNoChangeSpatial,
kOneHalfSpatialUniform, // 3/4 x 3/4: 9/6 ~1/2 pixel reduction.
kOneQuarterSpatialUniform, // 1/2 x 1/2: 1/4 pixel reduction.
kNumModesSpatial
};
enum TemporalAction {
kNoChangeTemporal,
kTwoThirdsTemporal, // 2/3 frame rate reduction
kOneHalfTemporal, // 1/2 frame rate reduction
kNumModesTemporal
};
struct ResolutionAction {
ResolutionAction() : spatial(kNoChangeSpatial), temporal(kNoChangeTemporal) {}
SpatialAction spatial;
TemporalAction temporal;
};
// Down-sampling factors for spatial (width and height), and temporal.
const float kFactorWidthSpatial[kNumModesSpatial] = {1.0f, 4.0f / 3.0f, 2.0f};
const float kFactorHeightSpatial[kNumModesSpatial] = {1.0f, 4.0f / 3.0f, 2.0f};
const float kFactorTemporal[kNumModesTemporal] = {1.0f, 1.5f, 2.0f};
enum EncoderState {
kStableEncoding, // Low rate mis-match, stable buffer levels.
kStressedEncoding, // Significant over-shooting of target rate,
// Buffer under-flow, etc.
kEasyEncoding // Significant under-shooting of target rate.
};
// QmMethod class: main class for resolution and robustness settings
class VCMQmMethod {
public:
VCMQmMethod();
virtual ~VCMQmMethod();
// Reset values
void ResetQM();
virtual void Reset() = 0;
// Compute content class.
uint8_t ComputeContentClass();
// Update with the content metrics.
void UpdateContent(const VideoContentMetrics* content_metrics);
// Compute spatial texture magnitude and level.
// Spatial texture is a spatial prediction error measure.
void ComputeSpatial();
// Compute motion magnitude and level for NFD metric.
// NFD is normalized frame difference (normalized by spatial variance).
void ComputeMotionNFD();
// Get the imageType (CIF, VGA, HD, etc) for the system width/height.
ImageType GetImageType(uint16_t width, uint16_t height);
// Return the closest image type.
ImageType FindClosestImageType(uint16_t width, uint16_t height);
// Get the frame rate level.
FrameRateLevelClass FrameRateLevel(float frame_rate);
protected:
// Content Data.
const VideoContentMetrics* content_metrics_;
// Encoder frame sizes and native frame sizes.
uint16_t width_;
uint16_t height_;
float user_frame_rate_;
uint16_t native_width_;
uint16_t native_height_;
float native_frame_rate_;
float aspect_ratio_;
// Image type and frame rate leve, for the current encoder resolution.
ImageType image_type_;
FrameRateLevelClass framerate_level_;
// Content class data.
VCMContFeature motion_;
VCMContFeature spatial_;
uint8_t content_class_;
bool init_;
};
// Resolution settings class
class VCMQmResolution : public VCMQmMethod {
public:
VCMQmResolution();
virtual ~VCMQmResolution();
// Reset all quantities.
virtual void Reset();
// Reset rate quantities and counters after every SelectResolution() call.
void ResetRates();
// Reset down-sampling state.
void ResetDownSamplingState();
// Get the encoder state.
EncoderState GetEncoderState();
// Initialize after SetEncodingData in media_opt.
int Initialize(float bitrate,
float user_framerate,
uint16_t width,
uint16_t height,
int num_layers);
// Update the encoder frame size.
void UpdateCodecParameters(float frame_rate, uint16_t width, uint16_t height);
// Update with actual bit rate (size of the latest encoded frame)
// and frame type, after every encoded frame.
void UpdateEncodedSize(size_t encoded_size);
// Update with new target bitrate, actual encoder sent rate, frame_rate,
// loss rate: every ~1 sec from SetTargetRates in media_opt.
void UpdateRates(float target_bitrate,
float encoder_sent_rate,
float incoming_framerate,
uint8_t packet_loss);
// Extract ST (spatio-temporal) resolution action.
// Inputs: qm: Reference to the quality modes pointer.
// Output: the spatial and/or temporal scale change.
int SelectResolution(VCMResolutionScale** qm);
private:
// Set the default resolution action.
void SetDefaultAction();
// Compute rates for the selection of down-sampling action.
void ComputeRatesForSelection();
// Compute the encoder state.
void ComputeEncoderState();
// Return true if the action is to go back up in resolution.
bool GoingUpResolution();
// Return true if the action is to go down in resolution.
bool GoingDownResolution();
// Check the condition for going up in resolution by the scale factors:
// |facWidth|, |facHeight|, |facTemp|.
// |scaleFac| is a scale factor for the transition rate.
bool ConditionForGoingUp(float fac_width,
float fac_height,
float fac_temp,
float scale_fac);
// Get the bitrate threshold for the resolution action.
// The case |facWidth|=|facHeight|=|facTemp|==1 is for down-sampling action.
// |scaleFac| is a scale factor for the transition rate.
float GetTransitionRate(float fac_width,
float fac_height,
float fac_temp,
float scale_fac);
// Update the down-sampling state.
void UpdateDownsamplingState(UpDownAction up_down);
// Update the codec frame size and frame rate.
void UpdateCodecResolution();
// Return a state based on average target rate relative transition rate.
uint8_t RateClass(float transition_rate);
// Adjust the action selected from the table.
void AdjustAction();
// Covert 2 stages of 3/4 (=9/16) spatial decimation to 1/2.
void ConvertSpatialFractionalToWhole();
// Returns true if the new frame sizes, under the selected spatial action,
// are of even size.
bool EvenFrameSize();
// Insert latest down-sampling action into the history list.
void InsertLatestDownAction();
// Remove the last (first element) down-sampling action from the list.
void RemoveLastDownAction();
// Check constraints on the amount of down-sampling allowed.
void ConstrainAmountOfDownSampling();
// For going up in resolution: pick spatial or temporal action,
// if both actions were separately selected.
void PickSpatialOrTemporal();
// Select the directional (1x2 or 2x1) spatial down-sampling action.
void SelectSpatialDirectionMode(float transition_rate);
enum { kDownActionHistorySize = 10 };
VCMResolutionScale* qm_;
// Encoder rate control parameters.
float target_bitrate_;
float incoming_framerate_;
float per_frame_bandwidth_;
float buffer_level_;
// Data accumulated every ~1sec from MediaOpt.
float sum_target_rate_;
float sum_incoming_framerate_;
float sum_rate_MM_;
float sum_rate_MM_sgn_;
float sum_packet_loss_;
// Counters.
uint32_t frame_cnt_;
uint32_t frame_cnt_delta_;
uint32_t update_rate_cnt_;
uint32_t low_buffer_cnt_;
// Resolution state parameters.
float state_dec_factor_spatial_;
float state_dec_factor_temporal_;
// Quantities used for selection.
float avg_target_rate_;
float avg_incoming_framerate_;
float avg_ratio_buffer_low_;
float avg_rate_mismatch_;
float avg_rate_mismatch_sgn_;
float avg_packet_loss_;
EncoderState encoder_state_;
ResolutionAction action_;
// Short history of the down-sampling actions from the Initialize() state.
// This is needed for going up in resolution. Since the total amount of
// down-sampling actions are constrained, the length of the list need not be
// large: i.e., (4/3) ^{kDownActionHistorySize} <= kMaxDownSample.
ResolutionAction down_action_history_[kDownActionHistorySize];
int num_layers_;
};
} // namespace webrtc
#endif // WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_H_

227
webrtc/modules/video_coding/qm_select_data.h
View File

@ -1,227 +0,0 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_DATA_H_
#define WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_DATA_H_
/***************************************************************
*QMSelectData.h
* This file includes parameters for content-aware media optimization
****************************************************************/
#include "webrtc/typedefs.h"
namespace webrtc {
//
// PARAMETERS FOR RESOLUTION ADAPTATION
//
// Initial level of buffer in secs.
const float kInitBufferLevel = 0.5f;
// Threshold of (max) buffer size below which we consider too low (underflow).
const float kPercBufferThr = 0.10f;
// Threshold on the occurrences of low buffer levels.
const float kMaxBufferLow = 0.30f;
// Threshold on rate mismatch.
const float kMaxRateMisMatch = 0.5f;
// Threshold on amount of under/over encoder shooting.
const float kRateOverShoot = 0.75f;
const float kRateUnderShoot = 0.75f;
// Factor to favor weighting the average rates with the current/last data.
const float kWeightRate = 0.70f;
// Factor for transitional rate for going back up in resolution.
const float kTransRateScaleUpSpatial = 1.25f;
const float kTransRateScaleUpTemp = 1.25f;
const float kTransRateScaleUpSpatialTemp = 1.25f;
// Threshold on packet loss rate, above which favor resolution reduction.
const float kPacketLossThr = 0.1f;
// Factor for reducing transitional bitrate under packet loss.
const float kPacketLossRateFac = 1.0f;
// Maximum possible transitional rate for down-sampling:
// (units in kbps), for 30fps.
const uint16_t kMaxRateQm[9] = {
0, // QCIF
50, // kHCIF
125, // kQVGA
200, // CIF
280, // HVGA
400, // VGA
700, // QFULLHD
1000, // WHD
1500 // FULLHD
};
// Frame rate scale for maximum transition rate.
const float kFrameRateFac[4] = {
0.5f, // Low
0.7f, // Middle level 1
0.85f, // Middle level 2
1.0f, // High
};
// Scale for transitional rate: based on content class
// motion=L/H/D,spatial==L/H/D: for low, high, middle levels
const float kScaleTransRateQm[18] = {
// VGA and lower
0.40f, // L, L
0.50f, // L, H
0.40f, // L, D
0.60f, // H ,L
0.60f, // H, H
0.60f, // H, D
0.50f, // D, L
0.50f, // D, D
0.50f, // D, H
// over VGA
0.40f, // L, L
0.50f, // L, H
0.40f, // L, D
0.60f, // H ,L
0.60f, // H, H
0.60f, // H, D
0.50f, // D, L
0.50f, // D, D
0.50f, // D, H
};
// Threshold on the target rate relative to transitional rate.
const float kFacLowRate = 0.5f;
// Action for down-sampling:
// motion=L/H/D,spatial==L/H/D, for low, high, middle levels;
// rate = 0/1/2, for target rate state relative to transition rate.
const uint8_t kSpatialAction[27] = {
// rateClass = 0:
1, // L, L
1, // L, H
1, // L, D
4, // H ,L
1, // H, H
4, // H, D
4, // D, L
1, // D, H
2, // D, D
// rateClass = 1:
1, // L, L
1, // L, H
1, // L, D
2, // H ,L
1, // H, H
2, // H, D
2, // D, L
1, // D, H
2, // D, D
// rateClass = 2:
1, // L, L
1, // L, H
1, // L, D
2, // H ,L
1, // H, H
2, // H, D
2, // D, L
1, // D, H
2, // D, D
};
const uint8_t kTemporalAction[27] = {
// rateClass = 0:
3, // L, L
2, // L, H
2, // L, D
1, // H ,L
3, // H, H
1, // H, D
1, // D, L
2, // D, H
1, // D, D
// rateClass = 1:
3, // L, L
3, // L, H
3, // L, D
1, // H ,L
3, // H, H
1, // H, D
1, // D, L
3, // D, H
1, // D, D
// rateClass = 2:
1, // L, L
3, // L, H
3, // L, D
1, // H ,L
3, // H, H
1, // H, D
1, // D, L
3, // D, H
1, // D, D
};
// Control the total amount of down-sampling allowed.
const float kMaxSpatialDown = 8.0f;
const float kMaxTempDown = 3.0f;
const float kMaxTotalDown = 9.0f;
// Minimum image size for a spatial down-sampling.
const int kMinImageSize = 176 * 144;
// Minimum frame rate for temporal down-sampling:
// no frame rate reduction if incomingFrameRate <= MIN_FRAME_RATE.
const int kMinFrameRate = 8;
//
// PARAMETERS FOR FEC ADJUSTMENT: TODO (marpan)
//
//
// PARAMETETS FOR SETTING LOW/HIGH STATES OF CONTENT METRICS:
//
// Thresholds for frame rate:
const int kLowFrameRate = 10;
const int kMiddleFrameRate = 15;
const int kHighFrameRate = 25;
// Thresholds for motion: motion level is from NFD.
const float kHighMotionNfd = 0.075f;
const float kLowMotionNfd = 0.03f;
// Thresholds for spatial prediction error:
// this is applied on the average of (2x2,1x2,2x1).
const float kHighTexture = 0.035f;
const float kLowTexture = 0.020f;
// Used to reduce thresholds for larger/HD scenes: correction factor since
// higher correlation in HD scenes means lower spatial prediction error.
const float kScaleTexture = 0.9f;
// Percentage reduction in transitional bitrate for 2x2 selected over 1x2/2x1.
const float kRateRedSpatial2X2 = 0.6f;
const float kSpatialErr2x2VsHoriz = 0.1f; // percentage to favor 2x2 over H
const float kSpatialErr2X2VsVert = 0.1f; // percentage to favor 2x2 over V
const float kSpatialErrVertVsHoriz = 0.1f; // percentage to favor H over V
} // namespace webrtc
#endif // WEBRTC_MODULES_VIDEO_CODING_QM_SELECT_DATA_H_

1302
webrtc/modules/video_coding/qm_select_unittest.cc
View File

File diff suppressed because it is too large Load Diff

5
webrtc/modules/video_coding/video_coding.gypi
View File

@ -28,7 +28,6 @@
# headers # headers
'codec_database.h', 'codec_database.h',
'codec_timer.h', 'codec_timer.h',
'content_metrics_processing.h',
'decoding_state.h', 'decoding_state.h',
'encoded_frame.h', 'encoded_frame.h',
'fec_tables_xor.h', 'fec_tables_xor.h',
@ -49,8 +48,6 @@
'packet.h', 'packet.h',
'packet_buffer.h', 'packet_buffer.h',
'percentile_filter.h', 'percentile_filter.h',
'qm_select_data.h',
'qm_select.h',
'receiver.h', 'receiver.h',
'rtt_filter.h', 'rtt_filter.h',
'session_info.h', 'session_info.h',
@ -61,7 +58,6 @@
# sources # sources
'codec_database.cc', 'codec_database.cc',
'codec_timer.cc', 'codec_timer.cc',
'content_metrics_processing.cc',
'decoding_state.cc', 'decoding_state.cc',
'encoded_frame.cc', 'encoded_frame.cc',
'frame_buffer.cc', 'frame_buffer.cc',
@ -78,7 +74,6 @@
'packet.cc', 'packet.cc',
'packet_buffer.cc', 'packet_buffer.cc',
'percentile_filter.cc', 'percentile_filter.cc',
'qm_select.cc',
'receiver.cc', 'receiver.cc',
'rtt_filter.cc', 'rtt_filter.cc',
'session_info.cc', 'session_info.cc',

21
webrtc/modules/video_coding/video_coding_impl.cc
View File

@ -74,16 +74,11 @@ class VideoCodingModuleImpl : public VideoCodingModule {
VideoCodingModuleImpl(Clock* clock, VideoCodingModuleImpl(Clock* clock,
EventFactory* event_factory, EventFactory* event_factory,
VideoEncoderRateObserver* encoder_rate_observer, VideoEncoderRateObserver* encoder_rate_observer,
VCMQMSettingsCallback* qm_settings_callback,
NackSender* nack_sender, NackSender* nack_sender,
KeyFrameRequestSender* keyframe_request_sender, KeyFrameRequestSender* keyframe_request_sender,
EncodedImageCallback* pre_decode_image_callback) EncodedImageCallback* pre_decode_image_callback)
: VideoCodingModule(), : VideoCodingModule(),
sender_(clock, sender_(clock, &post_encode_callback_, encoder_rate_observer, nullptr),
&post_encode_callback_,
encoder_rate_observer,
qm_settings_callback,
nullptr),
receiver_(clock, receiver_(clock,
event_factory, event_factory,
pre_decode_image_callback, pre_decode_image_callback,
@ -147,9 +142,8 @@ class VideoCodingModuleImpl : public VideoCodingModule {
} }
int32_t AddVideoFrame(const VideoFrame& videoFrame, int32_t AddVideoFrame(const VideoFrame& videoFrame,
const VideoContentMetrics* contentMetrics,
const CodecSpecificInfo* codecSpecificInfo) override { const CodecSpecificInfo* codecSpecificInfo) override {
return sender_.AddVideoFrame(videoFrame, contentMetrics, codecSpecificInfo); return sender_.AddVideoFrame(videoFrame, codecSpecificInfo);
} }
int32_t IntraFrameRequest(size_t stream_index) override { int32_t IntraFrameRequest(size_t stream_index) override {
@ -298,9 +292,9 @@ VideoCodingModule* VideoCodingModule::Create(
NackSender* nack_sender, NackSender* nack_sender,
KeyFrameRequestSender* keyframe_request_sender, KeyFrameRequestSender* keyframe_request_sender,
EncodedImageCallback* pre_decode_image_callback) { EncodedImageCallback* pre_decode_image_callback) {
return new VideoCodingModuleImpl( return new VideoCodingModuleImpl(clock, nullptr, encoder_rate_observer,
clock, nullptr, encoder_rate_observer, qm_settings_callback, nack_sender, nack_sender, keyframe_request_sender,
keyframe_request_sender, pre_decode_image_callback); pre_decode_image_callback);
} }
// Create method for current interface, will be removed when the // Create method for current interface, will be removed when the
@ -320,9 +314,8 @@ VideoCodingModule* VideoCodingModule::Create(
KeyFrameRequestSender* keyframe_request_sender) { KeyFrameRequestSender* keyframe_request_sender) {
assert(clock); assert(clock);
assert(event_factory); assert(event_factory);
return new VideoCodingModuleImpl(clock, event_factory, nullptr, nullptr, return new VideoCodingModuleImpl(clock, event_factory, nullptr, nack_sender,
nack_sender, keyframe_request_sender, keyframe_request_sender, nullptr);
nullptr);
} }
} // namespace webrtc } // namespace webrtc

3
webrtc/modules/video_coding/video_coding_impl.h
View File

@ -59,7 +59,6 @@ class VideoSender : public Module {
VideoSender(Clock* clock, VideoSender(Clock* clock,
EncodedImageCallback* post_encode_callback, EncodedImageCallback* post_encode_callback,
VideoEncoderRateObserver* encoder_rate_observer, VideoEncoderRateObserver* encoder_rate_observer,
VCMQMSettingsCallback* qm_settings_callback,
VCMSendStatisticsCallback* send_stats_callback); VCMSendStatisticsCallback* send_stats_callback);
~VideoSender(); ~VideoSender();
@ -85,7 +84,6 @@ class VideoSender : public Module {
void SetVideoProtection(VCMVideoProtection videoProtection); void SetVideoProtection(VCMVideoProtection videoProtection);
int32_t AddVideoFrame(const VideoFrame& videoFrame, int32_t AddVideoFrame(const VideoFrame& videoFrame,
const VideoContentMetrics* _contentMetrics,
const CodecSpecificInfo* codecSpecificInfo); const CodecSpecificInfo* codecSpecificInfo);
int32_t IntraFrameRequest(size_t stream_index); int32_t IntraFrameRequest(size_t stream_index);
@ -116,7 +114,6 @@ class VideoSender : public Module {
VideoCodec current_codec_; VideoCodec current_codec_;
rtc::ThreadChecker main_thread_; rtc::ThreadChecker main_thread_;
VCMQMSettingsCallback* const qm_settings_callback_;
VCMProtectionCallback* protection_callback_; VCMProtectionCallback* protection_callback_;
rtc::CriticalSection params_crit_; rtc::CriticalSection params_crit_;

12
webrtc/modules/video_coding/video_sender.cc
View File

@ -27,7 +27,6 @@ namespace vcm {
VideoSender::VideoSender(Clock* clock, VideoSender::VideoSender(Clock* clock,
EncodedImageCallback* post_encode_callback, EncodedImageCallback* post_encode_callback,
VideoEncoderRateObserver* encoder_rate_observer, VideoEncoderRateObserver* encoder_rate_observer,
VCMQMSettingsCallback* qm_settings_callback,
VCMSendStatisticsCallback* send_stats_callback) VCMSendStatisticsCallback* send_stats_callback)
: clock_(clock), : clock_(clock),
_encoder(nullptr), _encoder(nullptr),
@ -38,16 +37,14 @@ VideoSender::VideoSender(Clock* clock,
frame_dropper_enabled_(true), frame_dropper_enabled_(true),
_sendStatsTimer(1000, clock_), _sendStatsTimer(1000, clock_),
current_codec_(), current_codec_(),
qm_settings_callback_(qm_settings_callback),
protection_callback_(nullptr), protection_callback_(nullptr),
encoder_params_({0, 0, 0, 0}), encoder_params_({0, 0, 0, 0}),
encoder_has_internal_source_(false), encoder_has_internal_source_(false),
next_frame_types_(1, kVideoFrameDelta) { next_frame_types_(1, kVideoFrameDelta) {
_mediaOpt.Reset();
// Allow VideoSender to be created on one thread but used on another, post // Allow VideoSender to be created on one thread but used on another, post
// construction. This is currently how this class is being used by at least // construction. This is currently how this class is being used by at least
// one external project (diffractor). // one external project (diffractor).
_mediaOpt.EnableQM(qm_settings_callback_ != nullptr);
_mediaOpt.Reset();
main_thread_.DetachFromThread(); main_thread_.DetachFromThread();
} }
@ -203,9 +200,8 @@ int VideoSender::FrameRate(unsigned int* framerate) const {
int32_t VideoSender::SetChannelParameters(uint32_t target_bitrate, int32_t VideoSender::SetChannelParameters(uint32_t target_bitrate,
uint8_t lossRate, uint8_t lossRate,
int64_t rtt) { int64_t rtt) {
uint32_t target_rate = uint32_t target_rate = _mediaOpt.SetTargetRates(target_bitrate, lossRate, rtt,
_mediaOpt.SetTargetRates(target_bitrate, lossRate, rtt, protection_callback_);
protection_callback_, qm_settings_callback_);
uint32_t input_frame_rate = _mediaOpt.InputFrameRate(); uint32_t input_frame_rate = _mediaOpt.InputFrameRate();
@ -274,7 +270,6 @@ void VideoSender::SetVideoProtection(VCMVideoProtection videoProtection) {
} }
// Add one raw video frame to the encoder, blocking. // Add one raw video frame to the encoder, blocking.
int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame, int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame,
const VideoContentMetrics* contentMetrics,
const CodecSpecificInfo* codecSpecificInfo) { const CodecSpecificInfo* codecSpecificInfo) {
EncoderParameters encoder_params; EncoderParameters encoder_params;
std::vector<FrameType> next_frame_types; std::vector<FrameType> next_frame_types;
@ -296,7 +291,6 @@ int32_t VideoSender::AddVideoFrame(const VideoFrame& videoFrame,
_encoder->OnDroppedFrame(); _encoder->OnDroppedFrame();
return VCM_OK; return VCM_OK;
} }
_mediaOpt.UpdateContentData(contentMetrics);
// TODO(pbos): Make sure setting send codec is synchronized with video // TODO(pbos): Make sure setting send codec is synchronized with video
// processing so frame size always matches. // processing so frame size always matches.
if (!_codecDataBase.MatchesCurrentResolution(videoFrame.width(), if (!_codecDataBase.MatchesCurrentResolution(videoFrame.width(),

6
webrtc/modules/video_coding/video_sender_unittest.cc
View File

@ -180,13 +180,13 @@ class TestVideoSender : public ::testing::Test {
TestVideoSender() : clock_(1000), encoded_frame_callback_(&clock_) {} TestVideoSender() : clock_(1000), encoded_frame_callback_(&clock_) {}
void SetUp() override { void SetUp() override {
sender_.reset(new VideoSender(&clock_, &encoded_frame_callback_, nullptr, sender_.reset(
nullptr, nullptr)); new VideoSender(&clock_, &encoded_frame_callback_, nullptr, nullptr));
} }
void AddFrame() { void AddFrame() {
assert(generator_.get()); assert(generator_.get());
sender_->AddVideoFrame(*generator_->NextFrame(), NULL, NULL); sender_->AddVideoFrame(*generator_->NextFrame(), NULL);
} }
SimulatedClock clock_; SimulatedClock clock_;

3
webrtc/modules/video_processing/BUILD.gn
View File

@ -13,8 +13,6 @@ build_video_processing_sse2 = current_cpu == "x86" || current_cpu == "x64"
source_set("video_processing") { source_set("video_processing") {
sources = [ sources = [
"content_analysis.cc",
"content_analysis.h",
"frame_preprocessor.cc", "frame_preprocessor.cc",
"frame_preprocessor.h", "frame_preprocessor.h",
"include/video_processing.h", "include/video_processing.h",
@ -63,7 +61,6 @@ source_set("video_processing") {
if (build_video_processing_sse2) { if (build_video_processing_sse2) {
source_set("video_processing_sse2") { source_set("video_processing_sse2") {
sources = [ sources = [
"content_analysis_sse2.cc",
"util/denoiser_filter_sse2.cc", "util/denoiser_filter_sse2.cc",
"util/denoiser_filter_sse2.h", "util/denoiser_filter_sse2.h",
] ]

280
webrtc/modules/video_processing/content_analysis.cc
View File

@ -1,280 +0,0 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/modules/video_processing/content_analysis.h"
#include <math.h>
#include <stdlib.h>
#include "webrtc/system_wrappers/include/cpu_features_wrapper.h"
namespace webrtc {
VPMContentAnalysis::VPMContentAnalysis(bool runtime_cpu_detection)
: orig_frame_(NULL),
prev_frame_(NULL),
width_(0),
height_(0),
skip_num_(1),
border_(8),
motion_magnitude_(0.0f),
spatial_pred_err_(0.0f),
spatial_pred_err_h_(0.0f),
spatial_pred_err_v_(0.0f),
first_frame_(true),
ca_Init_(false),
content_metrics_(NULL) {
ComputeSpatialMetrics = &VPMContentAnalysis::ComputeSpatialMetrics_C;
TemporalDiffMetric = &VPMContentAnalysis::TemporalDiffMetric_C;
if (runtime_cpu_detection) {
#if defined(WEBRTC_ARCH_X86_FAMILY)
if (WebRtc_GetCPUInfo(kSSE2)) {
ComputeSpatialMetrics = &VPMContentAnalysis::ComputeSpatialMetrics_SSE2;
TemporalDiffMetric = &VPMContentAnalysis::TemporalDiffMetric_SSE2;
}
#endif
}
Release();
}
VPMContentAnalysis::~VPMContentAnalysis() {
Release();
}
VideoContentMetrics* VPMContentAnalysis::ComputeContentMetrics(
const VideoFrame& inputFrame) {
if (inputFrame.IsZeroSize())
return NULL;
// Init if needed (native dimension change).
if (width_ != inputFrame.width() || height_ != inputFrame.height()) {
if (VPM_OK != Initialize(inputFrame.width(), inputFrame.height()))
return NULL;
}
// Only interested in the Y plane.
orig_frame_ = inputFrame.buffer(kYPlane);
// Compute spatial metrics: 3 spatial prediction errors.
(this->*ComputeSpatialMetrics)();
// Compute motion metrics
if (first_frame_ == false)
ComputeMotionMetrics();
// Saving current frame as previous one: Y only.
memcpy(prev_frame_, orig_frame_, width_ * height_);
first_frame_ = false;
ca_Init_ = true;
return ContentMetrics();
}
int32_t VPMContentAnalysis::Release() {
if (content_metrics_ != NULL) {
delete content_metrics_;
content_metrics_ = NULL;
}
if (prev_frame_ != NULL) {
delete[] prev_frame_;
prev_frame_ = NULL;
}
width_ = 0;
height_ = 0;
first_frame_ = true;
return VPM_OK;
}
int32_t VPMContentAnalysis::Initialize(int width, int height) {
width_ = width;
height_ = height;
first_frame_ = true;
// skip parameter: # of skipped rows: for complexity reduction
// temporal also currently uses it for column reduction.
skip_num_ = 1;
// use skipNum = 2 for 4CIF, WHD
if ((height_ >= 576) && (width_ >= 704)) {
skip_num_ = 2;
}
// use skipNum = 4 for FULLL_HD images
if ((height_ >= 1080) && (width_ >= 1920)) {
skip_num_ = 4;
}
if (content_metrics_ != NULL) {
delete content_metrics_;
}
if (prev_frame_ != NULL) {
delete[] prev_frame_;
}
// Spatial Metrics don't work on a border of 8. Minimum processing
// block size is 16 pixels. So make sure the width and height support this.
if (width_ <= 32 || height_ <= 32) {
ca_Init_ = false;
return VPM_PARAMETER_ERROR;
}
content_metrics_ = new VideoContentMetrics();
if (content_metrics_ == NULL) {
return VPM_MEMORY;
}
prev_frame_ = new uint8_t[width_ * height_]; // Y only.
if (prev_frame_ == NULL)
return VPM_MEMORY;
return VPM_OK;
}
// Compute motion metrics: magnitude over non-zero motion vectors,
// and size of zero cluster
int32_t VPMContentAnalysis::ComputeMotionMetrics() {
// Motion metrics: only one is derived from normalized
// (MAD) temporal difference
(this->*TemporalDiffMetric)();
return VPM_OK;
}
// Normalized temporal difference (MAD): used as a motion level metric
// Normalize MAD by spatial contrast: images with more contrast
// (pixel variance) likely have larger temporal difference
// To reduce complexity, we compute the metric for a reduced set of points.
int32_t VPMContentAnalysis::TemporalDiffMetric_C() {
// size of original frame
int sizei = height_;
int sizej = width_;
uint32_t tempDiffSum = 0;
uint32_t pixelSum = 0;
uint64_t pixelSqSum = 0;
uint32_t num_pixels = 0; // Counter for # of pixels.
const int width_end = ((width_ - 2 * border_) & -16) + border_;
for (int i = border_; i < sizei - border_; i += skip_num_) {
for (int j = border_; j < width_end; j++) {
num_pixels += 1;
int ssn = i * sizej + j;
uint8_t currPixel = orig_frame_[ssn];
uint8_t prevPixel = prev_frame_[ssn];
tempDiffSum +=
static_cast<uint32_t>(abs((int16_t)(currPixel - prevPixel)));
pixelSum += static_cast<uint32_t>(currPixel);
pixelSqSum += static_cast<uint64_t>(currPixel * currPixel);
}
}
// Default.
motion_magnitude_ = 0.0f;
if (tempDiffSum == 0)
return VPM_OK;
// Normalize over all pixels.
float const tempDiffAvg =
static_cast<float>(tempDiffSum) / static_cast<float>(num_pixels);
float const pixelSumAvg =
static_cast<float>(pixelSum) / static_cast<float>(num_pixels);
float const pixelSqSumAvg =
static_cast<float>(pixelSqSum) / static_cast<float>(num_pixels);
float contrast = pixelSqSumAvg - (pixelSumAvg * pixelSumAvg);
if (contrast > 0.0) {
contrast = sqrt(contrast);
motion_magnitude_ = tempDiffAvg / contrast;
}
return VPM_OK;
}
// Compute spatial metrics:
// To reduce complexity, we compute the metric for a reduced set of points.
// The spatial metrics are rough estimates of the prediction error cost for
// each QM spatial mode: 2x2,1x2,2x1
// The metrics are a simple estimate of the up-sampling prediction error,
// estimated assuming sub-sampling for decimation (no filtering),
// and up-sampling back up with simple bilinear interpolation.
int32_t VPMContentAnalysis::ComputeSpatialMetrics_C() {
const int sizei = height_;
const int sizej = width_;
// Pixel mean square average: used to normalize the spatial metrics.
uint32_t pixelMSA = 0;
uint32_t spatialErrSum = 0;
uint32_t spatialErrVSum = 0;
uint32_t spatialErrHSum = 0;
// make sure work section is a multiple of 16
const int width_end = ((sizej - 2 * border_) & -16) + border_;
for (int i = border_; i < sizei - border_; i += skip_num_) {
for (int j = border_; j < width_end; j++) {
int ssn1 = i * sizej + j;
int ssn2 = (i + 1) * sizej + j; // bottom
int ssn3 = (i - 1) * sizej + j; // top
int ssn4 = i * sizej + j + 1; // right
int ssn5 = i * sizej + j - 1; // left
uint16_t refPixel1 = orig_frame_[ssn1] << 1;
uint16_t refPixel2 = orig_frame_[ssn1] << 2;
uint8_t bottPixel = orig_frame_[ssn2];
uint8_t topPixel = orig_frame_[ssn3];
uint8_t rightPixel = orig_frame_[ssn4];
uint8_t leftPixel = orig_frame_[ssn5];
spatialErrSum += static_cast<uint32_t>(abs(static_cast<int16_t>(
refPixel2 - static_cast<uint16_t>(bottPixel + topPixel + leftPixel +
rightPixel))));
spatialErrVSum += static_cast<uint32_t>(abs(static_cast<int16_t>(
refPixel1 - static_cast<uint16_t>(bottPixel + topPixel))));
spatialErrHSum += static_cast<uint32_t>(abs(static_cast<int16_t>(
refPixel1 - static_cast<uint16_t>(leftPixel + rightPixel))));
pixelMSA += orig_frame_[ssn1];
}
}
// Normalize over all pixels.
const float spatialErr = static_cast<float>(spatialErrSum >> 2);
const float spatialErrH = static_cast<float>(spatialErrHSum >> 1);
const float spatialErrV = static_cast<float>(spatialErrVSum >> 1);
const float norm = static_cast<float>(pixelMSA);
// 2X2:
spatial_pred_err_ = spatialErr / norm;
// 1X2:
spatial_pred_err_h_ = spatialErrH / norm;
// 2X1:
spatial_pred_err_v_ = spatialErrV / norm;
return VPM_OK;
}
VideoContentMetrics* VPMContentAnalysis::ContentMetrics() {
if (ca_Init_ == false)
return NULL;
content_metrics_->spatial_pred_err = spatial_pred_err_;
content_metrics_->spatial_pred_err_h = spatial_pred_err_h_;
content_metrics_->spatial_pred_err_v = spatial_pred_err_v_;
// Motion metric: normalized temporal difference (MAD).
content_metrics_->motion_magnitude = motion_magnitude_;
return content_metrics_;
}
} // namespace webrtc

87
webrtc/modules/video_processing/content_analysis.h
View File

@ -1,87 +0,0 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_VIDEO_PROCESSING_CONTENT_ANALYSIS_H_
#define WEBRTC_MODULES_VIDEO_PROCESSING_CONTENT_ANALYSIS_H_
#include "webrtc/modules/include/module_common_types.h"
#include "webrtc/modules/video_processing/include/video_processing_defines.h"
#include "webrtc/typedefs.h"
#include "webrtc/video_frame.h"
namespace webrtc {
class VPMContentAnalysis {
public:
// When |runtime_cpu_detection| is true, runtime selection of an optimized
// code path is allowed.
explicit VPMContentAnalysis(bool runtime_cpu_detection);
~VPMContentAnalysis();
// Initialize ContentAnalysis - should be called prior to
// extractContentFeature
// Inputs: width, height
// Return value: 0 if OK, negative value upon error
int32_t Initialize(int width, int height);
// Extract content Feature - main function of ContentAnalysis
// Input: new frame
// Return value: pointer to structure containing content Analysis
// metrics or NULL value upon error
VideoContentMetrics* ComputeContentMetrics(const VideoFrame& inputFrame);
// Release all allocated memory
// Output: 0 if OK, negative value upon error
int32_t Release();
private:
// return motion metrics
VideoContentMetrics* ContentMetrics();
// Normalized temporal difference metric: for motion magnitude
typedef int32_t (VPMContentAnalysis::*TemporalDiffMetricFunc)();
TemporalDiffMetricFunc TemporalDiffMetric;
int32_t TemporalDiffMetric_C();
// Motion metric method: call 2 metrics (magnitude and size)
int32_t ComputeMotionMetrics();
// Spatial metric method: computes the 3 frame-average spatial
// prediction errors (1x2,2x1,2x2)
typedef int32_t (VPMContentAnalysis::*ComputeSpatialMetricsFunc)();
ComputeSpatialMetricsFunc ComputeSpatialMetrics;
int32_t ComputeSpatialMetrics_C();
#if defined(WEBRTC_ARCH_X86_FAMILY)
int32_t ComputeSpatialMetrics_SSE2();
int32_t TemporalDiffMetric_SSE2();
#endif
const uint8_t* orig_frame_;
uint8_t* prev_frame_;
int width_;
int height_;
int skip_num_;
int border_;
// Content Metrics: Stores the local average of the metrics.
float motion_magnitude_; // motion class
float spatial_pred_err_; // spatial class
float spatial_pred_err_h_; // spatial class
float spatial_pred_err_v_; // spatial class
bool first_frame_;
bool ca_Init_;
VideoContentMetrics* content_metrics_;
};
} // namespace webrtc
#endif // WEBRTC_MODULES_VIDEO_PROCESSING_CONTENT_ANALYSIS_H_

271
webrtc/modules/video_processing/content_analysis_sse2.cc
View File

@ -1,271 +0,0 @@
/*
* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "webrtc/modules/video_processing/content_analysis.h"
#include <emmintrin.h>
#include <math.h>
namespace webrtc {
int32_t VPMContentAnalysis::TemporalDiffMetric_SSE2() {
uint32_t num_pixels = 0; // counter for # of pixels
const uint8_t* imgBufO = orig_frame_ + border_ * width_ + border_;
const uint8_t* imgBufP = prev_frame_ + border_ * width_ + border_;
const int32_t width_end = ((width_ - 2 * border_) & -16) + border_;
__m128i sad_64 = _mm_setzero_si128();
__m128i sum_64 = _mm_setzero_si128();
__m128i sqsum_64 = _mm_setzero_si128();
const __m128i z = _mm_setzero_si128();
for (uint16_t i = 0; i < (height_ - 2 * border_); i += skip_num_) {
__m128i sqsum_32 = _mm_setzero_si128();
const uint8_t* lineO = imgBufO;
const uint8_t* lineP = imgBufP;
// Work on 16 pixels at a time. For HD content with a width of 1920
// this loop will run ~67 times (depending on border). Maximum for
// abs(o-p) and sum(o) will be 255. _mm_sad_epu8 produces 2 64 bit
// results which are then accumulated. There is no chance of
// rollover for these two accumulators.
// o*o will have a maximum of 255*255 = 65025. This will roll over
// a 16 bit accumulator as 67*65025 > 65535, but will fit in a
// 32 bit accumulator.
for (uint16_t j = 0; j < width_end - border_; j += 16) {
const __m128i o = _mm_loadu_si128((__m128i*)(lineO));
const __m128i p = _mm_loadu_si128((__m128i*)(lineP));
lineO += 16;
lineP += 16;
// Abs pixel difference between frames.
sad_64 = _mm_add_epi64(sad_64, _mm_sad_epu8(o, p));
// sum of all pixels in frame
sum_64 = _mm_add_epi64(sum_64, _mm_sad_epu8(o, z));
// Squared sum of all pixels in frame.
const __m128i olo = _mm_unpacklo_epi8(o, z);
const __m128i ohi = _mm_unpackhi_epi8(o, z);
const __m128i sqsum_32_lo = _mm_madd_epi16(olo, olo);
const __m128i sqsum_32_hi = _mm_madd_epi16(ohi, ohi);
sqsum_32 = _mm_add_epi32(sqsum_32, sqsum_32_lo);
sqsum_32 = _mm_add_epi32(sqsum_32, sqsum_32_hi);
}
// Add to 64 bit running sum as to not roll over.
sqsum_64 =
_mm_add_epi64(sqsum_64, _mm_add_epi64(_mm_unpackhi_epi32(sqsum_32, z),
_mm_unpacklo_epi32(sqsum_32, z)));
imgBufO += width_ * skip_num_;
imgBufP += width_ * skip_num_;
num_pixels += (width_end - border_);
}
__m128i sad_final_128;
__m128i sum_final_128;
__m128i sqsum_final_128;
// Bring sums out of vector registers and into integer register
// domain, summing them along the way.
_mm_store_si128(&sad_final_128, sad_64);
_mm_store_si128(&sum_final_128, sum_64);
_mm_store_si128(&sqsum_final_128, sqsum_64);
uint64_t* sad_final_64 = reinterpret_cast<uint64_t*>(&sad_final_128);
uint64_t* sum_final_64 = reinterpret_cast<uint64_t*>(&sum_final_128);
uint64_t* sqsum_final_64 = reinterpret_cast<uint64_t*>(&sqsum_final_128);
const uint32_t pixelSum = sum_final_64[0] + sum_final_64[1];
const uint64_t pixelSqSum = sqsum_final_64[0] + sqsum_final_64[1];
const uint32_t tempDiffSum = sad_final_64[0] + sad_final_64[1];
// Default.
motion_magnitude_ = 0.0f;
if (tempDiffSum == 0)
return VPM_OK;
// Normalize over all pixels.
const float tempDiffAvg =
static_cast<float>(tempDiffSum) / static_cast<float>(num_pixels);
const float pixelSumAvg =
static_cast<float>(pixelSum) / static_cast<float>(num_pixels);
const float pixelSqSumAvg =
static_cast<float>(pixelSqSum) / static_cast<float>(num_pixels);
float contrast = pixelSqSumAvg - (pixelSumAvg * pixelSumAvg);
if (contrast > 0.0) {
contrast = sqrt(contrast);
motion_magnitude_ = tempDiffAvg / contrast;
}
return VPM_OK;
}
int32_t VPMContentAnalysis::ComputeSpatialMetrics_SSE2() {
const uint8_t* imgBuf = orig_frame_ + border_ * width_;
const int32_t width_end = ((width_ - 2 * border_) & -16) + border_;
__m128i se_32 = _mm_setzero_si128();
__m128i sev_32 = _mm_setzero_si128();
__m128i seh_32 = _mm_setzero_si128();
__m128i msa_32 = _mm_setzero_si128();
const __m128i z = _mm_setzero_si128();
// Error is accumulated as a 32 bit value. Looking at HD content with a
// height of 1080 lines, or about 67 macro blocks. If the 16 bit row
// value is maxed out at 65529 for every row, 65529*1080 = 70777800, which
// will not roll over a 32 bit accumulator.
// skip_num_ is also used to reduce the number of rows
for (int32_t i = 0; i < (height_ - 2 * border_); i += skip_num_) {
__m128i se_16 = _mm_setzero_si128();
__m128i sev_16 = _mm_setzero_si128();
__m128i seh_16 = _mm_setzero_si128();
__m128i msa_16 = _mm_setzero_si128();
// Row error is accumulated as a 16 bit value. There are 8
// accumulators. Max value of a 16 bit number is 65529. Looking
// at HD content, 1080p, has a width of 1920, 120 macro blocks.
// A mb at a time is processed at a time. Absolute max error at
// a point would be abs(0-255+255+255+255) which equals 1020.
// 120*1020 = 122400. The probability of hitting this is quite low
// on well behaved content. A specially crafted image could roll over.
// border_ could also be adjusted to concentrate on just the center of
// the images for an HD capture in order to reduce the possiblity of
// rollover.
const uint8_t* lineTop = imgBuf - width_ + border_;
const uint8_t* lineCen = imgBuf + border_;
const uint8_t* lineBot = imgBuf + width_ + border_;
for (int32_t j = 0; j < width_end - border_; j += 16) {
const __m128i t = _mm_loadu_si128((__m128i*)(lineTop));
const __m128i l = _mm_loadu_si128((__m128i*)(lineCen - 1));
const __m128i c = _mm_loadu_si128((__m128i*)(lineCen));
const __m128i r = _mm_loadu_si128((__m128i*)(lineCen + 1));
const __m128i b = _mm_loadu_si128((__m128i*)(lineBot));
lineTop += 16;
lineCen += 16;
lineBot += 16;
// center pixel unpacked
__m128i clo = _mm_unpacklo_epi8(c, z);
__m128i chi = _mm_unpackhi_epi8(c, z);
// left right pixels unpacked and added together
const __m128i lrlo =
_mm_add_epi16(_mm_unpacklo_epi8(l, z), _mm_unpacklo_epi8(r, z));
const __m128i lrhi =
_mm_add_epi16(_mm_unpackhi_epi8(l, z), _mm_unpackhi_epi8(r, z));
// top & bottom pixels unpacked and added together
const __m128i tblo =
_mm_add_epi16(_mm_unpacklo_epi8(t, z), _mm_unpacklo_epi8(b, z));
const __m128i tbhi =
_mm_add_epi16(_mm_unpackhi_epi8(t, z), _mm_unpackhi_epi8(b, z));
// running sum of all pixels
msa_16 = _mm_add_epi16(msa_16, _mm_add_epi16(chi, clo));
clo = _mm_slli_epi16(clo, 1);
chi = _mm_slli_epi16(chi, 1);
const __m128i sevtlo = _mm_subs_epi16(clo, tblo);
const __m128i sevthi = _mm_subs_epi16(chi, tbhi);
const __m128i sehtlo = _mm_subs_epi16(clo, lrlo);
const __m128i sehthi = _mm_subs_epi16(chi, lrhi);
clo = _mm_slli_epi16(clo, 1);
chi = _mm_slli_epi16(chi, 1);
const __m128i setlo = _mm_subs_epi16(clo, _mm_add_epi16(lrlo, tblo));
const __m128i sethi = _mm_subs_epi16(chi, _mm_add_epi16(lrhi, tbhi));
// Add to 16 bit running sum
se_16 =
_mm_add_epi16(se_16, _mm_max_epi16(setlo, _mm_subs_epi16(z, setlo)));
se_16 =
_mm_add_epi16(se_16, _mm_max_epi16(sethi, _mm_subs_epi16(z, sethi)));
sev_16 = _mm_add_epi16(sev_16,
_mm_max_epi16(sevtlo, _mm_subs_epi16(z, sevtlo)));
sev_16 = _mm_add_epi16(sev_16,
_mm_max_epi16(sevthi, _mm_subs_epi16(z, sevthi)));
seh_16 = _mm_add_epi16(seh_16,
_mm_max_epi16(sehtlo, _mm_subs_epi16(z, sehtlo)));
seh_16 = _mm_add_epi16(seh_16,
_mm_max_epi16(sehthi, _mm_subs_epi16(z, sehthi)));
}
// Add to 32 bit running sum as to not roll over.
se_32 = _mm_add_epi32(se_32, _mm_add_epi32(_mm_unpackhi_epi16(se_16, z),
_mm_unpacklo_epi16(se_16, z)));
sev_32 =
_mm_add_epi32(sev_32, _mm_add_epi32(_mm_unpackhi_epi16(sev_16, z),
_mm_unpacklo_epi16(sev_16, z)));
seh_32 =
_mm_add_epi32(seh_32, _mm_add_epi32(_mm_unpackhi_epi16(seh_16, z),
_mm_unpacklo_epi16(seh_16, z)));
msa_32 =
_mm_add_epi32(msa_32, _mm_add_epi32(_mm_unpackhi_epi16(msa_16, z),
_mm_unpacklo_epi16(msa_16, z)));
imgBuf += width_ * skip_num_;
}
__m128i se_128;
__m128i sev_128;
__m128i seh_128;
__m128i msa_128;
// Bring sums out of vector registers and into integer register
// domain, summing them along the way.
_mm_store_si128(&se_128, _mm_add_epi64(_mm_unpackhi_epi32(se_32, z),
_mm_unpacklo_epi32(se_32, z)));
_mm_store_si128(&sev_128, _mm_add_epi64(_mm_unpackhi_epi32(sev_32, z),
_mm_unpacklo_epi32(sev_32, z)));
_mm_store_si128(&seh_128, _mm_add_epi64(_mm_unpackhi_epi32(seh_32, z),
_mm_unpacklo_epi32(seh_32, z)));
_mm_store_si128(&msa_128, _mm_add_epi64(_mm_unpackhi_epi32(msa_32, z),
_mm_unpacklo_epi32(msa_32, z)));
uint64_t* se_64 = reinterpret_cast<uint64_t*>(&se_128);
uint64_t* sev_64 = reinterpret_cast<uint64_t*>(&sev_128);
uint64_t* seh_64 = reinterpret_cast<uint64_t*>(&seh_128);
uint64_t* msa_64 = reinterpret_cast<uint64_t*>(&msa_128);
const uint32_t spatialErrSum = se_64[0] + se_64[1];
const uint32_t spatialErrVSum = sev_64[0] + sev_64[1];
const uint32_t spatialErrHSum = seh_64[0] + seh_64[1];
const uint32_t pixelMSA = msa_64[0] + msa_64[1];
// Normalize over all pixels.
const float spatialErr = static_cast<float>(spatialErrSum >> 2);
const float spatialErrH = static_cast<float>(spatialErrHSum >> 1);
const float spatialErrV = static_cast<float>(spatialErrVSum >> 1);
const float norm = static_cast<float>(pixelMSA);
// 2X2:
spatial_pred_err_ = spatialErr / norm;
// 1X2:
spatial_pred_err_h_ = spatialErrH / norm;
// 2X1:
spatial_pred_err_v_ = spatialErrV / norm;
return VPM_OK;
}
} // namespace webrtc

24
webrtc/modules/video_processing/frame_preprocessor.cc
View File

@ -15,12 +15,8 @@
namespace webrtc { namespace webrtc {
VPMFramePreprocessor::VPMFramePreprocessor() VPMFramePreprocessor::VPMFramePreprocessor()
: content_metrics_(nullptr), : resampled_frame_(), frame_cnt_(0) {
resampled_frame_(),
enable_ca_(false),
frame_cnt_(0) {
spatial_resampler_ = new VPMSimpleSpatialResampler(); spatial_resampler_ = new VPMSimpleSpatialResampler();
ca_ = new VPMContentAnalysis(true);
vd_ = new VPMVideoDecimator(); vd_ = new VPMVideoDecimator();
EnableDenoising(false); EnableDenoising(false);
denoised_frame_toggle_ = 0; denoised_frame_toggle_ = 0;
@ -28,17 +24,13 @@ VPMFramePreprocessor::VPMFramePreprocessor()
VPMFramePreprocessor::~VPMFramePreprocessor() { VPMFramePreprocessor::~VPMFramePreprocessor() {
Reset(); Reset();
delete ca_;
delete vd_; delete vd_;
delete spatial_resampler_; delete spatial_resampler_;
} }
void VPMFramePreprocessor::Reset() { void VPMFramePreprocessor::Reset() {
ca_->Release();
vd_->Reset(); vd_->Reset();
content_metrics_ = nullptr;
spatial_resampler_->Reset(); spatial_resampler_->Reset();
enable_ca_ = false;
frame_cnt_ = 0; frame_cnt_ = 0;
} }
@ -46,10 +38,6 @@ void VPMFramePreprocessor::EnableTemporalDecimation(bool enable) {
vd_->EnableTemporalDecimation(enable); vd_->EnableTemporalDecimation(enable);
} }
void VPMFramePreprocessor::EnableContentAnalysis(bool enable) {
enable_ca_ = enable;
}
void VPMFramePreprocessor::SetInputFrameResampleMode( void VPMFramePreprocessor::SetInputFrameResampleMode(
VideoFrameResampling resampling_mode) { VideoFrameResampling resampling_mode) {
spatial_resampler_->SetInputFrameResampleMode(resampling_mode); spatial_resampler_->SetInputFrameResampleMode(resampling_mode);
@ -131,18 +119,8 @@ const VideoFrame* VPMFramePreprocessor::PreprocessFrame(
current_frame = &resampled_frame_; current_frame = &resampled_frame_;
} }
// Perform content analysis on the frame to be encoded.
if (enable_ca_ && frame_cnt_ % kSkipFrameCA == 0) {
// Compute new metrics every |kSkipFramesCA| frames, starting with
// the first frame.
content_metrics_ = ca_->ComputeContentMetrics(*current_frame);
}
++frame_cnt_; ++frame_cnt_;
return current_frame; return current_frame;
} }
VideoContentMetrics* VPMFramePreprocessor::GetContentMetrics() const {
return content_metrics_;
}
} // namespace webrtc } // namespace webrtc

8
webrtc/modules/video_processing/frame_preprocessor.h
View File

@ -14,7 +14,6 @@
#include <memory> #include <memory>
#include "webrtc/modules/video_processing/include/video_processing.h" #include "webrtc/modules/video_processing/include/video_processing.h"
#include "webrtc/modules/video_processing/content_analysis.h"
#include "webrtc/modules/video_processing/spatial_resampler.h" #include "webrtc/modules/video_processing/spatial_resampler.h"
#include "webrtc/modules/video_processing/video_decimator.h" #include "webrtc/modules/video_processing/video_decimator.h"
#include "webrtc/typedefs.h" #include "webrtc/typedefs.h"
@ -38,9 +37,6 @@ class VPMFramePreprocessor {
void SetInputFrameResampleMode(VideoFrameResampling resampling_mode); void SetInputFrameResampleMode(VideoFrameResampling resampling_mode);
// Enable content analysis.
void EnableContentAnalysis(bool enable);
// Set target resolution: frame rate and dimension. // Set target resolution: frame rate and dimension.
int32_t SetTargetResolution(uint32_t width, int32_t SetTargetResolution(uint32_t width,
uint32_t height, uint32_t height,
@ -59,21 +55,17 @@ class VPMFramePreprocessor {
// Preprocess output: // Preprocess output:
void EnableDenoising(bool enable); void EnableDenoising(bool enable);
const VideoFrame* PreprocessFrame(const VideoFrame& frame); const VideoFrame* PreprocessFrame(const VideoFrame& frame);
VideoContentMetrics* GetContentMetrics() const;
private: private:
// The content does not change so much every frame, so to reduce complexity // The content does not change so much every frame, so to reduce complexity
// we can compute new content metrics every |kSkipFrameCA| frames. // we can compute new content metrics every |kSkipFrameCA| frames.
enum { kSkipFrameCA = 2 }; enum { kSkipFrameCA = 2 };
VideoContentMetrics* content_metrics_;
VideoFrame denoised_frame_[2]; VideoFrame denoised_frame_[2];
VideoFrame resampled_frame_; VideoFrame resampled_frame_;
VPMSpatialResampler* spatial_resampler_; VPMSpatialResampler* spatial_resampler_;
VPMContentAnalysis* ca_;
VPMVideoDecimator* vd_; VPMVideoDecimator* vd_;
std::unique_ptr<VideoDenoiser> denoiser_; std::unique_ptr<VideoDenoiser> denoiser_;
bool enable_ca_;
uint8_t denoised_frame_toggle_; uint8_t denoised_frame_toggle_;
uint32_t frame_cnt_; uint32_t frame_cnt_;
}; };

3
webrtc/modules/video_processing/include/video_processing.h
View File

@ -53,9 +53,6 @@ class VideoProcessing {
virtual void EnableDenoising(bool enable) = 0; virtual void EnableDenoising(bool enable) = 0;
virtual const VideoFrame* PreprocessFrame(const VideoFrame& frame) = 0; virtual const VideoFrame* PreprocessFrame(const VideoFrame& frame) = 0;
virtual VideoContentMetrics* GetContentMetrics() const = 0;
virtual void EnableContentAnalysis(bool enable) = 0;
}; };
} // namespace webrtc } // namespace webrtc

50
webrtc/modules/video_processing/test/content_metrics_test.cc
View File

@ -1,50 +0,0 @@
/*
* Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <memory>
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
#include "webrtc/modules/video_processing/include/video_processing.h"
#include "webrtc/modules/video_processing/content_analysis.h"
#include "webrtc/modules/video_processing/test/video_processing_unittest.h"
namespace webrtc {
#if defined(WEBRTC_IOS)
TEST_F(VideoProcessingTest, DISABLED_ContentAnalysis) {
#else
TEST_F(VideoProcessingTest, ContentAnalysis) {
#endif
VPMContentAnalysis ca__c(false);
VPMContentAnalysis ca__sse(true);
VideoContentMetrics* _cM_c;
VideoContentMetrics* _cM_SSE;
ca__c.Initialize(width_, height_);
ca__sse.Initialize(width_, height_);
std::unique_ptr<uint8_t[]> video_buffer(new uint8_t[frame_length_]);
while (fread(video_buffer.get(), 1, frame_length_, source_file_) ==
frame_length_) {
// Using ConvertToI420 to add stride to the image.
EXPECT_EQ(0, ConvertToI420(kI420, video_buffer.get(), 0, 0, width_, height_,
0, kVideoRotation_0, &video_frame_));
_cM_c = ca__c.ComputeContentMetrics(video_frame_);
_cM_SSE = ca__sse.ComputeContentMetrics(video_frame_);
ASSERT_EQ(_cM_c->spatial_pred_err, _cM_SSE->spatial_pred_err);
ASSERT_EQ(_cM_c->spatial_pred_err_v, _cM_SSE->spatial_pred_err_v);
ASSERT_EQ(_cM_c->spatial_pred_err_h, _cM_SSE->spatial_pred_err_h);
ASSERT_EQ(_cM_c->motion_magnitude, _cM_SSE->motion_magnitude);
}
ASSERT_NE(0, feof(source_file_)) << "Error reading source file";
}
} // namespace webrtc

2
webrtc/modules/video_processing/test/video_processing_unittest.cc
View File

@ -126,8 +126,6 @@ TEST_F(VideoProcessingTest, Resampler) {
rewind(source_file_); rewind(source_file_);
ASSERT_TRUE(source_file_ != NULL) << "Cannot read input file \n"; ASSERT_TRUE(source_file_ != NULL) << "Cannot read input file \n";
// CA not needed here
vp_->EnableContentAnalysis(false);
// no temporal decimation // no temporal decimation
vp_->EnableTemporalDecimation(false); vp_->EnableTemporalDecimation(false);

3
webrtc/modules/video_processing/video_processing.gypi
View File

@ -20,8 +20,6 @@
'sources': [ 'sources': [
'include/video_processing.h', 'include/video_processing.h',
'include/video_processing_defines.h', 'include/video_processing_defines.h',
'content_analysis.cc',
'content_analysis.h',
'frame_preprocessor.cc', 'frame_preprocessor.cc',
'frame_preprocessor.h', 'frame_preprocessor.h',
'spatial_resampler.cc', 'spatial_resampler.cc',
@ -58,7 +56,6 @@
'target_name': 'video_processing_sse2', 'target_name': 'video_processing_sse2',
'type': 'static_library', 'type': 'static_library',
'sources': [ 'sources': [
'content_analysis_sse2.cc',
'util/denoiser_filter_sse2.cc', 'util/denoiser_filter_sse2.cc',
'util/denoiser_filter_sse2.h', 'util/denoiser_filter_sse2.h',
], ],

10
webrtc/modules/video_processing/video_processing_impl.cc
View File

@ -69,14 +69,4 @@ const VideoFrame* VideoProcessingImpl::PreprocessFrame(
return frame_pre_processor_.PreprocessFrame(frame); return frame_pre_processor_.PreprocessFrame(frame);
} }
VideoContentMetrics* VideoProcessingImpl::GetContentMetrics() const {
rtc::CritScope mutex(&mutex_);
return frame_pre_processor_.GetContentMetrics();
}
void VideoProcessingImpl::EnableContentAnalysis(bool enable) {
rtc::CritScope mutex(&mutex_);
frame_pre_processor_.EnableContentAnalysis(enable);
}
} // namespace webrtc } // namespace webrtc

2
webrtc/modules/video_processing/video_processing_impl.h
View File

@ -26,7 +26,6 @@ class VideoProcessingImpl : public VideoProcessing {
// Implements VideoProcessing. // Implements VideoProcessing.
void EnableTemporalDecimation(bool enable) override; void EnableTemporalDecimation(bool enable) override;
void SetInputFrameResampleMode(VideoFrameResampling resampling_mode) override; void SetInputFrameResampleMode(VideoFrameResampling resampling_mode) override;
void EnableContentAnalysis(bool enable) override;
int32_t SetTargetResolution(uint32_t width, int32_t SetTargetResolution(uint32_t width,
uint32_t height, uint32_t height,
uint32_t frame_rate) override; uint32_t frame_rate) override;
@ -35,7 +34,6 @@ class VideoProcessingImpl : public VideoProcessing {
uint32_t GetDecimatedHeight() const override; uint32_t GetDecimatedHeight() const override;
void EnableDenoising(bool enable) override; void EnableDenoising(bool enable) override;
const VideoFrame* PreprocessFrame(const VideoFrame& frame) override; const VideoFrame* PreprocessFrame(const VideoFrame& frame) override;
VideoContentMetrics* GetContentMetrics() const override;
private: private:
rtc::CriticalSection mutex_; rtc::CriticalSection mutex_;

44
webrtc/video/vie_encoder.cc
View File

@ -30,33 +30,13 @@ namespace webrtc {
static const float kStopPaddingThresholdMs = 2000; static const float kStopPaddingThresholdMs = 2000;
class QMVideoSettingsCallback : public VCMQMSettingsCallback {
public:
explicit QMVideoSettingsCallback(VideoProcessing* vpm);
~QMVideoSettingsCallback();
// Update VPM with QM (quality modes: frame size & frame rate) settings.
int32_t SetVideoQMSettings(const uint32_t frame_rate,
const uint32_t width,
const uint32_t height);
private:
VideoProcessing* vp_;
};
ViEEncoder::ViEEncoder(uint32_t number_of_cores, ViEEncoder::ViEEncoder(uint32_t number_of_cores,
ProcessThread* module_process_thread, ProcessThread* module_process_thread,
SendStatisticsProxy* stats_proxy, SendStatisticsProxy* stats_proxy,
OveruseFrameDetector* overuse_detector) OveruseFrameDetector* overuse_detector)
: number_of_cores_(number_of_cores), : number_of_cores_(number_of_cores),
vp_(VideoProcessing::Create()), vp_(VideoProcessing::Create()),
qm_callback_(new QMVideoSettingsCallback(vp_.get())), video_sender_(Clock::GetRealTimeClock(), this, this, this),
video_sender_(Clock::GetRealTimeClock(),
this,
this,
qm_callback_.get(),
this),
stats_proxy_(stats_proxy), stats_proxy_(stats_proxy),
overuse_detector_(overuse_detector), overuse_detector_(overuse_detector),
time_of_last_frame_activity_ms_(0), time_of_last_frame_activity_ms_(0),
@ -74,9 +54,6 @@ ViEEncoder::ViEEncoder(uint32_t number_of_cores,
video_suspended_(false) { video_suspended_(false) {
module_process_thread_->RegisterModule(&video_sender_); module_process_thread_->RegisterModule(&video_sender_);
vp_->EnableTemporalDecimation(true); vp_->EnableTemporalDecimation(true);
// Enable/disable content analysis: off by default for now.
vp_->EnableContentAnalysis(false);
} }
vcm::VideoSender* ViEEncoder::video_sender() { vcm::VideoSender* ViEEncoder::video_sender() {
@ -291,11 +268,10 @@ void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame) {
has_received_rpsi_ = false; has_received_rpsi_ = false;
} }
video_sender_.AddVideoFrame(*frame_to_send, vp_->GetContentMetrics(), video_sender_.AddVideoFrame(*frame_to_send, &codec_specific_info);
&codec_specific_info);
return; return;
} }
video_sender_.AddVideoFrame(*frame_to_send, nullptr, nullptr); video_sender_.AddVideoFrame(*frame_to_send, nullptr);
} }
void ViEEncoder::SendKeyFrame() { void ViEEncoder::SendKeyFrame() {
@ -391,18 +367,4 @@ void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps,
stats_proxy_->OnSuspendChange(video_is_suspended); stats_proxy_->OnSuspendChange(video_is_suspended);
} }
QMVideoSettingsCallback::QMVideoSettingsCallback(VideoProcessing* vpm)
: vp_(vpm) {
}
QMVideoSettingsCallback::~QMVideoSettingsCallback() {
}
int32_t QMVideoSettingsCallback::SetVideoQMSettings(
const uint32_t frame_rate,
const uint32_t width,
const uint32_t height) {
return vp_->SetTargetResolution(width, height, frame_rate);
}
} // namespace webrtc } // namespace webrtc

2
webrtc/video/vie_encoder.h
View File

@ -34,7 +34,6 @@ class EncodedImageCallback;
class OveruseFrameDetector; class OveruseFrameDetector;
class PacedSender; class PacedSender;
class ProcessThread; class ProcessThread;
class QMVideoSettingsCallback;
class SendStatisticsProxy; class SendStatisticsProxy;
class ViEBitrateObserver; class ViEBitrateObserver;
class ViEEffectFilter; class ViEEffectFilter;
@ -124,7 +123,6 @@ class ViEEncoder : public VideoEncoderRateObserver,
const uint32_t number_of_cores_; const uint32_t number_of_cores_;
const std::unique_ptr<VideoProcessing> vp_; const std::unique_ptr<VideoProcessing> vp_;
const std::unique_ptr<QMVideoSettingsCallback> qm_callback_;
vcm::VideoSender video_sender_; vcm::VideoSender video_sender_;
rtc::CriticalSection data_cs_; rtc::CriticalSection data_cs_;