zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactoring MediaOptimization so it can easily be turned into a thread-safe class.

Browse Source 
BUG=2732
R=stefan@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/6149004

git-svn-id: http://webrtc.googlecode.com/svn/trunk@5322 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
andresp@webrtc.org
2013-12-19 10:59:48 +00:00
parent faada6e604
commit e682aa5077
5 changed files with 221 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

254
webrtc/modules/video_coding/main/source/media_optimization.cc
View File

@ -17,6 +17,61 @@
namespace webrtc { namespace webrtc {
namespace media_optimization { namespace media_optimization {
namespace {
void UpdateProtectionCallback(
VCMProtectionMethod* selected_method,
uint32_t* video_rate_bps,
uint32_t* nack_overhead_rate_bps,
uint32_t* fec_overhead_rate_bps,
VCMProtectionCallback* video_protection_callback) {
FecProtectionParams delta_fec_params;
FecProtectionParams key_fec_params;
// Get the FEC code rate for Key frames (set to 0 when NA).
key_fec_params.fec_rate = selected_method->RequiredProtectionFactorK();
// Get the FEC code rate for Delta frames (set to 0 when NA).
delta_fec_params.fec_rate = selected_method->RequiredProtectionFactorD();
// Get the FEC-UEP protection status for Key frames: UEP on/off.
key_fec_params.use_uep_protection = selected_method->RequiredUepProtectionK();
// Get the FEC-UEP protection status for Delta frames: UEP on/off.
delta_fec_params.use_uep_protection =
selected_method->RequiredUepProtectionD();
// The RTP module currently requires the same |max_fec_frames| for both
// key and delta frames.
delta_fec_params.max_fec_frames = selected_method->MaxFramesFec();
key_fec_params.max_fec_frames = selected_method->MaxFramesFec();
// Set the FEC packet mask type. |kFecMaskBursty| is more effective for
// consecutive losses and little/no packet re-ordering. As we currently
// do not have feedback data on the degree of correlated losses and packet
// re-ordering, we keep default setting to |kFecMaskRandom| for now.
delta_fec_params.fec_mask_type = kFecMaskRandom;
key_fec_params.fec_mask_type = kFecMaskRandom;
// TODO(Marco): Pass FEC protection values per layer.
video_protection_callback->ProtectionRequest(&delta_fec_params,
&key_fec_params,
video_rate_bps,
nack_overhead_rate_bps,
fec_overhead_rate_bps);
}
} // namespace
struct MediaOptimization::EncodedFrameSample {
EncodedFrameSample(int size_bytes,
uint32_t timestamp,
int64_t time_complete_ms)
: size_bytes(size_bytes),
timestamp(timestamp),
time_complete_ms(time_complete_ms) {}
uint32_t size_bytes;
uint32_t timestamp;
int64_t time_complete_ms;
};
MediaOptimization::MediaOptimization(int32_t id, Clock* clock) MediaOptimization::MediaOptimization(int32_t id, Clock* clock)
: id_(id), : id_(id),
@ -35,8 +90,6 @@ MediaOptimization::MediaOptimization(int32_t id, Clock* clock)
target_bit_rate_(0), target_bit_rate_(0),
incoming_frame_rate_(0), incoming_frame_rate_(0),
enable_qm_(false), enable_qm_(false),
video_protection_callback_(NULL),
video_qmsettings_callback_(NULL),
encoded_frame_samples_(), encoded_frame_samples_(),
avg_sent_bit_rate_bps_(0), avg_sent_bit_rate_bps_(0),
avg_sent_framerate_(0), avg_sent_framerate_(0),
@ -59,7 +112,8 @@ MediaOptimization::~MediaOptimization(void) {
loss_prot_logic_->Release(); loss_prot_logic_->Release();
} }
int32_t MediaOptimization::Reset() { void MediaOptimization::Reset() {
SetEncodingData(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0, max_payload_size_);
memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
incoming_frame_rate_ = 0.0; incoming_frame_rate_ = 0.0;
frame_dropper_->Reset(); frame_dropper_->Reset();
@ -81,12 +135,52 @@ int32_t MediaOptimization::Reset() {
encoded_frame_samples_.clear(); encoded_frame_samples_.clear();
avg_sent_bit_rate_bps_ = 0; avg_sent_bit_rate_bps_ = 0;
num_layers_ = 1; num_layers_ = 1;
return VCM_OK;
} }
uint32_t MediaOptimization::SetTargetRates(uint32_t target_bitrate, void MediaOptimization::SetEncodingData(VideoCodecType send_codec_type,
uint8_t fraction_lost, int32_t max_bit_rate,
uint32_t round_trip_time_ms) { uint32_t frame_rate,
uint32_t target_bitrate,
uint16_t width,
uint16_t height,
int num_layers,
int32_t mtu) {
// Everything codec specific should be reset here since this means the codec
// has changed. If native dimension values have changed, then either user
// initiated change, or QM initiated change. Will be able to determine only
// after the processing of the first frame.
last_change_time_ = clock_->TimeInMilliseconds();
content_->Reset();
content_->UpdateFrameRate(frame_rate);
max_bit_rate_ = max_bit_rate;
send_codec_type_ = send_codec_type;
target_bit_rate_ = target_bitrate;
float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
loss_prot_logic_->UpdateFrameRate(static_cast<float>(frame_rate));
loss_prot_logic_->UpdateFrameSize(width, height);
loss_prot_logic_->UpdateNumLayers(num_layers);
frame_dropper_->Reset();
frame_dropper_->SetRates(target_bitrate_kbps, static_cast<float>(frame_rate));
user_frame_rate_ = static_cast<float>(frame_rate);
codec_width_ = width;
codec_height_ = height;
num_layers_ = (num_layers <= 1) ? 1 : num_layers; // Can also be zero.
max_payload_size_ = mtu;
qm_resolution_->Initialize(target_bitrate_kbps,
user_frame_rate_,
codec_width_,
codec_height_,
num_layers_);
}
uint32_t MediaOptimization::SetTargetRates(
uint32_t target_bitrate,
uint8_t fraction_lost,
uint32_t round_trip_time_ms,
VCMProtectionCallback* protection_callback,
VCMQMSettingsCallback* qmsettings_callback) {
// TODO(holmer): Consider putting this threshold only on the video bitrate, // TODO(holmer): Consider putting this threshold only on the video bitrate,
// and not on protection. // and not on protection.
if (max_bit_rate_ > 0 && if (max_bit_rate_ > 0 &&
@ -145,10 +239,13 @@ uint32_t MediaOptimization::SetTargetRates(uint32_t target_bitrate,
// Get the bit cost of protection method, based on the amount of // Get the bit cost of protection method, based on the amount of
// overhead data actually transmitted (including headers) the last // overhead data actually transmitted (including headers) the last
// second. // second.
UpdateProtectionCallback(selected_method, if (protection_callback) {
&sent_video_rate_bps, UpdateProtectionCallback(selected_method,
&sent_nack_rate_bps, &sent_video_rate_bps,
&sent_fec_rate_bps); &sent_nack_rate_bps,
&sent_fec_rate_bps,
protection_callback);
}
uint32_t sent_total_rate_bps = uint32_t sent_total_rate_bps =
sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps; sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps;
// Estimate the overhead costs of the next second as staying the same // Estimate the overhead costs of the next second as staying the same
@ -178,7 +275,7 @@ uint32_t MediaOptimization::SetTargetRates(uint32_t target_bitrate,
static_cast<float>(target_bit_rate_) / 1000.0f; static_cast<float>(target_bit_rate_) / 1000.0f;
frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_); frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_);
if (enable_qm_) { if (enable_qm_ && qmsettings_callback) {
// Update QM with rates. // Update QM with rates.
qm_resolution_->UpdateRates(target_video_bitrate_kbps, qm_resolution_->UpdateRates(target_video_bitrate_kbps,
sent_video_rate_kbps, sent_video_rate_kbps,
@ -187,7 +284,7 @@ uint32_t MediaOptimization::SetTargetRates(uint32_t target_bitrate,
// Check for QM selection. // Check for QM selection.
bool select_qm = CheckStatusForQMchange(); bool select_qm = CheckStatusForQMchange();
if (select_qm) { if (select_qm) {
SelectQuality(); SelectQuality(qmsettings_callback);
} }
// Reset the short-term averaged content data. // Reset the short-term averaged content data.
content_->ResetShortTermAvgData(); content_->ResetShortTermAvgData();
@ -198,44 +295,6 @@ uint32_t MediaOptimization::SetTargetRates(uint32_t target_bitrate,
return target_bit_rate_; return target_bit_rate_;
} }
int32_t MediaOptimization::SetEncodingData(VideoCodecType send_codec_type,
int32_t max_bit_rate,
uint32_t frame_rate,
uint32_t target_bitrate,
uint16_t width,
uint16_t height,
int num_layers) {
// Everything codec specific should be reset here since this means the codec
// has changed. If native dimension values have changed, then either user
// initiated change, or QM initiated change. Will be able to determine only
// after the processing of the first frame.
last_change_time_ = clock_->TimeInMilliseconds();
content_->Reset();
content_->UpdateFrameRate(frame_rate);
max_bit_rate_ = max_bit_rate;
send_codec_type_ = send_codec_type;
target_bit_rate_ = target_bitrate;
float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
loss_prot_logic_->UpdateFrameRate(static_cast<float>(frame_rate));
loss_prot_logic_->UpdateFrameSize(width, height);
loss_prot_logic_->UpdateNumLayers(num_layers);
frame_dropper_->Reset();
frame_dropper_->SetRates(target_bitrate_kbps, static_cast<float>(frame_rate));
user_frame_rate_ = static_cast<float>(frame_rate);
codec_width_ = width;
codec_height_ = height;
num_layers_ = (num_layers <= 1) ? 1 : num_layers; // Can also be zero.
int32_t ret = VCM_OK;
ret = qm_resolution_->Initialize(target_bitrate_kbps,
user_frame_rate_,
codec_width_,
codec_height_,
num_layers_);
return ret;
}
void MediaOptimization::EnableProtectionMethod(bool enable, void MediaOptimization::EnableProtectionMethod(bool enable,
VCMProtectionMethodEnum method) { VCMProtectionMethodEnum method) {
bool updated = false; bool updated = false;
@ -249,11 +308,6 @@ void MediaOptimization::EnableProtectionMethod(bool enable,
} }
} }
bool MediaOptimization::IsProtectionMethodEnabled(
VCMProtectionMethodEnum method) {
return (loss_prot_logic_->SelectedType() == method);
}
uint32_t MediaOptimization::InputFrameRate() { uint32_t MediaOptimization::InputFrameRate() {
ProcessIncomingFrameRate(clock_->TimeInMilliseconds()); ProcessIncomingFrameRate(clock_->TimeInMilliseconds());
return uint32_t(incoming_frame_rate_ + 0.5f); return uint32_t(incoming_frame_rate_ + 0.5f);
@ -272,6 +326,13 @@ uint32_t MediaOptimization::SentBitRate() {
return avg_sent_bit_rate_bps_; return avg_sent_bit_rate_bps_;
} }
VCMFrameCount MediaOptimization::SentFrameCount() {
VCMFrameCount count;
count.numDeltaFrames = delta_frame_cnt_;
count.numKeyFrames = key_frame_cnt_;
return count;
}
int32_t MediaOptimization::UpdateWithEncodedData(int encoded_length, int32_t MediaOptimization::UpdateWithEncodedData(int encoded_length,
uint32_t timestamp, uint32_t timestamp,
FrameType encoded_frame_type) { FrameType encoded_frame_type) {
@ -325,29 +386,14 @@ int32_t MediaOptimization::UpdateWithEncodedData(int encoded_length,
return VCM_OK; return VCM_OK;
} }
int32_t MediaOptimization::RegisterProtectionCallback( void MediaOptimization::EnableQM(bool enable) { enable_qm_ = enable; }
VCMProtectionCallback* protection_callback) {
video_protection_callback_ = protection_callback;
return VCM_OK;
}
int32_t MediaOptimization::RegisterVideoQMCallback(
VCMQMSettingsCallback* video_qmsettings) {
video_qmsettings_callback_ = video_qmsettings;
// Callback setting controls QM.
if (video_qmsettings_callback_ != NULL) {
enable_qm_ = true;
} else {
enable_qm_ = false;
}
return VCM_OK;
}
void MediaOptimization::EnableFrameDropper(bool enable) { void MediaOptimization::EnableFrameDropper(bool enable) {
frame_dropper_->Enable(enable); frame_dropper_->Enable(enable);
} }
bool MediaOptimization::DropFrame() { bool MediaOptimization::DropFrame() {
UpdateIncomingFrameRate();
// Leak appropriate number of bytes. // Leak appropriate number of bytes.
frame_dropper_->Leak((uint32_t)(InputFrameRate() + 0.5f)); frame_dropper_->Leak((uint32_t)(InputFrameRate() + 0.5f));
if (video_suspended_) { if (video_suspended_) {
@ -356,12 +402,6 @@ bool MediaOptimization::DropFrame() {
return frame_dropper_->DropFrame(); return frame_dropper_->DropFrame();
} }
int32_t MediaOptimization::SentFrameCount(VCMFrameCount* frame_count) const {
frame_count->numDeltaFrames = delta_frame_cnt_;
frame_count->numKeyFrames = key_frame_cnt_;
return VCM_OK;
}
void MediaOptimization::UpdateIncomingFrameRate() { void MediaOptimization::UpdateIncomingFrameRate() {
int64_t now = clock_->TimeInMilliseconds(); int64_t now = clock_->TimeInMilliseconds();
if (incoming_frame_times_[0] == 0) { if (incoming_frame_times_[0] == 0) {
@ -388,7 +428,8 @@ void MediaOptimization::UpdateContentData(
} }
} }
int32_t MediaOptimization::SelectQuality() { int32_t MediaOptimization::SelectQuality(
VCMQMSettingsCallback* video_qmsettings_callback) {
// Reset quantities for QM select. // Reset quantities for QM select.
qm_resolution_->ResetQM(); qm_resolution_->ResetQM();
@ -403,7 +444,7 @@ int32_t MediaOptimization::SelectQuality() {
} }
// Check for updates to spatial/temporal modes. // Check for updates to spatial/temporal modes.
QMUpdate(qm); QMUpdate(qm, video_qmsettings_callback);
// Reset all the rate and related frame counters quantities. // Reset all the rate and related frame counters quantities.
qm_resolution_->ResetRates(); qm_resolution_->ResetRates();
@ -426,50 +467,7 @@ void MediaOptimization::SuspendBelowMinBitrate(int threshold_bps,
video_suspended_ = false; video_suspended_ = false;
} }
// Private methods below this line. bool MediaOptimization::IsVideoSuspended() const { return video_suspended_; }
int MediaOptimization::UpdateProtectionCallback(
VCMProtectionMethod* selected_method,
uint32_t* video_rate_bps,
uint32_t* nack_overhead_rate_bps,
uint32_t* fec_overhead_rate_bps) {
if (!video_protection_callback_) {
return VCM_OK;
}
FecProtectionParams delta_fec_params;
FecProtectionParams key_fec_params;
// Get the FEC code rate for Key frames (set to 0 when NA).
key_fec_params.fec_rate = selected_method->RequiredProtectionFactorK();
// Get the FEC code rate for Delta frames (set to 0 when NA).
delta_fec_params.fec_rate = selected_method->RequiredProtectionFactorD();
// Get the FEC-UEP protection status for Key frames: UEP on/off.
key_fec_params.use_uep_protection = selected_method->RequiredUepProtectionK();
// Get the FEC-UEP protection status for Delta frames: UEP on/off.
delta_fec_params.use_uep_protection =
selected_method->RequiredUepProtectionD();
// The RTP module currently requires the same |max_fec_frames| for both
// key and delta frames.
delta_fec_params.max_fec_frames = selected_method->MaxFramesFec();
key_fec_params.max_fec_frames = selected_method->MaxFramesFec();
// Set the FEC packet mask type. |kFecMaskBursty| is more effective for
// consecutive losses and little/no packet re-ordering. As we currently
// do not have feedback data on the degree of correlated losses and packet
// re-ordering, we keep default setting to |kFecMaskRandom| for now.
delta_fec_params.fec_mask_type = kFecMaskRandom;
key_fec_params.fec_mask_type = kFecMaskRandom;
// TODO(Marco): Pass FEC protection values per layer.
return video_protection_callback_->ProtectionRequest(&delta_fec_params,
&key_fec_params,
video_rate_bps,
nack_overhead_rate_bps,
fec_overhead_rate_bps);
}
void MediaOptimization::PurgeOldFrameSamples(int64_t now_ms) { void MediaOptimization::PurgeOldFrameSamples(int64_t now_ms) {
while (!encoded_frame_samples_.empty()) { while (!encoded_frame_samples_.empty()) {
@ -518,7 +516,9 @@ void MediaOptimization::UpdateSentFramerate() {
} }
} }
bool MediaOptimization::QMUpdate(VCMResolutionScale* qm) { bool MediaOptimization::QMUpdate(
VCMResolutionScale* qm,
VCMQMSettingsCallback* video_qmsettings_callback) {
// Check for no change. // Check for no change.
if (!qm->change_resolution_spatial && !qm->change_resolution_temporal) { if (!qm->change_resolution_spatial && !qm->change_resolution_temporal) {
return false; return false;
@ -551,7 +551,7 @@ bool MediaOptimization::QMUpdate(VCMResolutionScale* qm) {
// will vary/fluctuate, and since we don't want to change the state of the // will vary/fluctuate, and since we don't want to change the state of the
// VPM frame dropper, unless a temporal action was selected, we use the // VPM frame dropper, unless a temporal action was selected, we use the
// quantity |qm->frame_rate| for updating. // quantity |qm->frame_rate| for updating.
video_qmsettings_callback_->SetVideoQMSettings( video_qmsettings_callback->SetVideoQMSettings(
qm->frame_rate, codec_width_, codec_height_); qm->frame_rate, codec_width_, codec_height_);
content_->UpdateFrameRate(qm->frame_rate); content_->UpdateFrameRate(qm->frame_rate);
qm_resolution_->UpdateCodecParameters( qm_resolution_->UpdateCodecParameters(

132
webrtc/modules/video_coding/main/source/media_optimization.h
View File

@ -29,33 +29,25 @@ class VCMContentMetricsProcessing;
namespace media_optimization { namespace media_optimization {
enum { // TODO(andresp): Make thread safe.
kBitrateMaxFrameSamples = 60
};
enum {
kBitrateAverageWinMs = 1000
};
struct EncodedFrameSample {
EncodedFrameSample(int size_bytes,
uint32_t timestamp,
int64_t time_complete_ms)
: size_bytes(size_bytes),
timestamp(timestamp),
time_complete_ms(time_complete_ms) {}
uint32_t size_bytes;
uint32_t timestamp;
int64_t time_complete_ms;
};
class MediaOptimization { class MediaOptimization {
public: public:
MediaOptimization(int32_t id, Clock* clock); MediaOptimization(int32_t id, Clock* clock);
~MediaOptimization(void); ~MediaOptimization();
// Resets the Media Optimization module. // TODO(andresp): Can Reset and SetEncodingData be done at construction time
int32_t Reset(); // only?
void Reset();
// Informs media optimization of initial encoding state.
void SetEncodingData(VideoCodecType send_codec_type,
int32_t max_bit_rate,
uint32_t frame_rate,
uint32_t bit_rate,
uint16_t width,
uint16_t height,
int num_temporal_layers,
int32_t mtu);
// Sets target rates for the encoder given the channel parameters. // Sets target rates for the encoder given the channel parameters.
// Inputs: target bitrate - the encoder target bitrate in bits/s. // Inputs: target bitrate - the encoder target bitrate in bits/s.
@ -63,95 +55,64 @@ class MediaOptimization {
// round_trip_time_ms - round trip time in milliseconds. // round_trip_time_ms - round trip time in milliseconds.
// min_bit_rate - the bit rate of the end-point with lowest rate. // min_bit_rate - the bit rate of the end-point with lowest rate.
// max_bit_rate - the bit rate of the end-point with highest rate. // max_bit_rate - the bit rate of the end-point with highest rate.
// TODO(andresp): Find if the callbacks can be triggered only after releasing
// an internal critical section.
uint32_t SetTargetRates(uint32_t target_bitrate, uint32_t SetTargetRates(uint32_t target_bitrate,
uint8_t fraction_lost, uint8_t fraction_lost,
uint32_t round_trip_time_ms); uint32_t round_trip_time_ms,
VCMProtectionCallback* protection_callback,
VCMQMSettingsCallback* qmsettings_callback);
// Informs media optimization of initial encoding state.
int32_t SetEncodingData(VideoCodecType send_codec_type,
int32_t max_bit_rate,
uint32_t frame_rate,
uint32_t bit_rate,
uint16_t width,
uint16_t height,
int num_temporal_layers);
// Enables protection method.
void EnableProtectionMethod(bool enable, VCMProtectionMethodEnum method); void EnableProtectionMethod(bool enable, VCMProtectionMethodEnum method);
void EnableQM(bool enable);
void EnableFrameDropper(bool enable);
// Returns weather or not protection method is enabled. // Lets the sender suspend video when the rate drops below
bool IsProtectionMethodEnabled(VCMProtectionMethodEnum method); // |threshold_bps|, and turns back on when the rate goes back up above
// |threshold_bps| + |window_bps|.
void SuspendBelowMinBitrate(int threshold_bps, int window_bps);
bool IsVideoSuspended() const;
// Returns the actual input frame rate. bool DropFrame();
uint32_t InputFrameRate();
// Returns the actual sent frame rate. void UpdateContentData(const VideoContentMetrics* content_metrics);
uint32_t SentFrameRate();
// Returns the actual sent bit rate.
uint32_t SentBitRate();
// Informs Media Optimization of encoding output: Length and frame type. // Informs Media Optimization of encoding output: Length and frame type.
int32_t UpdateWithEncodedData(int encoded_length, int32_t UpdateWithEncodedData(int encoded_length,
uint32_t timestamp, uint32_t timestamp,
FrameType encoded_frame_type); FrameType encoded_frame_type);
// Registers a protection callback to be used to inform the user about the uint32_t InputFrameRate();
// protection methods used. uint32_t SentFrameRate();
int32_t RegisterProtectionCallback( uint32_t SentBitRate();
VCMProtectionCallback* protection_callback); VCMFrameCount SentFrameCount();
// Registers a quality settings callback to be used to inform VPM/user.
int32_t RegisterVideoQMCallback(VCMQMSettingsCallback* video_qmsettings);
void EnableFrameDropper(bool enable);
bool DropFrame();
// Returns the number of key/delta frames encoded.
int32_t SentFrameCount(VCMFrameCount* frame_count) const;
// Updates incoming frame rate value.
void UpdateIncomingFrameRate();
// Update content metric data.
void UpdateContentData(const VideoContentMetrics* content_metrics);
// Computes new Quality Mode.
int32_t SelectQuality();
// Lets the sender suspend video when the rate drops below
// |threshold_bps|, and turns back on when the rate goes back up above
// |threshold_bps| + |window_bps|.
void SuspendBelowMinBitrate(int threshold_bps, int window_bps);
// Accessors and mutators.
int32_t max_bit_rate() const { return max_bit_rate_; }
void set_max_payload_size(int32_t mtu) { max_payload_size_ = mtu; }
bool video_suspended() const { return video_suspended_; }
private: private:
typedef std::list<EncodedFrameSample> FrameSampleList;
enum { enum {
kFrameCountHistorySize = 90 kFrameCountHistorySize = 90
}; };
enum { enum {
kFrameHistoryWinMs = 2000 kFrameHistoryWinMs = 2000
}; };
enum {
kBitrateAverageWinMs = 1000
};
// Updates protection callback with protection settings. struct EncodedFrameSample;
int UpdateProtectionCallback(VCMProtectionMethod* selected_method, typedef std::list<EncodedFrameSample> FrameSampleList;
uint32_t* total_video_rate_bps,
uint32_t* nack_overhead_rate_bps,
uint32_t* fec_overhead_rate_bps);
void UpdateIncomingFrameRate();
void PurgeOldFrameSamples(int64_t now_ms); void PurgeOldFrameSamples(int64_t now_ms);
void UpdateSentBitrate(int64_t now_ms); void UpdateSentBitrate(int64_t now_ms);
void UpdateSentFramerate(); void UpdateSentFramerate();
// Computes new Quality Mode.
int32_t SelectQuality(VCMQMSettingsCallback* qmsettings_callback);
// Verifies if QM settings differ from default, i.e. if an update is required. // Verifies if QM settings differ from default, i.e. if an update is required.
// Computes actual values, as will be sent to the encoder. // Computes actual values, as will be sent to the encoder.
bool QMUpdate(VCMResolutionScale* qm); bool QMUpdate(VCMResolutionScale* qm,
VCMQMSettingsCallback* qmsettings_callback);
// Checks if we should make a QM change. Return true if yes, false otherwise. // Checks if we should make a QM change. Return true if yes, false otherwise.
bool CheckStatusForQMchange(); bool CheckStatusForQMchange();
@ -180,8 +141,6 @@ class MediaOptimization {
float incoming_frame_rate_; float incoming_frame_rate_;
int64_t incoming_frame_times_[kFrameCountHistorySize]; int64_t incoming_frame_times_[kFrameCountHistorySize];
bool enable_qm_; bool enable_qm_;
VCMProtectionCallback* video_protection_callback_;
VCMQMSettingsCallback* video_qmsettings_callback_;
std::list<EncodedFrameSample> encoded_frame_samples_; std::list<EncodedFrameSample> encoded_frame_samples_;
uint32_t avg_sent_bit_rate_bps_; uint32_t avg_sent_bit_rate_bps_;
uint32_t avg_sent_framerate_; uint32_t avg_sent_framerate_;
@ -196,8 +155,7 @@ class MediaOptimization {
bool video_suspended_; bool video_suspended_;
int suspension_threshold_bps_; int suspension_threshold_bps_;
int suspension_window_bps_; int suspension_window_bps_;
}; // End of MediaOptimization class declaration. };
} // namespace media_optimization } // namespace media_optimization
} // namespace webrtc } // namespace webrtc

27
webrtc/modules/video_coding/main/source/media_optimization_unittest.cc
View File

@ -35,7 +35,6 @@ class TestMediaOptimization : public ::testing::Test {
// This method mimics what happens in VideoSender::AddVideoFrame. // This method mimics what happens in VideoSender::AddVideoFrame.
void AddFrameAndAdvanceTime(int bitrate_bps, bool expect_frame_drop) { void AddFrameAndAdvanceTime(int bitrate_bps, bool expect_frame_drop) {
ASSERT_GE(bitrate_bps, 0); ASSERT_GE(bitrate_bps, 0);
media_opt_.UpdateIncomingFrameRate();
bool frame_dropped = media_opt_.DropFrame(); bool frame_dropped = media_opt_.DropFrame();
EXPECT_EQ(expect_frame_drop, frame_dropped); EXPECT_EQ(expect_frame_drop, frame_dropped);
if (!frame_dropped) { if (!frame_dropped) {
@ -63,12 +62,14 @@ TEST_F(TestMediaOptimization, VerifyMuting) {
media_opt_.SuspendBelowMinBitrate(kThresholdBps, kWindowBps); media_opt_.SuspendBelowMinBitrate(kThresholdBps, kWindowBps);
// The video should not be suspended from the start. // The video should not be suspended from the start.
EXPECT_FALSE(media_opt_.video_suspended()); EXPECT_FALSE(media_opt_.IsVideoSuspended());
int target_bitrate_kbps = 100; int target_bitrate_kbps = 100;
media_opt_.SetTargetRates(target_bitrate_kbps * 1000, media_opt_.SetTargetRates(target_bitrate_kbps * 1000,
0, // Lossrate. 0, // Lossrate.
100); // RTT in ms. 100,
NULL,
NULL); // RTT in ms.
media_opt_.EnableFrameDropper(true); media_opt_.EnableFrameDropper(true);
for (int time = 0; time < 2000; time += frame_time_ms_) { for (int time = 0; time < 2000; time += frame_time_ms_) {
ASSERT_NO_FATAL_FAILURE(AddFrameAndAdvanceTime(target_bitrate_kbps, false)); ASSERT_NO_FATAL_FAILURE(AddFrameAndAdvanceTime(target_bitrate_kbps, false));
@ -77,11 +78,13 @@ TEST_F(TestMediaOptimization, VerifyMuting) {
// Set the target rate below the limit for muting. // Set the target rate below the limit for muting.
media_opt_.SetTargetRates(kThresholdBps - 1000, media_opt_.SetTargetRates(kThresholdBps - 1000,
0, // Lossrate. 0, // Lossrate.
100); // RTT in ms. 100,
NULL,
NULL); // RTT in ms.
// Expect the muter to engage immediately and stay muted. // Expect the muter to engage immediately and stay muted.
// Test during 2 seconds. // Test during 2 seconds.
for (int time = 0; time < 2000; time += frame_time_ms_) { for (int time = 0; time < 2000; time += frame_time_ms_) {
EXPECT_TRUE(media_opt_.video_suspended()); EXPECT_TRUE(media_opt_.IsVideoSuspended());
ASSERT_NO_FATAL_FAILURE(AddFrameAndAdvanceTime(target_bitrate_kbps, true)); ASSERT_NO_FATAL_FAILURE(AddFrameAndAdvanceTime(target_bitrate_kbps, true));
} }
@ -89,22 +92,26 @@ TEST_F(TestMediaOptimization, VerifyMuting) {
// limit + window. // limit + window.
media_opt_.SetTargetRates(kThresholdBps + 1000, media_opt_.SetTargetRates(kThresholdBps + 1000,
0, // Lossrate. 0, // Lossrate.
100); // RTT in ms. 100,
// Expect the muter to stay muted. NULL,
NULL); // RTT in ms.
// Expect the muter to stay muted.
// Test during 2 seconds. // Test during 2 seconds.
for (int time = 0; time < 2000; time += frame_time_ms_) { for (int time = 0; time < 2000; time += frame_time_ms_) {
EXPECT_TRUE(media_opt_.video_suspended()); EXPECT_TRUE(media_opt_.IsVideoSuspended());
ASSERT_NO_FATAL_FAILURE(AddFrameAndAdvanceTime(target_bitrate_kbps, true)); ASSERT_NO_FATAL_FAILURE(AddFrameAndAdvanceTime(target_bitrate_kbps, true));
} }
// Set the target above limit + window. // Set the target above limit + window.
media_opt_.SetTargetRates(kThresholdBps + kWindowBps + 1000, media_opt_.SetTargetRates(kThresholdBps + kWindowBps + 1000,
0, // Lossrate. 0, // Lossrate.
100); // RTT in ms. 100,
NULL,
NULL); // RTT in ms.
// Expect the muter to disengage immediately. // Expect the muter to disengage immediately.
// Test during 2 seconds. // Test during 2 seconds.
for (int time = 0; time < 2000; time += frame_time_ms_) { for (int time = 0; time < 2000; time += frame_time_ms_) {
EXPECT_FALSE(media_opt_.video_suspended()); EXPECT_FALSE(media_opt_.IsVideoSuspended());
ASSERT_NO_FATAL_FAILURE( ASSERT_NO_FATAL_FAILURE(
AddFrameAndAdvanceTime((kThresholdBps + kWindowBps) / 1000, false)); AddFrameAndAdvanceTime((kThresholdBps + kWindowBps) / 1000, false));
} }

7
webrtc/modules/video_coding/main/source/video_coding_impl.h
View File

@ -68,7 +68,8 @@ class VideoSender {
uint8_t payloadType, uint8_t payloadType,
bool internalSource); bool internalSource);
int32_t CodecConfigParameters(uint8_t* buffer, int32_t size); int32_t CodecConfigParameters(uint8_t* buffer, int32_t size) const;
int32_t SentFrameCount(VCMFrameCount* frameCount);
int Bitrate(unsigned int* bitrate) const; int Bitrate(unsigned int* bitrate) const;
int FrameRate(unsigned int* framerate) const; int FrameRate(unsigned int* framerate) const;
@ -88,7 +89,6 @@ class VideoSender {
int32_t IntraFrameRequest(int stream_index); int32_t IntraFrameRequest(int stream_index);
int32_t EnableFrameDropper(bool enable); int32_t EnableFrameDropper(bool enable);
int32_t SentFrameCount(VCMFrameCount* frameCount) const;
int SetSenderNackMode(SenderNackMode mode); int SetSenderNackMode(SenderNackMode mode);
int SetSenderReferenceSelection(bool enable); int SetSenderReferenceSelection(bool enable);
@ -122,6 +122,9 @@ class VideoSender {
VCMCodecDataBase _codecDataBase; VCMCodecDataBase _codecDataBase;
bool frame_dropper_enabled_; bool frame_dropper_enabled_;
VCMProcessTimer _sendStatsTimer; VCMProcessTimer _sendStatsTimer;
VCMQMSettingsCallback* qm_settings_callback_;
VCMProtectionCallback* protection_callback_;
}; };
class VideoReceiver { class VideoReceiver {

48
webrtc/modules/video_coding/main/source/video_sender.cc
View File

@ -34,7 +34,9 @@ VideoSender::VideoSender(const int32_t id, Clock* clock)
_encoderInputFile(NULL), _encoderInputFile(NULL),
_codecDataBase(id), _codecDataBase(id),
frame_dropper_enabled_(true), frame_dropper_enabled_(true),
_sendStatsTimer(1000, clock_) {} _sendStatsTimer(1000, clock_),
qm_settings_callback_(NULL),
protection_callback_(NULL) {}
VideoSender::~VideoSender() { VideoSender::~VideoSender() {
delete _sendCritSect; delete _sendCritSect;
@ -70,8 +72,6 @@ int32_t VideoSender::InitializeSender() {
_codecDataBase.ResetSender(); _codecDataBase.ResetSender();
_encoder = NULL; _encoder = NULL;
_encodedFrameCallback.SetTransportCallback(NULL); _encodedFrameCallback.SetTransportCallback(NULL);
// setting default bitRate and frameRate to 0
_mediaOpt.SetEncodingData(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0);
_mediaOpt.Reset(); // Resetting frame dropper _mediaOpt.Reset(); // Resetting frame dropper
return VCM_OK; return VCM_OK;
} }
@ -125,9 +125,8 @@ int32_t VideoSender::RegisterSendCodec(const VideoCodec* sendCodec,
sendCodec->startBitrate * 1000, sendCodec->startBitrate * 1000,
sendCodec->width, sendCodec->width,
sendCodec->height, sendCodec->height,
numLayers); numLayers,
_mediaOpt.set_max_payload_size(maxPayloadSize); maxPayloadSize);
return VCM_OK; return VCM_OK;
} }
@ -171,7 +170,8 @@ int32_t VideoSender::RegisterExternalEncoder(VideoEncoder* externalEncoder,
} }
// Get codec config parameters // Get codec config parameters
int32_t VideoSender::CodecConfigParameters(uint8_t* buffer, int32_t size) { int32_t VideoSender::CodecConfigParameters(uint8_t* buffer,
int32_t size) const {
CriticalSectionScoped cs(_sendCritSect); CriticalSectionScoped cs(_sendCritSect);
if (_encoder != NULL) { if (_encoder != NULL) {
return _encoder->CodecConfigParameters(buffer, size); return _encoder->CodecConfigParameters(buffer, size);
@ -179,6 +179,14 @@ int32_t VideoSender::CodecConfigParameters(uint8_t* buffer, int32_t size) {
return VCM_UNINITIALIZED; return VCM_UNINITIALIZED;
} }
// TODO(andresp): Make const once media_opt is thread-safe and this has a
// pointer to it.
int32_t VideoSender::SentFrameCount(VCMFrameCount* frameCount) {
CriticalSectionScoped cs(_sendCritSect);
*frameCount = _mediaOpt.SentFrameCount();
return VCM_OK;
}
// Get encode bitrate // Get encode bitrate
int VideoSender::Bitrate(unsigned int* bitrate) const { int VideoSender::Bitrate(unsigned int* bitrate) const {
CriticalSectionScoped cs(_sendCritSect); CriticalSectionScoped cs(_sendCritSect);
@ -208,8 +216,11 @@ int32_t VideoSender::SetChannelParameters(uint32_t target_bitrate,
int32_t ret = 0; int32_t ret = 0;
{ {
CriticalSectionScoped sendCs(_sendCritSect); CriticalSectionScoped sendCs(_sendCritSect);
uint32_t targetRate = uint32_t targetRate = _mediaOpt.SetTargetRates(target_bitrate,
_mediaOpt.SetTargetRates(target_bitrate, lossRate, rtt); lossRate,
rtt,
protection_callback_,
qm_settings_callback_);
if (_encoder != NULL) { if (_encoder != NULL) {
ret = _encoder->SetChannelParameters(lossRate, rtt); ret = _encoder->SetChannelParameters(lossRate, rtt);
if (ret < 0) { if (ret < 0) {
@ -247,17 +258,19 @@ int32_t VideoSender::RegisterSendStatisticsCallback(
// Register a video quality settings callback which will be called when frame // Register a video quality settings callback which will be called when frame
// rate/dimensions need to be updated for video quality optimization // rate/dimensions need to be updated for video quality optimization
int32_t VideoSender::RegisterVideoQMCallback( int32_t VideoSender::RegisterVideoQMCallback(
VCMQMSettingsCallback* videoQMSettings) { VCMQMSettingsCallback* qm_settings_callback) {
CriticalSectionScoped cs(_sendCritSect); CriticalSectionScoped cs(_sendCritSect);
return _mediaOpt.RegisterVideoQMCallback(videoQMSettings); qm_settings_callback_ = qm_settings_callback;
_mediaOpt.EnableQM(qm_settings_callback_ != NULL);
return VCM_OK;
} }
// Register a video protection callback which will be called to deliver the // Register a video protection callback which will be called to deliver the
// requested FEC rate and NACK status (on/off). // requested FEC rate and NACK status (on/off).
int32_t VideoSender::RegisterProtectionCallback( int32_t VideoSender::RegisterProtectionCallback(
VCMProtectionCallback* protection) { VCMProtectionCallback* protection_callback) {
CriticalSectionScoped cs(_sendCritSect); CriticalSectionScoped cs(_sendCritSect);
_mediaOpt.RegisterProtectionCallback(protection); protection_callback_ = protection_callback;
return VCM_OK; return VCM_OK;
} }
@ -314,8 +327,6 @@ int32_t VideoSender::AddVideoFrame(const I420VideoFrame& videoFrame,
if (_nextFrameTypes[0] == kFrameEmpty) { if (_nextFrameTypes[0] == kFrameEmpty) {
return VCM_OK; return VCM_OK;
} }
_mediaOpt.UpdateIncomingFrameRate();
if (_mediaOpt.DropFrame()) { if (_mediaOpt.DropFrame()) {
WEBRTC_TRACE(webrtc::kTraceStream, WEBRTC_TRACE(webrtc::kTraceStream,
webrtc::kTraceVideoCoding, webrtc::kTraceVideoCoding,
@ -369,11 +380,6 @@ int32_t VideoSender::EnableFrameDropper(bool enable) {
return VCM_OK; return VCM_OK;
} }
int32_t VideoSender::SentFrameCount(VCMFrameCount* frameCount) const {
CriticalSectionScoped cs(_sendCritSect);
return _mediaOpt.SentFrameCount(frameCount);
}
int VideoSender::SetSenderNackMode(SenderNackMode mode) { int VideoSender::SetSenderNackMode(SenderNackMode mode) {
CriticalSectionScoped cs(_sendCritSect); CriticalSectionScoped cs(_sendCritSect);
@ -443,7 +449,7 @@ void VideoSender::SuspendBelowMinBitrate() {
bool VideoSender::VideoSuspended() const { bool VideoSender::VideoSuspended() const {
CriticalSectionScoped cs(_sendCritSect); CriticalSectionScoped cs(_sendCritSect);
return _mediaOpt.video_suspended(); return _mediaOpt.IsVideoSuspended();
} }
void VideoSender::RegisterPostEncodeImageCallback( void VideoSender::RegisterPostEncodeImageCallback(