|
|
|
@ -8,8 +8,10 @@
|
|
|
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifndef WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_
|
|
|
|
|
#define WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_
|
|
|
|
|
#ifndef WEBRTC_MODULES_VIDEO_CODING_MAIN_SOURCE_MEDIA_OPTIMIZATION_H_
|
|
|
|
|
#define WEBRTC_MODULES_VIDEO_CODING_MAIN_SOURCE_MEDIA_OPTIMIZATION_H_
|
|
|
|
|
|
|
|
|
|
#include <list>
|
|
|
|
|
|
|
|
|
|
#include "webrtc/modules/interface/module_common_types.h"
|
|
|
|
|
#include "webrtc/modules/video_coding/main/interface/video_coding.h"
|
|
|
|
@ -17,22 +19,26 @@
|
|
|
|
|
#include "webrtc/modules/video_coding/main/source/qm_select.h"
|
|
|
|
|
#include "webrtc/system_wrappers/interface/trace.h"
|
|
|
|
|
|
|
|
|
|
#include <list>
|
|
|
|
|
|
|
|
|
|
namespace webrtc {
|
|
|
|
|
|
|
|
|
|
// Forward declarations.
|
|
|
|
|
class Clock;
|
|
|
|
|
class FrameDropper;
|
|
|
|
|
class VCMContentMetricsProcessing;
|
|
|
|
|
|
|
|
|
|
namespace media_optimization {
|
|
|
|
|
|
|
|
|
|
enum { kBitrateMaxFrameSamples = 60 };
|
|
|
|
|
enum { kBitrateAverageWinMs = 1000 };
|
|
|
|
|
enum {
|
|
|
|
|
kBitrateMaxFrameSamples = 60
|
|
|
|
|
};
|
|
|
|
|
enum {
|
|
|
|
|
kBitrateAverageWinMs = 1000
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct VCMEncodedFrameSample {
|
|
|
|
|
VCMEncodedFrameSample(int size_bytes, uint32_t timestamp,
|
|
|
|
|
int64_t time_complete_ms)
|
|
|
|
|
struct EncodedFrameSample {
|
|
|
|
|
EncodedFrameSample(int size_bytes,
|
|
|
|
|
uint32_t timestamp,
|
|
|
|
|
int64_t time_complete_ms)
|
|
|
|
|
: size_bytes(size_bytes),
|
|
|
|
|
timestamp(timestamp),
|
|
|
|
|
time_complete_ms(time_complete_ms) {}
|
|
|
|
@ -42,181 +48,141 @@ struct VCMEncodedFrameSample {
|
|
|
|
|
int64_t time_complete_ms;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
class VCMMediaOptimization
|
|
|
|
|
{
|
|
|
|
|
public:
|
|
|
|
|
VCMMediaOptimization(int32_t id, Clock* clock);
|
|
|
|
|
~VCMMediaOptimization(void);
|
|
|
|
|
/*
|
|
|
|
|
* Reset the Media Optimization module
|
|
|
|
|
*/
|
|
|
|
|
int32_t Reset();
|
|
|
|
|
/**
|
|
|
|
|
* Set target Rates for the encoder given the channel parameters
|
|
|
|
|
* Inputs: target bitrate - the encoder target bitrate in bits/s.
|
|
|
|
|
* fractionLost - packet loss in % in the network
|
|
|
|
|
* roundTripTimeMs - round trip time in milliseconds
|
|
|
|
|
* minBitRate - the bit rate of the end-point with lowest rate
|
|
|
|
|
* maxBitRate - the bit rate of the end-point with highest rate
|
|
|
|
|
*/
|
|
|
|
|
uint32_t SetTargetRates(uint32_t target_bitrate,
|
|
|
|
|
uint8_t &fractionLost,
|
|
|
|
|
uint32_t roundTripTimeMs);
|
|
|
|
|
class MediaOptimization {
|
|
|
|
|
public:
|
|
|
|
|
MediaOptimization(int32_t id, Clock* clock);
|
|
|
|
|
~MediaOptimization(void);
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Inform media optimization of initial encoding state
|
|
|
|
|
*/
|
|
|
|
|
int32_t SetEncodingData(VideoCodecType sendCodecType,
|
|
|
|
|
int32_t maxBitRate,
|
|
|
|
|
uint32_t frameRate,
|
|
|
|
|
uint32_t bitRate,
|
|
|
|
|
uint16_t width,
|
|
|
|
|
uint16_t height,
|
|
|
|
|
int numTemporalLayers);
|
|
|
|
|
/**
|
|
|
|
|
* Enable protection method
|
|
|
|
|
*/
|
|
|
|
|
void EnableProtectionMethod(bool enable, VCMProtectionMethodEnum method);
|
|
|
|
|
/**
|
|
|
|
|
* Returns weather or not protection method is enabled
|
|
|
|
|
*/
|
|
|
|
|
bool IsProtectionMethodEnabled(VCMProtectionMethodEnum method);
|
|
|
|
|
/**
|
|
|
|
|
* Updates the max pay load size
|
|
|
|
|
*/
|
|
|
|
|
void SetMtu(int32_t mtu);
|
|
|
|
|
/*
|
|
|
|
|
* Get actual input frame rate
|
|
|
|
|
*/
|
|
|
|
|
uint32_t InputFrameRate();
|
|
|
|
|
// Resets the Media Optimization module.
|
|
|
|
|
int32_t Reset();
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Get actual sent frame rate
|
|
|
|
|
*/
|
|
|
|
|
uint32_t SentFrameRate();
|
|
|
|
|
/*
|
|
|
|
|
* Get actual sent bit rate
|
|
|
|
|
*/
|
|
|
|
|
uint32_t SentBitRate();
|
|
|
|
|
/*
|
|
|
|
|
* Get maximum allowed bit rate
|
|
|
|
|
*/
|
|
|
|
|
int32_t MaxBitRate();
|
|
|
|
|
/*
|
|
|
|
|
* Inform Media Optimization of encoding output: Length and frame type
|
|
|
|
|
*/
|
|
|
|
|
int32_t UpdateWithEncodedData(int encodedLength,
|
|
|
|
|
uint32_t timestamp,
|
|
|
|
|
FrameType encodedFrameType);
|
|
|
|
|
/*
|
|
|
|
|
* Register a protection callback to be used to inform the user about the
|
|
|
|
|
* protection methods used
|
|
|
|
|
*/
|
|
|
|
|
int32_t RegisterProtectionCallback(VCMProtectionCallback*
|
|
|
|
|
protectionCallback);
|
|
|
|
|
/*
|
|
|
|
|
* Register a quality settings callback to be used to inform VPM/user about
|
|
|
|
|
*/
|
|
|
|
|
int32_t RegisterVideoQMCallback(VCMQMSettingsCallback* videoQMSettings);
|
|
|
|
|
void EnableFrameDropper(bool enable);
|
|
|
|
|
// Sets target rates for the encoder given the channel parameters.
|
|
|
|
|
// Inputs: target bitrate - the encoder target bitrate in bits/s.
|
|
|
|
|
// fraction_lost - packet loss rate in % in the network.
|
|
|
|
|
// round_trip_time_ms - round trip time in milliseconds.
|
|
|
|
|
// min_bit_rate - the bit rate of the end-point with lowest rate.
|
|
|
|
|
// max_bit_rate - the bit rate of the end-point with highest rate.
|
|
|
|
|
uint32_t SetTargetRates(uint32_t target_bitrate,
|
|
|
|
|
uint8_t fraction_lost,
|
|
|
|
|
uint32_t round_trip_time_ms);
|
|
|
|
|
|
|
|
|
|
bool DropFrame();
|
|
|
|
|
// Informs media optimization of initial encoding state.
|
|
|
|
|
int32_t SetEncodingData(VideoCodecType send_codec_type,
|
|
|
|
|
int32_t max_bit_rate,
|
|
|
|
|
uint32_t frame_rate,
|
|
|
|
|
uint32_t bit_rate,
|
|
|
|
|
uint16_t width,
|
|
|
|
|
uint16_t height,
|
|
|
|
|
int num_temporal_layers);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Get number of key/delta frames encoded
|
|
|
|
|
*/
|
|
|
|
|
int32_t SentFrameCount(VCMFrameCount &frameCount) const;
|
|
|
|
|
// Enables protection method.
|
|
|
|
|
void EnableProtectionMethod(bool enable, VCMProtectionMethodEnum method);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* update incoming frame rate value
|
|
|
|
|
*/
|
|
|
|
|
void UpdateIncomingFrameRate();
|
|
|
|
|
// Returns weather or not protection method is enabled.
|
|
|
|
|
bool IsProtectionMethodEnabled(VCMProtectionMethodEnum method);
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Update content metric Data
|
|
|
|
|
*/
|
|
|
|
|
void UpdateContentData(const VideoContentMetrics* contentMetrics);
|
|
|
|
|
// Returns the actual input frame rate.
|
|
|
|
|
uint32_t InputFrameRate();
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Compute new Quality Mode
|
|
|
|
|
*/
|
|
|
|
|
int32_t SelectQuality();
|
|
|
|
|
// Returns the actual sent frame rate.
|
|
|
|
|
uint32_t SentFrameRate();
|
|
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
typedef std::list<VCMEncodedFrameSample> FrameSampleList;
|
|
|
|
|
// Returns the actual sent bit rate.
|
|
|
|
|
uint32_t SentBitRate();
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Update protection callback with protection settings
|
|
|
|
|
*/
|
|
|
|
|
int UpdateProtectionCallback(VCMProtectionMethod *selected_method,
|
|
|
|
|
uint32_t* total_video_rate_bps,
|
|
|
|
|
uint32_t* nack_overhead_rate_bps,
|
|
|
|
|
uint32_t* fec_overhead_rate_bps);
|
|
|
|
|
// Informs Media Optimization of encoding output: Length and frame type.
|
|
|
|
|
int32_t UpdateWithEncodedData(int encoded_length,
|
|
|
|
|
uint32_t timestamp,
|
|
|
|
|
FrameType encoded_frame_type);
|
|
|
|
|
|
|
|
|
|
void PurgeOldFrameSamples(int64_t now_ms);
|
|
|
|
|
void UpdateSentBitrate(int64_t nowMs);
|
|
|
|
|
void UpdateSentFramerate();
|
|
|
|
|
// Registers a protection callback to be used to inform the user about the
|
|
|
|
|
// protection methods used.
|
|
|
|
|
int32_t RegisterProtectionCallback(
|
|
|
|
|
VCMProtectionCallback* protection_callback);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* verify if QM settings differ from default, i.e. if an update is required
|
|
|
|
|
* Compute actual values, as will be sent to the encoder
|
|
|
|
|
*/
|
|
|
|
|
bool QMUpdate(VCMResolutionScale* qm);
|
|
|
|
|
/**
|
|
|
|
|
* check if we should make a QM change
|
|
|
|
|
* will return 1 if yes, 0 otherwise
|
|
|
|
|
*/
|
|
|
|
|
bool CheckStatusForQMchange();
|
|
|
|
|
// Registers a quality settings callback to be used to inform VPM/user.
|
|
|
|
|
int32_t RegisterVideoQMCallback(VCMQMSettingsCallback* video_qmsettings);
|
|
|
|
|
|
|
|
|
|
void ProcessIncomingFrameRate(int64_t now);
|
|
|
|
|
void EnableFrameDropper(bool enable);
|
|
|
|
|
|
|
|
|
|
enum { kFrameCountHistorySize = 90};
|
|
|
|
|
enum { kFrameHistoryWinMs = 2000};
|
|
|
|
|
bool DropFrame();
|
|
|
|
|
|
|
|
|
|
int32_t _id;
|
|
|
|
|
Clock* _clock;
|
|
|
|
|
int32_t _maxBitRate;
|
|
|
|
|
VideoCodecType _sendCodecType;
|
|
|
|
|
uint16_t _codecWidth;
|
|
|
|
|
uint16_t _codecHeight;
|
|
|
|
|
float _userFrameRate;
|
|
|
|
|
// Returns the number of key/delta frames encoded.
|
|
|
|
|
int32_t SentFrameCount(VCMFrameCount* frame_count) const;
|
|
|
|
|
|
|
|
|
|
FrameDropper* _frameDropper;
|
|
|
|
|
VCMLossProtectionLogic* _lossProtLogic;
|
|
|
|
|
uint8_t _fractionLost;
|
|
|
|
|
// Updates incoming frame rate value.
|
|
|
|
|
void UpdateIncomingFrameRate();
|
|
|
|
|
|
|
|
|
|
// Update content metric data.
|
|
|
|
|
void UpdateContentData(const VideoContentMetrics* content_metrics);
|
|
|
|
|
|
|
|
|
|
uint32_t _sendStatistics[4];
|
|
|
|
|
uint32_t _sendStatisticsZeroEncode;
|
|
|
|
|
int32_t _maxPayloadSize;
|
|
|
|
|
uint32_t _targetBitRate;
|
|
|
|
|
// Computes new Quality Mode.
|
|
|
|
|
int32_t SelectQuality();
|
|
|
|
|
|
|
|
|
|
float _incomingFrameRate;
|
|
|
|
|
int64_t _incomingFrameTimes[kFrameCountHistorySize];
|
|
|
|
|
// Accessors and mutators.
|
|
|
|
|
int32_t max_bit_rate() const { return max_bit_rate_; }
|
|
|
|
|
void set_max_payload_size(int32_t mtu) { max_payload_size_ = mtu; }
|
|
|
|
|
|
|
|
|
|
bool _enableQm;
|
|
|
|
|
private:
|
|
|
|
|
typedef std::list<EncodedFrameSample> FrameSampleList;
|
|
|
|
|
enum {
|
|
|
|
|
kFrameCountHistorySize = 90
|
|
|
|
|
};
|
|
|
|
|
enum {
|
|
|
|
|
kFrameHistoryWinMs = 2000
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
VCMProtectionCallback* _videoProtectionCallback;
|
|
|
|
|
VCMQMSettingsCallback* _videoQMSettingsCallback;
|
|
|
|
|
// Updates protection callback with protection settings.
|
|
|
|
|
int UpdateProtectionCallback(VCMProtectionMethod* selected_method,
|
|
|
|
|
uint32_t* total_video_rate_bps,
|
|
|
|
|
uint32_t* nack_overhead_rate_bps,
|
|
|
|
|
uint32_t* fec_overhead_rate_bps);
|
|
|
|
|
|
|
|
|
|
std::list<VCMEncodedFrameSample> _encodedFrameSamples;
|
|
|
|
|
uint32_t _avgSentBitRateBps;
|
|
|
|
|
uint32_t _avgSentFramerate;
|
|
|
|
|
void PurgeOldFrameSamples(int64_t now_ms);
|
|
|
|
|
void UpdateSentBitrate(int64_t now_ms);
|
|
|
|
|
void UpdateSentFramerate();
|
|
|
|
|
|
|
|
|
|
uint32_t _keyFrameCnt;
|
|
|
|
|
uint32_t _deltaFrameCnt;
|
|
|
|
|
// Verifies if QM settings differ from default, i.e. if an update is required.
|
|
|
|
|
// Computes actual values, as will be sent to the encoder.
|
|
|
|
|
bool QMUpdate(VCMResolutionScale* qm);
|
|
|
|
|
|
|
|
|
|
VCMContentMetricsProcessing* _content;
|
|
|
|
|
VCMQmResolution* _qmResolution;
|
|
|
|
|
// Checks if we should make a QM change. Return true if yes, false otherwise.
|
|
|
|
|
bool CheckStatusForQMchange();
|
|
|
|
|
|
|
|
|
|
int64_t _lastQMUpdateTime;
|
|
|
|
|
int64_t _lastChangeTime; // content/user triggered
|
|
|
|
|
int _numLayers;
|
|
|
|
|
void ProcessIncomingFrameRate(int64_t now);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
}; // end of VCMMediaOptimization class definition
|
|
|
|
|
int32_t id_;
|
|
|
|
|
Clock* clock_;
|
|
|
|
|
int32_t max_bit_rate_;
|
|
|
|
|
VideoCodecType send_codec_type_;
|
|
|
|
|
uint16_t codec_width_;
|
|
|
|
|
uint16_t codec_height_;
|
|
|
|
|
float user_frame_rate_;
|
|
|
|
|
FrameDropper* frame_dropper_;
|
|
|
|
|
VCMLossProtectionLogic* loss_prot_logic_;
|
|
|
|
|
uint8_t fraction_lost_;
|
|
|
|
|
uint32_t send_statistics_[4];
|
|
|
|
|
uint32_t send_statistics_zero_encode_;
|
|
|
|
|
int32_t max_payload_size_;
|
|
|
|
|
uint32_t target_bit_rate_;
|
|
|
|
|
float incoming_frame_rate_;
|
|
|
|
|
int64_t incoming_frame_times_[kFrameCountHistorySize];
|
|
|
|
|
bool enable_qm_;
|
|
|
|
|
VCMProtectionCallback* video_protection_callback_;
|
|
|
|
|
VCMQMSettingsCallback* video_qmsettings_callback_;
|
|
|
|
|
std::list<EncodedFrameSample> encoded_frame_samples_;
|
|
|
|
|
uint32_t avg_sent_bit_rate_bps_;
|
|
|
|
|
uint32_t avg_sent_framerate_;
|
|
|
|
|
uint32_t key_frame_cnt_;
|
|
|
|
|
uint32_t delta_frame_cnt_;
|
|
|
|
|
VCMContentMetricsProcessing* content_;
|
|
|
|
|
VCMQmResolution* qm_resolution_;
|
|
|
|
|
int64_t last_qm_update_time_;
|
|
|
|
|
int64_t last_change_time_; // Content/user triggered.
|
|
|
|
|
int num_layers_;
|
|
|
|
|
}; // End of MediaOptimization class declaration.
|
|
|
|
|
|
|
|
|
|
} // namespace media_optimization
|
|
|
|
|
} // namespace webrtc
|
|
|
|
|
|
|
|
|
|
#endif // WEBRTC_MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_
|
|
|
|
|
#endif // WEBRTC_MODULES_VIDEO_CODING_MAIN_SOURCE_MEDIA_OPTIMIZATION_H_
|
|
|
|
|