diff --git a/src/modules/video_coding/main/source/media_opt_util.cc b/src/modules/video_coding/main/source/media_opt_util.cc index a0937b66e6..4f7f40c11b 100644 --- a/src/modules/video_coding/main/source/media_opt_util.cc +++ b/src/modules/video_coding/main/source/media_opt_util.cc @@ -21,6 +21,13 @@ namespace webrtc { +void +VCMProtectionMethod::UpdateContentMetrics( + const VideoContentMetrics* contentMetrics) +{ + _qmRobustness->UpdateContent(contentMetrics); +} + bool VCMProtectionMethod::BetterThan(VCMProtectionMethod *pm) { @@ -417,6 +424,10 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters) codeRateDelta = plossMax - 1; } + codeRateDelta = _qmRobustness->AdjustFecFactor(codeRateDelta, bitRate, + parameters->frameRate, + parameters->rtt, packetLoss); + // For Key frame: // Effectively at a higher rate, so we scale/boost the rate // The boost factor may depend on several factors: ratio of packet @@ -462,6 +473,14 @@ VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters) _protectionFactorK = codeRateKey; _protectionFactorD = codeRateDelta; + + // Set the UEP protection on/off for Key and Delta frames + _uepKey = _qmRobustness->SetUepProtection(codeRateKey, bitRate, + packetLoss, 0); + + _uepDelta = _qmRobustness->SetUepProtection(codeRateKey, bitRate, + packetLoss, 1); + // DONE WITH FEC PROTECTION SETTINGS return true; } diff --git a/src/modules/video_coding/main/source/media_opt_util.h b/src/modules/video_coding/main/source/media_opt_util.h index 8298406569..3f78dd7381 100644 --- a/src/modules/video_coding/main/source/media_opt_util.h +++ b/src/modules/video_coding/main/source/media_opt_util.h @@ -17,6 +17,7 @@ #include "exp_filter.h" #include "internal_defines.h" #include "tick_time.h" +#include "qm_select.h" #include #include @@ -97,8 +98,10 @@ public: //friend VCMProtectionMethod; VCMProtectionMethod(VCMProtectionMethodEnum type) : _protectionFactorK(0), _protectionFactorD(0), _residualPacketLoss(0.0), _scaleProtKey(2.0), - _maxPayloadSize(1460), _efficiency(0), _score(0), _type(type) {} - virtual ~VCMProtectionMethod() {} + _maxPayloadSize(1460), _efficiency(0), _score(0), _type(type), + _uepKey(0), _uepDelta(1) + {_qmRobustness = new VCMQmRobustness();} + virtual ~VCMProtectionMethod() { delete _qmRobustness;} // Updates the efficiency of the method using the parameters provided // @@ -142,6 +145,9 @@ public: // Return value : Required protectionFactor for delta frame virtual WebRtc_UWord8 RequiredProtectionFactorD() { return _protectionFactorD; } + // Updates content metrics + void UpdateContentMetrics(const VideoContentMetrics* contentMetrics); + WebRtc_UWord8 _effectivePacketLoss; WebRtc_UWord8 _protectionFactorK; WebRtc_UWord8 _protectionFactorD; @@ -149,6 +155,10 @@ public: float _scaleProtKey; WebRtc_Word32 _maxPayloadSize; + VCMQmRobustness* _qmRobustness; + bool _uepKey; + bool _uepDelta; + protected: float _efficiency; float _score; diff --git a/src/modules/video_coding/main/source/media_optimization.cc b/src/modules/video_coding/main/source/media_optimization.cc index e215986154..3b0351574d 100644 --- a/src/modules/video_coding/main/source/media_optimization.cc +++ b/src/modules/video_coding/main/source/media_optimization.cc @@ -46,7 +46,7 @@ _lastChangeTime(0) _frameDropper = new VCMFrameDropper(_id); _lossProtLogic = new VCMLossProtectionLogic(); _content = new VCMContentMetricsProcessing(); - _qms = new VCMQmSelect(); + _qmResolution = new VCMQmResolution(); } VCMMediaOptimization::~VCMMediaOptimization(void) @@ -55,7 +55,7 @@ VCMMediaOptimization::~VCMMediaOptimization(void) delete _lossProtLogic; delete _frameDropper; delete _content; - delete _qms; + delete _qmResolution; } WebRtc_Word32 @@ -67,7 +67,7 @@ VCMMediaOptimization::Reset() _lossProtLogic->Reset(); _frameDropper->SetRates(0, 0); _content->Reset(); - _qms->Reset(); + _qmResolution->Reset(); _lossProtLogic->UpdateFrameRate(_incomingFrameRate); _lossProtLogic->Reset(); _sendStatisticsZeroEncode = 0; @@ -135,6 +135,10 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, selectedMethod->Type() == kNackFec )) { + // Update protection method with content metrics + selectedMethod->UpdateContentMetrics(_content->ShortTermAvgData()); + + // Update method will compute the robustness settings for the given // protection method and the overhead cost // the protection method is set by the user via SetVideoProtection. @@ -197,8 +201,8 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, if (_enableQm) { //Update QM with rates - _qms->UpdateRates((float)_targetBitRate, _avgSentBitRateBps, - _incomingFrameRate, _fractionLost); + _qmResolution->UpdateRates((float)_targetBitRate, _avgSentBitRateBps, + _incomingFrameRate, _fractionLost); //Check for QM selection bool selectQM = checkStatusForQMchange(); if (selectQM) @@ -249,11 +253,12 @@ VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, WebRtc_Word3 _lossProtLogic->UpdateFrameSize(width, height); _frameDropper->Reset(); _frameDropper->SetRates(static_cast(bitRate), static_cast(frameRate)); - _userFrameRate = (float)frameRate; + _userFrameRate = static_cast(frameRate); _codecWidth = width; _codecHeight = height; WebRtc_Word32 ret = VCM_OK; - ret = _qms->Initialize((float)_targetBitRate, _userFrameRate, _codecWidth, _codecHeight); + ret = _qmResolution->Initialize((float)_targetBitRate, _userFrameRate, + _codecWidth, _codecHeight); return ret; } @@ -420,7 +425,8 @@ VCMMediaOptimization::UpdateWithEncodedData(WebRtc_Word32 encodedLength, if (_enableQm) { // update quality select with encoded length - _qms->UpdateEncodedSize(encodedLength, encodedFrameType); + _qmResolution->UpdateEncodedSize(encodedLength, + encodedFrameType); } } if (!deltaFrame && encodedLength > 0) @@ -525,7 +531,7 @@ VCMMediaOptimization::updateContentData(const VideoContentMetrics *contentMetric { //No QM if metrics are NULL _enableQm = false; - _qms->Reset(); + _qmResolution->Reset(); } else { @@ -537,14 +543,14 @@ WebRtc_Word32 VCMMediaOptimization::SelectQuality() { // Reset quantities for QM select - _qms->ResetQM(); + _qmResolution->ResetQM(); // Update QM will long-term averaged content metrics. - _qms->UpdateContent(_content->LongTermAvgData()); + _qmResolution->UpdateContent(_content->LongTermAvgData()); // Select quality mode - VCMQualityMode* qm = NULL; - WebRtc_Word32 ret = _qms->SelectQuality(&qm); + VCMResolutionScale* qm = NULL; + WebRtc_Word32 ret = _qmResolution->SelectResolution(&qm); if (ret < 0) { return ret; @@ -554,7 +560,7 @@ VCMMediaOptimization::SelectQuality() QMUpdate(qm); // Reset all the rate and related frame counters quantities - _qms->ResetRates(); + _qmResolution->ResetRates(); // Reset counters _lastQMUpdateTime = VCMTickTime::MillisecondTimestamp(); @@ -592,7 +598,7 @@ VCMMediaOptimization::checkStatusForQMchange() } bool -VCMMediaOptimization::QMUpdate(VCMQualityMode* qm) +VCMMediaOptimization::QMUpdate(VCMResolutionScale* qm) { // Check for no change if (qm->spatialHeightFact == 1 && @@ -606,7 +612,9 @@ VCMMediaOptimization::QMUpdate(VCMQualityMode* qm) VideoContentMetrics* cm = _content->LongTermAvgData(); // Temporal - WebRtc_UWord32 frameRate = static_cast(_incomingFrameRate + 0.5f); + WebRtc_UWord32 frameRate = static_cast + (_incomingFrameRate + 0.5f); + // Check if go back up in temporal resolution if (qm->temporalFact == 0) { diff --git a/src/modules/video_coding/main/source/media_optimization.h b/src/modules/video_coding/main/source/media_optimization.h index 8a9993e5f6..eecf270745 100644 --- a/src/modules/video_coding/main/source/media_optimization.h +++ b/src/modules/video_coding/main/source/media_optimization.h @@ -159,7 +159,7 @@ private: * verify if QM settings differ from default, i.e. if an update is required * Compute actual values, as will be sent to the encoder */ - bool QMUpdate(VCMQualityMode* qm); + bool QMUpdate(VCMResolutionScale* qm); /** * check if we should make a QM change * will return 1 if yes, 0 otherwise @@ -207,7 +207,7 @@ private: WebRtc_UWord32 _deltaFrameCnt; VCMContentMetricsProcessing* _content; - VCMQmSelect* _qms; + VCMQmResolution* _qmResolution; WebRtc_Word64 _lastQMUpdateTime; WebRtc_Word64 _lastChangeTime; // content or user triggered diff --git a/src/modules/video_coding/main/source/qm_select.cc b/src/modules/video_coding/main/source/qm_select.cc index 3220db08a2..82c31daa21 100644 --- a/src/modules/video_coding/main/source/qm_select.cc +++ b/src/modules/video_coding/main/source/qm_select.cc @@ -20,36 +20,204 @@ namespace webrtc { -VCMQmSelect::VCMQmSelect() +// QM-METHOD class + +VCMQmMethod::VCMQmMethod() { - _qm = new VCMQualityMode(); _contentMetrics = new VideoContentMetrics(); - Reset(); + ResetQM(); } -VCMQmSelect::~VCMQmSelect() +VCMQmMethod::~VCMQmMethod() { - delete _qm; delete _contentMetrics; } void -VCMQmSelect::ResetQM() +VCMQmMethod::ResetQM() { _motion.Reset(); _spatial.Reset(); _coherence.Reset(); _stationaryMotion = 0; _aspectRatio = 1; - _maxRateQM = 0; - _imageType = 1; - _userResolutionPref = 50; // Neutral - _qm->Reset(); + _imageType = 2; return; } void -VCMQmSelect::ResetRates() +VCMQmMethod::UpdateContent(const VideoContentMetrics* contentMetrics) +{ + _contentMetrics = contentMetrics; +} + +void +VCMQmMethod::MotionNFD() +{ + _motion.value = _contentMetrics->motionMagnitudeNZ; + + // Determine motion level + if (_motion.value < LOW_MOTION_NFD) + { + _motion.level = kLow; + } + else if (_motion.value > HIGH_MOTION_NFD) + { + _motion.level = kHigh; + } + else + { + _motion.level = kDefault; + } + +} + +void +VCMQmMethod::Motion() +{ + + float sizeZeroMotion = _contentMetrics->sizeZeroMotion; + float motionMagNZ = _contentMetrics->motionMagnitudeNZ; + + // Take product of size and magnitude with equal weight + _motion.value = (1.0f - sizeZeroMotion) * motionMagNZ; + + // Stabilize: motionMagNZ could be large when only a + // few motion blocks are non-zero + _stationaryMotion = false; + if (sizeZeroMotion > HIGH_ZERO_MOTION_SIZE) + { + _motion.value = 0.0f; + _stationaryMotion = true; + } + // Determine motion level + if (_motion.value < LOW_MOTION) + { + _motion.level = kLow; + } + else if (_motion.value > HIGH_MOTION) + { + _motion.level = kHigh; + } + else + { + _motion.level = kDefault; + } +} + + +void +VCMQmMethod::Spatial() +{ + float spatialErr = _contentMetrics->spatialPredErr; + float spatialErrH = _contentMetrics->spatialPredErrH; + float spatialErrV = _contentMetrics->spatialPredErrV; + // Spatial measure: take average of 3 prediction errors + _spatial.value = (spatialErr + spatialErrH + spatialErrV) / 3.0f; + + float scale = 1.0f; + // Reduce thresholds for HD scenes + if (_imageType > 3) + { + scale = (float)SCALE_TEXTURE_HD; + } + + if (_spatial.value > scale * HIGH_TEXTURE) + { + _spatial.level = kHigh; + } + else if (_spatial.value < scale * LOW_TEXTURE) + { + _spatial.level = kLow; + } + else + { + _spatial.level = kDefault; + } +} + +void +VCMQmMethod::Coherence() +{ + float horizNZ = _contentMetrics->motionHorizontalness; + float distortionNZ = _contentMetrics->motionClusterDistortion; + + // Coherence measure: combine horizontalness with cluster distortion + _coherence.value = COH_MAX; + if (distortionNZ > 0.) + { + _coherence.value = horizNZ / distortionNZ; + } + _coherence.value = VCM_MIN(COH_MAX, _coherence.value); + + if (_coherence.value < COHERENCE_THR) + { + _coherence.level = kLow; + } + else + { + _coherence.level = kHigh; + } + +} + +WebRtc_Word8 +VCMQmMethod::GetImageType(WebRtc_UWord32 width, WebRtc_UWord32 height) +{ + // Match image type + WebRtc_UWord32 imageSize = width * height; + WebRtc_Word8 imageType; + + if (imageSize < kFrameSizeTh[0]) + { + imageType = 0; + } + else if (imageSize < kFrameSizeTh[1]) + { + imageType = 1; + } + else if (imageSize < kFrameSizeTh[2]) + { + imageType = 2; + } + else if (imageSize < kFrameSizeTh[3]) + { + imageType = 3; + } + else if (imageSize < kFrameSizeTh[4]) + { + imageType = 4; + } + else if (imageSize < kFrameSizeTh[5]) + { + imageType = 5; + } + else + { + imageType = 6; + } + + return imageType; +} + +// DONE WITH QM CLASS + + +//RESOLUTION CLASS + +VCMQmResolution::VCMQmResolution() +{ + _qm = new VCMResolutionScale(); + Reset(); +} + +VCMQmResolution::~VCMQmResolution() +{ + delete _qm; +} + +void +VCMQmResolution::ResetRates() { _sumEncodedBytes = 0; _sumTargetRate = 0.0f; @@ -65,7 +233,7 @@ VCMQmSelect::ResetRates() } void -VCMQmSelect::Reset() +VCMQmResolution::Reset() { _stateDecFactorSpatial = 1; _stateDecFactorTemp = 1; @@ -74,17 +242,14 @@ VCMQmSelect::Reset() _incomingFrameRate = 0.0f; _userFrameRate = 0.0f; _perFrameBandwidth =0.0f; - _prevTotalRate = 0.0f; - _prevRttTime = 0; - _prevPacketLoss = 0; - ResetQM(); - ResetRates(); - return; + ResetRates(); + ResetQM(); + return; } -//Initialize after reset of encoder +// Initialize rate control quantities after reset of encoder WebRtc_Word32 -VCMQmSelect::Initialize(float bitRate, float userFrameRate, +VCMQmResolution::Initialize(float bitRate, float userFrameRate, WebRtc_UWord32 width, WebRtc_UWord32 height) { if (userFrameRate == 0.0f || width == 0 || height == 0) @@ -98,6 +263,12 @@ VCMQmSelect::Initialize(float bitRate, float userFrameRate, _width = width; _height = height; + // Aspect ratio: used for selection of 1x2,2x1,2x2 + _aspectRatio = static_cast(_width) / static_cast(_height); + + // Set the imageType for the encoder width/height. + _imageType = GetImageType(_width, _height); + // Initial buffer level _bufferLevel = INIT_BUFFER_LEVEL * _targetBitRate; @@ -121,19 +292,9 @@ VCMQmSelect::Initialize(float bitRate, float userFrameRate, return VCM_OK; } -WebRtc_Word32 -VCMQmSelect::SetPreferences(WebRtc_Word8 resolPref) -{ - // Preference setting for temporal over spatial resolution - // 100 means temporal, 0 means spatial, 50 is neutral - _userResolutionPref = resolPref; - - return VCM_OK; -} - -//Update after every encoded frame +// Update after every encoded frame void -VCMQmSelect::UpdateEncodedSize(WebRtc_Word64 encodedSize, +VCMQmResolution::UpdateEncodedSize(WebRtc_Word64 encodedSize, FrameType encodedFrameType) { // Update encoded size; @@ -167,16 +328,16 @@ VCMQmSelect::UpdateEncodedSize(WebRtc_Word64 encodedSize, */ // Counter for occurrences of low buffer level - if (_bufferLevel <= PERC_BUFFER_THR * INIT_BUFFER_LEVEL * _targetBitRate) + if (_bufferLevel <= PERC_BUFFER_THR * OPT_BUFFER_LEVEL * _targetBitRate) { _lowBufferCnt++; } } -//Update various quantities after SetTargetRates in MediaOpt +// Update various quantities after SetTargetRates in MediaOpt void -VCMQmSelect::UpdateRates(float targetBitRate, float avgSentBitRate, +VCMQmResolution::UpdateRates(float targetBitRate, float avgSentBitRate, float incomingFrameRate, WebRtc_UWord8 packetLoss) { @@ -219,42 +380,10 @@ VCMQmSelect::UpdateRates(float targetBitRate, float avgSentBitRate, } -// Adjust the FEC rate based on the content and the network state -// (packet loss rate, total rate/bandwidth, round trip time). -// Note that packetLoss here is the filtered loss value. -WebRtc_UWord8 -VCMQmSelect::AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate, - float frameRate,WebRtc_UWord16 rttTime, - WebRtc_UWord8 packetLoss) -{ - // Default: no adjustment - WebRtc_UWord8 codeRateDeltaAdjust = codeRateDelta; - float adjustFec = 1.0f; - - // TODO (marpan): - // Set FEC adjustment factor - - codeRateDeltaAdjust = static_cast(codeRateDelta * adjustFec); - - // Keep track of previous values of network state: - // adjustment may be also based on pattern of changes in network state - _prevTotalRate = totalRate; - _prevRttTime = rttTime; - _prevPacketLoss = packetLoss; - - return codeRateDeltaAdjust; -} - -void -VCMQmSelect::UpdateContent(const VideoContentMetrics* contentMetrics) -{ - _contentMetrics = contentMetrics; -} - // Select the resolution factors: frame size and frame rate change: (QM modes) // Selection is for going back up in resolution, or going down in. WebRtc_Word32 -VCMQmSelect::SelectQuality(VCMQualityMode** qm) +VCMQmResolution::SelectResolution(VCMResolutionScale** qm) { if (!_init) { @@ -272,15 +401,11 @@ VCMQmSelect::SelectQuality(VCMQualityMode** qm) _qm->spatialHeightFact = 1; _qm->temporalFact = 1; - // Update native values _nativeWidth = _contentMetrics->nativeWidth; _nativeHeight = _contentMetrics->nativeHeight; _nativeFrameRate = _contentMetrics->nativeFrameRate; - // Aspect ratio: used for selection of 1x2,2x1,2x2 - _aspectRatio = (float)_width / (float)_height; - float avgTargetRate = 0.0f; float avgIncomingFrameRate = 0.0f; float ratioBufferLow = 0.0f; @@ -317,13 +442,12 @@ VCMQmSelect::SelectQuality(VCMQualityMode** qm) // for up-sampled spatial dimensions. // This is needed to get the transRate for going back up in // spatial resolution (only 2x2 allowed in this version). - SetMaxRateForQM(2 * _width, 2 * _height); - WebRtc_UWord8 imageType2 = _imageType; - WebRtc_UWord32 maxRateQM2 = _maxRateQM; + WebRtc_UWord8 imageType2 = GetImageType(2 * _width, 2 * _height); + WebRtc_UWord32 maxRateQM2 = kMaxRateQm[imageType2]; // Set the maximum transitional rate and image type: // for the encoder spatial dimensions. - SetMaxRateForQM(_width, _height); + WebRtc_UWord32 maxRateQM = kMaxRateQm[_imageType]; // Compute class state of the content. MotionNFD(); @@ -342,18 +466,21 @@ VCMQmSelect::SelectQuality(VCMQualityMode** qm) // Get image class and content class: for going up spatially WebRtc_UWord8 imageClass2 = 1; - if (imageType2 <= 3) imageClass2 = 0; + if (imageType2 <= 3) + { + imageClass2 = 0; + } WebRtc_UWord8 tableIndex2 = imageClass2 * 9 + contentClass; float scaleTransRate2 = kScaleTransRateQm[tableIndex2]; // Transitonal rate for going down WebRtc_UWord32 estimatedTransRateDown = static_cast - (_incomingFrameRate * scaleTransRate * _maxRateQM / 30); + (_incomingFrameRate * scaleTransRate * maxRateQM / 30); // Transitional rate for going up temporally WebRtc_UWord32 estimatedTransRateUpT = static_cast (TRANS_RATE_SCALE_UP_TEMP * 2 * _incomingFrameRate * - scaleTransRate * _maxRateQM / 30); + scaleTransRate * maxRateQM / 30); // Transitional rate for going up spatially WebRtc_UWord32 estimatedTransRateUpS = static_cast @@ -530,7 +657,7 @@ VCMQmSelect::SelectQuality(VCMQualityMode** qm) } WebRtc_Word32 -VCMQmSelect::SelectSpatialDirectionMode(float transRate) +VCMQmResolution::SelectSpatialDirectionMode(float transRate) { // Default is 1x2 (H) @@ -579,156 +706,82 @@ VCMQmSelect::SelectSpatialDirectionMode(float transRate) return VCM_OK; } -void -VCMQmSelect::Coherence() +// DONE WITH RESOLUTION CLASS + + +// ROBUSTNESS CLASS + +VCMQmRobustness::VCMQmRobustness() { - float horizNZ = _contentMetrics->motionHorizontalness; - float distortionNZ = _contentMetrics->motionClusterDistortion; + Reset(); +} - // Coherence measure: combine horizontalness with cluster distortion - _coherence.value = COH_MAX; - if (distortionNZ > 0.) - { - _coherence.value = horizNZ / distortionNZ; - } - _coherence.value = VCM_MIN(COH_MAX, _coherence.value); - - if (_coherence.value < COHERENCE_THR) - { - _coherence.level = kLow; - } - else - { - _coherence.level = kHigh; - } +VCMQmRobustness::~VCMQmRobustness() +{ } void -VCMQmSelect::MotionNFD() +VCMQmRobustness::Reset() { - _motion.value = _contentMetrics->motionMagnitudeNZ; - - // Determine motion level - if (_motion.value < LOW_MOTION_NFD) - { - _motion.level = kLow; - } - else if (_motion.value > HIGH_MOTION_NFD) - { - _motion.level = kHigh; - } - else - { - _motion.level = kDefault; - } - + _prevTotalRate = 0.0f; + _prevRttTime = 0; + _prevPacketLoss = 0; + ResetQM(); + return; } -void -VCMQmSelect::Motion() +// Adjust the FEC rate based on the content and the network state +// (packet loss rate, total rate/bandwidth, round trip time). +// Note that packetLoss here is the filtered loss value. +WebRtc_UWord8 +VCMQmRobustness::AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate, + float frameRate,WebRtc_UWord32 rttTime, + WebRtc_UWord8 packetLoss) { + if (_contentMetrics == NULL) + { + return VCM_OK; + } - float sizeZeroMotion = _contentMetrics->sizeZeroMotion; - float motionMagNZ = _contentMetrics->motionMagnitudeNZ; + // Default: no adjustment + WebRtc_UWord8 codeRateDeltaAdjust = codeRateDelta; + float adjustFec = 1.0f; - // Take product of size and magnitude with equal weight - _motion.value = (1.0f - sizeZeroMotion) * motionMagNZ; + // Compute class state of the content. + MotionNFD(); + Spatial(); - // Stabilize: motionMagNZ could be large when only a - // few motion blocks are non-zero - _stationaryMotion = false; - if (sizeZeroMotion > HIGH_ZERO_MOTION_SIZE) - { - _motion.value = 0.0f; - _stationaryMotion = true; - } - // Determine motion level - if (_motion.value < LOW_MOTION) - { - _motion.level = kLow; - } - else if (_motion.value > HIGH_MOTION) - { - _motion.level = kHigh; - } - else - { - _motion.level = kDefault; - } + // TODO (marpan): + // Set FEC adjustment factor + + codeRateDeltaAdjust = static_cast(codeRateDelta * adjustFec); + + // Keep track of previous values of network state: + // adjustment may be also based on pattern of changes in network state + _prevTotalRate = totalRate; + _prevRttTime = rttTime; + _prevPacketLoss = packetLoss; + + _prevCodeRateDelta = codeRateDelta; + + return codeRateDeltaAdjust; } - -void -VCMQmSelect::Spatial() +// Set the UEP (unequal-protection) on/off for the FEC +bool +VCMQmRobustness::SetUepProtection(WebRtc_UWord8 codeRateDelta, float totalRate, + WebRtc_UWord8 packetLoss, bool frameType) { - float spatialErr = _contentMetrics->spatialPredErr; - float spatialErrH = _contentMetrics->spatialPredErrH; - float spatialErrV = _contentMetrics->spatialPredErrV; - // Spatial measure: take average of 3 prediction errors - _spatial.value = (spatialErr + spatialErrH + spatialErrV) / 3.0f; - - float scale = 1.0f; - // Reduce thresholds for HD scenes - if (_imageType > 3) + if (_contentMetrics == NULL) { - scale = (float)SCALE_TEXTURE_HD; + return VCM_OK; } - if (_spatial.value > scale * HIGH_TEXTURE) - { - _spatial.level = kHigh; - } - else if (_spatial.value < scale * LOW_TEXTURE) - { - _spatial.level = kLow; - } - else - { - _spatial.level = kDefault; - } -} + // Default: UEP on + bool uepProtection = true; - -WebRtc_Word32 -VCMQmSelect::SetMaxRateForQM(WebRtc_UWord32 width, WebRtc_UWord32 height) -{ - // Match image type - WebRtc_UWord32 imageSize = width * height; - - if (imageSize < kFrameSizeTh[0]) - { - _imageType = 0; - } - else if (imageSize < kFrameSizeTh[1]) - { - _imageType = 1; - } - else if (imageSize < kFrameSizeTh[2]) - { - _imageType = 2; - } - else if (imageSize < kFrameSizeTh[3]) - { - _imageType = 3; - } - else if (imageSize < kFrameSizeTh[4]) - { - _imageType = 4; - } - else if (imageSize < kFrameSizeTh[5]) - { - _imageType = 5; - } - else - { - _imageType = 6; - } - - // Set max rate based on image size - _maxRateQM = kMaxRateQm[_imageType]; - - return VCM_OK; + return uepProtection; } } // end of namespace diff --git a/src/modules/video_coding/main/source/qm_select.h b/src/modules/video_coding/main/source/qm_select.h index 3bca4bce90..589a16dad7 100644 --- a/src/modules/video_coding/main/source/qm_select.h +++ b/src/modules/video_coding/main/source/qm_select.h @@ -13,25 +13,19 @@ #include "typedefs.h" #include "common_types.h" -/************************/ -/* Quality Modes */ -/**********************/ +/******************************************************/ +/* Quality Modes: Resolution and Robustness settings */ +/******************************************************/ namespace webrtc { struct VideoContentMetrics; -struct VCMQualityMode +struct VCMResolutionScale { - VCMQualityMode():spatialWidthFact(1), spatialHeightFact(1), + VCMResolutionScale(): spatialWidthFact(1), spatialHeightFact(1), temporalFact(1){} - void Reset() - { - spatialWidthFact = 1; - spatialHeightFact = 1; - temporalFact = 1; - } WebRtc_UWord16 spatialWidthFact; WebRtc_UWord16 spatialHeightFact; @@ -59,53 +53,20 @@ struct VCMContFeature VCMMagValues level; }; -class VCMQmSelect +// QmMethod class: main class for resolution and robustness settings + +class VCMQmMethod { public: - VCMQmSelect(); - ~VCMQmSelect(); + VCMQmMethod(); + ~VCMQmMethod(); - // Initialize: - WebRtc_Word32 Initialize(float bitRate, float userFrameRate, - WebRtc_UWord32 width, WebRtc_UWord32 height); - - // Allow the user to set preferences: favor frame rate/resolution - WebRtc_Word32 SetPreferences(WebRtc_Word8 resolPref); - - // Extract ST (spatio-temporal) QM behavior and make decision - // Inputs: qm: Reference to the quality modes pointer - WebRtc_Word32 SelectQuality(VCMQualityMode** qm); - - // Update QM with actual bit rate - // (size of the latest encoded frame) and frame type. - void UpdateEncodedSize(WebRtc_Word64 encodedSize, - FrameType encodedFrameType); - - // Update QM with new bit/frame/loss rates from SetTargetRates - void UpdateRates(float targetBitRate, float avgSentRate, - float incomingFrameRate, WebRtc_UWord8 packetLoss); - - // Update QM with the content metrics - void UpdateContent(const VideoContentMetrics* contentMetrics); - - // Adjust FEC rate based on content - WebRtc_UWord8 AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate, - float frameRate, WebRtc_UWord16 rttTime, - WebRtc_UWord8 packetLoss); - - - // Select 1x2,2x2,2x2 spatial sampling mode - WebRtc_Word32 SelectSpatialDirectionMode(float transRate); - - // Reset values prior to QMSelect + // Reset values void ResetQM(); + virtual void Reset() = 0; - // Reset rate quantities and counter values after every QMSelect call - void ResetRates(); - - // Reset all - void Reset(); -private: + // Update with the content metrics + void UpdateContent(const VideoContentMetrics* contentMetrics); // Compute spatial texture magnitude and level void Spatial(); @@ -119,36 +80,84 @@ private: // Compute coherence magnitude and level void Coherence(); - // Set the max rate for QM selection - WebRtc_Word32 SetMaxRateForQM(WebRtc_UWord32 width, WebRtc_UWord32 height); + // Get the imageType (CIF, VGA, HD, etc) for the system width/height + WebRtc_Word8 GetImageType(WebRtc_UWord32 width, WebRtc_UWord32 height); // Content Data - const VideoContentMetrics* _contentMetrics; + const VideoContentMetrics* _contentMetrics; - // Encoder rate control parameters, network parameters + // Encoder and native frame sizes, frame rate, aspect ratio, imageType + WebRtc_UWord32 _width; + WebRtc_UWord32 _height; + WebRtc_UWord32 _nativeWidth; + WebRtc_UWord32 _nativeHeight; + WebRtc_UWord32 _nativeFrameRate; + float _aspectRatio; + // Image type for the current encoder system size. + WebRtc_UWord8 _imageType; + + // Content L/M/H values. stationary flag + VCMContFeature _motion; + VCMContFeature _spatial; + VCMContFeature _coherence; + bool _stationaryMotion; + bool _init; + +}; + +// Resolution settings class + +class VCMQmResolution : public VCMQmMethod +{ +public: + VCMQmResolution(); + ~VCMQmResolution(); + + // Reset all quantities + virtual void Reset(); + + // Reset rate quantities and counter values after every Select Quality call + void ResetRates(); + + // Initialize rate control quantities after re-init of encoder. + WebRtc_Word32 Initialize(float bitRate, float userFrameRate, + WebRtc_UWord32 width, WebRtc_UWord32 height); + + // Update QM with actual bit rate (size of the latest encoded frame) + // and frame type, after every encoded frame. + void UpdateEncodedSize(WebRtc_Word64 encodedSize, + FrameType encodedFrameType); + + // Update QM with new bit/frame/loss rates every ~1 sec from SetTargetRates + void UpdateRates(float targetBitRate, float avgSentRate, + float incomingFrameRate, WebRtc_UWord8 packetLoss); + + // Extract ST (spatio-temporal) QM behavior and make decision + // Inputs: qm: Reference to the quality modes pointer + // Output: the spatial and/or temporal scale change + WebRtc_Word32 SelectResolution(VCMResolutionScale** qm); + + // Select 1x2,2x2,2x2 spatial sampling mode + WebRtc_Word32 SelectSpatialDirectionMode(float transRate); + +private: + // Encoder rate control parameter float _targetBitRate; float _userFrameRate; float _incomingFrameRate; float _perFrameBandwidth; float _bufferLevel; + + // Data accumulated every ~1sec from MediaOpt float _sumTargetRate; float _sumIncomingFrameRate; float _sumSeqRateMM; float _sumFrameRateMM; float _sumPacketLoss; - float _prevTotalRate; - WebRtc_UWord16 _prevRttTime; - WebRtc_UWord8 _prevPacketLoss; WebRtc_Word64 _sumEncodedBytes; - // Encoder and native frame sizes - WebRtc_UWord32 _width; - WebRtc_UWord32 _height; - WebRtc_UWord32 _nativeWidth; - WebRtc_UWord32 _nativeHeight; + // Resolution state parameters WebRtc_UWord8 _stateDecFactorSpatial; - - WebRtc_UWord32 _nativeFrameRate; WebRtc_UWord8 _stateDecFactorTemp; // Counters @@ -157,24 +166,36 @@ private: WebRtc_UWord32 _updateRateCnt; WebRtc_UWord32 _lowBufferCnt; - // Content L/M/H values - VCMContFeature _motion; - VCMContFeature _spatial; - VCMContFeature _coherence; - bool _stationaryMotion; + VCMResolutionScale* _qm; +}; - // Aspect ratio - float _aspectRatio; +// Robustness settings class - // Max rate to saturate the transitionalRate - WebRtc_UWord32 _maxRateQM; - WebRtc_UWord8 _imageType; +class VCMQmRobustness : public VCMQmMethod +{ +public: + VCMQmRobustness(); + ~VCMQmRobustness(); - // User preference for resolution or qmax change - WebRtc_UWord8 _userResolutionPref; - bool _init; - VCMQualityMode* _qm; + virtual void Reset(); + // Adjust FEC rate based on content: every ~1 sec from SetTargetRates + WebRtc_UWord8 AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate, + float frameRate, WebRtc_UWord32 rttTime, + WebRtc_UWord8 packetLoss); + + // Set the UEP protection on/off + bool SetUepProtection(WebRtc_UWord8 codeRateDelta, float totalRate, + WebRtc_UWord8 packetLoss, bool frameType); + +private: + // Previous state of network parameters + float _prevTotalRate; + WebRtc_UWord32 _prevRttTime; + WebRtc_UWord8 _prevPacketLoss; + + // Previous FEC rate + WebRtc_UWord8 _prevCodeRateDelta; }; } // namespace webrtc diff --git a/src/modules/video_coding/main/source/qm_select_data.h b/src/modules/video_coding/main/source/qm_select_data.h index 813c1109f8..64870eabed 100644 --- a/src/modules/video_coding/main/source/qm_select_data.h +++ b/src/modules/video_coding/main/source/qm_select_data.h @@ -28,6 +28,9 @@ namespace webrtc // Initial level of buffer in secs: should corresponds to wrapper settings #define INIT_BUFFER_LEVEL 0.5 +// Optimal level of buffer in secs: should corresponds to wrapper settings +#define OPT_BUFFER_LEVEL 0.6 + // Threshold of (max) buffer size below which we consider too low (underflow) #define PERC_BUFFER_THR 0.10