From 883e7163044ea888ec2c3df04ac12c20d8524a95 Mon Sep 17 00:00:00 2001 From: "marpan@webrtc.org" Date: Fri, 17 Feb 2012 18:35:23 +0000 Subject: [PATCH] Removed unused motion vector metrics from VideoContentMetrics; also removed other related unused variables and code. Reset frame rate estimate in mediaOpt when frame rate reduction is decided. Update content_metrics with frame rate and qm_resolution with frame size. Review URL: https://webrtc-codereview.appspot.com/395007 git-svn-id: http://webrtc.googlecode.com/svn/trunk@1718 4adac7df-926f-26a2-2b94-8c16560cd09d --- src/modules/interface/module_common_types.h | 40 ++-- .../main/source/content_metrics_processing.cc | 113 +++--------- .../main/source/content_metrics_processing.h | 6 +- .../main/source/media_optimization.cc | 66 ++++--- .../main/source/media_optimization.h | 2 + .../video_coding/main/source/qm_select.cc | 173 +++--------------- .../video_coding/main/source/qm_select.h | 15 +- .../main/source/content_analysis.cc | 15 +- .../main/source/content_analysis.h | 7 +- .../main/source/frame_preprocessor.cc | 11 +- .../main/source/frame_preprocessor.h | 4 +- 11 files changed, 122 insertions(+), 330 deletions(-) diff --git a/src/modules/interface/module_common_types.h b/src/modules/interface/module_common_types.h index 6d571a4f45..fb20725aa4 100644 --- a/src/modules/interface/module_common_types.h +++ b/src/modules/interface/module_common_types.h @@ -394,29 +394,23 @@ public: }; // Video Content Metrics -struct VideoContentMetrics -{ - VideoContentMetrics(): motionMagnitudeNZ(0), sizeZeroMotion(0), spatialPredErr(0), - spatialPredErrH(0), spatialPredErrV(0), motionPredErr(0), - motionHorizontalness(0), motionClusterDistortion(0), - nativeWidth(0), nativeHeight(0), contentChange(false) { } - void Reset(){ motionMagnitudeNZ = 0; sizeZeroMotion = 0; spatialPredErr = 0; - spatialPredErrH = 0; spatialPredErrV = 0; motionPredErr = 0; - motionHorizontalness = 0; motionClusterDistortion = 0; - nativeWidth = 0; nativeHeight = 0; contentChange = false; } - - float motionMagnitudeNZ; - float sizeZeroMotion; - float spatialPredErr; - float spatialPredErrH; - float spatialPredErrV; - float motionPredErr; - float motionHorizontalness; - float motionClusterDistortion; - WebRtc_UWord32 nativeWidth; - WebRtc_UWord32 nativeHeight; - WebRtc_UWord32 nativeFrameRate; - bool contentChange; +struct VideoContentMetrics { + VideoContentMetrics() + : motionMagnitudeNZ(0.0f), + spatialPredErr(0.0f), + spatialPredErrH(0.0f), + spatialPredErrV(0.0f) { + } + void Reset() { + motionMagnitudeNZ = 0.0f; + spatialPredErr = 0.0f; + spatialPredErrH = 0.0f; + spatialPredErrV = 0.0f; + } + float motionMagnitudeNZ; + float spatialPredErr; + float spatialPredErrH; + float spatialPredErrV; }; /************************************************* diff --git a/src/modules/video_coding/main/source/content_metrics_processing.cc b/src/modules/video_coding/main/source/content_metrics_processing.cc index 0805178c58..02fc190ac2 100644 --- a/src/modules/video_coding/main/source/content_metrics_processing.cc +++ b/src/modules/video_coding/main/source/content_metrics_processing.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -21,9 +21,7 @@ namespace webrtc { ////////////////////////////////// VCMContentMetricsProcessing::VCMContentMetricsProcessing(): -_frameRate(0), _recAvgFactor(1 / 150.0f), // matched to 30fps -_frameCntRecursiveAvg(0), _frameCntUniformAvg(0), _avgMotionLevel(0.0f), _avgSpatialLevel(0.0f) @@ -43,8 +41,6 @@ VCMContentMetricsProcessing::Reset() { _recursiveAvg->Reset(); _uniformAvg->Reset(); - _frameRate = 0; - _frameCntRecursiveAvg = 0; _frameCntUniformAvg = 0; _avgMotionLevel = 0.0f; _avgSpatialLevel = 0.0f; @@ -54,30 +50,20 @@ VCMContentMetricsProcessing::Reset() void VCMContentMetricsProcessing::UpdateFrameRate(WebRtc_UWord32 frameRate) { - _frameRate = frameRate; // Update factor for recursive averaging. - _recAvgFactor = (float) 1000.0f / ((float)(_frameRate * kQmMinIntervalMs)); + _recAvgFactor = (float) 1000.0f / ((float)(frameRate * kQmMinIntervalMs)); } VideoContentMetrics* VCMContentMetricsProcessing::LongTermAvgData() { - if (_frameCntRecursiveAvg == 0) - { - return NULL; - } return _recursiveAvg; } VideoContentMetrics* VCMContentMetricsProcessing::ShortTermAvgData() { - if (_frameCntUniformAvg == 0) - { - return NULL; - } - // Two metrics are used: motion and spatial level. _uniformAvg->motionMagnitudeNZ = _avgMotionLevel / (float)(_frameCntUniformAvg); @@ -97,7 +83,8 @@ VCMContentMetricsProcessing::ResetShortTermAvgData() } WebRtc_Word32 -VCMContentMetricsProcessing::UpdateContentData(const VideoContentMetrics *contentMetrics) +VCMContentMetricsProcessing::UpdateContentData( + const VideoContentMetrics *contentMetrics) { if (contentMetrics == NULL) { @@ -108,7 +95,8 @@ VCMContentMetricsProcessing::UpdateContentData(const VideoContentMetrics *conten } WebRtc_UWord32 -VCMContentMetricsProcessing::ProcessContent(const VideoContentMetrics *contentMetrics) +VCMContentMetricsProcessing::ProcessContent( + const VideoContentMetrics *contentMetrics) { // Update the recursive averaged metrics // average is over longer window of time: over QmMinIntervalMs ms. @@ -122,9 +110,9 @@ VCMContentMetricsProcessing::ProcessContent(const VideoContentMetrics *contentMe } void -VCMContentMetricsProcessing::UpdateUniformAvg(const VideoContentMetrics *contentMetrics) +VCMContentMetricsProcessing::UpdateUniformAvg( + const VideoContentMetrics *contentMetrics) { - // Update frame counter _frameCntUniformAvg += 1; @@ -135,78 +123,27 @@ VCMContentMetricsProcessing::UpdateUniformAvg(const VideoContentMetrics *content return; } -void -VCMContentMetricsProcessing::UpdateRecursiveAvg(const VideoContentMetrics *contentMetrics) -{ +void VCMContentMetricsProcessing::UpdateRecursiveAvg( + const VideoContentMetrics *contentMetrics) { - // Threshold for size of zero motion cluster: - // Use for updating 3 motion vector derived metrics: - // motion magnitude, cluster distortion, and horizontalness. - float nonZeroMvThr = 0.1f; + // Spatial metrics: 2x2, 1x2(H), 2x1(V). + _recursiveAvg->spatialPredErr = (1 - _recAvgFactor) * + _recursiveAvg->spatialPredErr + + _recAvgFactor * contentMetrics->spatialPredErr; - float tmpRecAvgFactor = _recAvgFactor; + _recursiveAvg->spatialPredErrH = (1 - _recAvgFactor) * + _recursiveAvg->spatialPredErrH + + _recAvgFactor * contentMetrics->spatialPredErrH; - // Take value as is for first frame (no motion search in frame zero). - if (_frameCntRecursiveAvg < 1) - { - tmpRecAvgFactor = 1; - } + _recursiveAvg->spatialPredErrV = (1 - _recAvgFactor) * + _recursiveAvg->spatialPredErrV + + _recAvgFactor * contentMetrics->spatialPredErrV; - _recursiveAvg->motionPredErr = (1 - tmpRecAvgFactor) * - _recursiveAvg->motionPredErr + - tmpRecAvgFactor * contentMetrics->motionPredErr; + // Motion metric: Derived from NFD (normalized frame difference) + _recursiveAvg->motionMagnitudeNZ = (1 - _recAvgFactor) * + _recursiveAvg->motionMagnitudeNZ + + _recAvgFactor * contentMetrics->motionMagnitudeNZ; - _recursiveAvg->sizeZeroMotion = (1 - tmpRecAvgFactor) * - _recursiveAvg->sizeZeroMotion + - tmpRecAvgFactor * contentMetrics->sizeZeroMotion; - - _recursiveAvg->spatialPredErr = (1 - tmpRecAvgFactor) * - _recursiveAvg->spatialPredErr + - tmpRecAvgFactor * contentMetrics->spatialPredErr; - - _recursiveAvg->spatialPredErrH = (1 - tmpRecAvgFactor) * - _recursiveAvg->spatialPredErrH + - tmpRecAvgFactor * contentMetrics->spatialPredErrH; - - _recursiveAvg->spatialPredErrV = (1 - tmpRecAvgFactor) * - _recursiveAvg->spatialPredErrV + - tmpRecAvgFactor * contentMetrics->spatialPredErrV; - - // motionMag metric is derived from NFD (normalized frame difference). - if (kNfdMetric == 1) - { - _recursiveAvg->motionMagnitudeNZ = (1 - tmpRecAvgFactor) * - _recursiveAvg->motionMagnitudeNZ + - tmpRecAvgFactor * contentMetrics->motionMagnitudeNZ; - } - - if (contentMetrics->sizeZeroMotion > nonZeroMvThr) - { - _recursiveAvg->motionClusterDistortion = (1 - tmpRecAvgFactor) * - _recursiveAvg->motionClusterDistortion + - tmpRecAvgFactor *contentMetrics->motionClusterDistortion; - - _recursiveAvg->motionHorizontalness = (1 - _recAvgFactor) * - _recursiveAvg->motionHorizontalness + - tmpRecAvgFactor * contentMetrics->motionHorizontalness; - - // motionMag metric is derived from motion vectors. - if (kNfdMetric == 0) - { - _recursiveAvg->motionMagnitudeNZ = (1 - tmpRecAvgFactor) * - _recursiveAvg->motionMagnitudeNZ + - tmpRecAvgFactor * contentMetrics->motionMagnitudeNZ; - } - } - - // Update native values: - // TODO (marpan): we don't need to update this every frame. - _recursiveAvg->nativeHeight = contentMetrics->nativeHeight; - _recursiveAvg->nativeWidth = contentMetrics->nativeWidth; - _recursiveAvg->nativeFrameRate = contentMetrics->nativeFrameRate; - - _frameCntRecursiveAvg++; - - return; + return; } } //end of namespace diff --git a/src/modules/video_coding/main/source/content_metrics_processing.h b/src/modules/video_coding/main/source/content_metrics_processing.h index 155c4adc61..72666fd9ba 100644 --- a/src/modules/video_coding/main/source/content_metrics_processing.h +++ b/src/modules/video_coding/main/source/content_metrics_processing.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -60,14 +60,12 @@ private: // Update the recursive averaged metrics: longer time average (~5/10 secs). void UpdateRecursiveAvg(const VideoContentMetrics *contentMetrics); - // Update the uniform averaged metrics: shorter time average (~RTCP reports). + // Update the uniform averaged metrics: shorter time average (~RTCP report). void UpdateUniformAvg(const VideoContentMetrics *contentMetrics); VideoContentMetrics* _recursiveAvg; VideoContentMetrics* _uniformAvg; - WebRtc_UWord32 _frameRate; float _recAvgFactor; - WebRtc_UWord32 _frameCntRecursiveAvg; WebRtc_UWord32 _frameCntUniformAvg; float _avgMotionLevel; float _avgSpatialLevel; diff --git a/src/modules/video_coding/main/source/media_optimization.cc b/src/modules/video_coding/main/source/media_optimization.cc index e2c4329803..334385fc14 100644 --- a/src/modules/video_coding/main/source/media_optimization.cc +++ b/src/modules/video_coding/main/source/media_optimization.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -24,6 +24,8 @@ _maxBitRate(0), _sendCodecType(kVideoCodecUnknown), _codecWidth(0), _codecHeight(0), +_initCodecWidth(0), +_initCodecHeight(0), _userFrameRate(0), _packetLossEnc(0), _fractionLost(0), @@ -64,7 +66,7 @@ WebRtc_Word32 VCMMediaOptimization::Reset() { memset(_incomingFrameTimes, -1, sizeof(_incomingFrameTimes)); - InputFrameRate(); // Resets _incomingFrameRate + _incomingFrameRate = 0.0; _frameDropper->Reset(); _lossProtLogic->Reset(_clock->MillisecondTimestamp()); _frameDropper->SetRates(0, 0); @@ -282,6 +284,8 @@ VCMMediaOptimization::SetEncodingData(VideoCodecType sendCodecType, _userFrameRate = static_cast(frameRate); _codecWidth = width; _codecHeight = height; + _initCodecWidth = width; + _initCodecHeight = height; _numLayers = (numLayers <= 1) ? 1 : numLayers; // Can also be zero. WebRtc_Word32 ret = VCM_OK; ret = _qmResolution->Initialize((float)_targetBitRate, _userFrameRate, @@ -575,43 +579,50 @@ VCMMediaOptimization::QMUpdate(VCMResolutionScale* qm) // Check for no change if (qm->spatialHeightFact == 1 && qm->spatialWidthFact == 1 && - qm->temporalFact == 1) - { + qm->temporalFact == 1) { return false; } - // Content metrics hold native values - VideoContentMetrics* cm = _content->LongTermAvgData(); - // Temporal WebRtc_UWord32 frameRate = static_cast (_incomingFrameRate + 0.5f); // Check if go back up in temporal resolution - if (qm->temporalFact == 0) - { - frameRate = (WebRtc_UWord32) 2 * _incomingFrameRate; + if (qm->temporalFact == 0) { + // Currently only allow for 1/2 frame rate reduction per action. + // TODO (marpan): allow for 2/3 reduction. + frameRate = (WebRtc_UWord32) 2 * _incomingFrameRate; } // go down in temporal resolution - else - { - frameRate = (WebRtc_UWord32)(_incomingFrameRate / qm->temporalFact + 1); + else { + frameRate = (WebRtc_UWord32)(_incomingFrameRate / qm->temporalFact + 1); + } + // Reset _incomingFrameRate if temporal action was selected. + if (qm->temporalFact != 1) { + memset(_incomingFrameTimes, -1, sizeof(_incomingFrameTimes)); + _incomingFrameRate = frameRate; } // Spatial WebRtc_UWord32 height = _codecHeight; WebRtc_UWord32 width = _codecWidth; - // Check if go back up in spatial resolution - if (qm->spatialHeightFact == 0 && qm->spatialWidthFact == 0) - { - height = cm->nativeHeight; - width = cm->nativeWidth; - } - else - { - height = _codecHeight / qm->spatialHeightFact; - width = _codecWidth / qm->spatialWidthFact; + // Check if go back up in spatial resolution, and update frame sizes. + // Currently only allow for 2x2 spatial down-sampling. + // TODO (marpan): allow for 1x2, 2x1, and 4/3x4/3 (or 3/2x3/2). + if (qm->spatialHeightFact == 0 && qm->spatialWidthFact == 0) { + width = _codecWidth * 2; + height = _codecHeight * 2; + } else { + width = _codecWidth / qm->spatialWidthFact; + height = _codecHeight / qm->spatialHeightFact; } + _codecWidth = width; + _codecHeight = height; + + // New frame sizes should never exceed the original sizes + // from SetEncodingData(). + assert(_codecWidth <= _initCodecWidth); + assert(_codecHeight <= _initCodecHeight); WEBRTC_TRACE(webrtc::kTraceDebug, webrtc::kTraceVideoCoding, _id, "Quality Mode Update: W = %d, H = %d, FR = %f", @@ -620,11 +631,12 @@ VCMMediaOptimization::QMUpdate(VCMResolutionScale* qm) // Update VPM with new target frame rate and size _videoQMSettingsCallback->SetVideoQMSettings(frameRate, width, height); + _content->UpdateFrameRate(frameRate); + _qmResolution->UpdateCodecFrameSize(width, height); + return true; } - - void VCMMediaOptimization::UpdateIncomingFrameRate() { @@ -671,10 +683,6 @@ VCMMediaOptimization::ProcessIncomingFrameRate(WebRtc_Word64 now) _incomingFrameRate = nrOfFrames * 1000.0f / static_cast(diff); } } - else - { - _incomingFrameRate = static_cast(nrOfFrames); - } } WebRtc_UWord32 diff --git a/src/modules/video_coding/main/source/media_optimization.h b/src/modules/video_coding/main/source/media_optimization.h index 7d87a6d041..14e5d1a2d7 100644 --- a/src/modules/video_coding/main/source/media_optimization.h +++ b/src/modules/video_coding/main/source/media_optimization.h @@ -168,6 +168,8 @@ private: VideoCodecType _sendCodecType; WebRtc_UWord16 _codecWidth; WebRtc_UWord16 _codecHeight; + WebRtc_UWord16 _initCodecWidth; + WebRtc_UWord16 _initCodecHeight; float _userFrameRate; VCMFrameDropper* _frameDropper; diff --git a/src/modules/video_coding/main/source/qm_select.cc b/src/modules/video_coding/main/source/qm_select.cc index 99fc89d1b1..a03fc358a4 100644 --- a/src/modules/video_coding/main/source/qm_select.cc +++ b/src/modules/video_coding/main/source/qm_select.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -28,7 +28,6 @@ VCMQmMethod::VCMQmMethod() _height(0), _nativeWidth(0), _nativeHeight(0), - _nativeFrameRate(0), _init(false) { ResetQM(); } @@ -43,8 +42,6 @@ VCMQmMethod::ResetQM() { _motion.Reset(); _spatial.Reset(); - _coherence.Reset(); - _stationaryMotion = 0; _aspectRatio = 1; _imageType = 2; return; @@ -77,40 +74,6 @@ VCMQmMethod::MotionNFD() } -void -VCMQmMethod::Motion() -{ - - float sizeZeroMotion = _contentMetrics->sizeZeroMotion; - float motionMagNZ = _contentMetrics->motionMagnitudeNZ; - - // Take product of size and magnitude with equal weight - _motion.value = (1.0f - sizeZeroMotion) * motionMagNZ; - - // Stabilize: motionMagNZ could be large when only a - // few motion blocks are non-zero - _stationaryMotion = false; - if (sizeZeroMotion > HIGH_ZERO_MOTION_SIZE) - { - _motion.value = 0.0f; - _stationaryMotion = true; - } - // Determine motion level - if (_motion.value < LOW_MOTION) - { - _motion.level = kLow; - } - else if (_motion.value > HIGH_MOTION) - { - _motion.level = kHigh; - } - else - { - _motion.level = kDefault; - } -} - - void VCMQmMethod::Spatial() { @@ -141,31 +104,6 @@ VCMQmMethod::Spatial() } } -void -VCMQmMethod::Coherence() -{ - float horizNZ = _contentMetrics->motionHorizontalness; - float distortionNZ = _contentMetrics->motionClusterDistortion; - - // Coherence measure: combine horizontalness with cluster distortion - _coherence.value = COH_MAX; - if (distortionNZ > 0.) - { - _coherence.value = horizNZ / distortionNZ; - } - _coherence.value = VCM_MIN(COH_MAX, _coherence.value); - - if (_coherence.value < COHERENCE_THR) - { - _coherence.level = kLow; - } - else - { - _coherence.level = kHigh; - } - -} - WebRtc_Word8 VCMQmMethod::GetImageType(WebRtc_UWord32 width, WebRtc_UWord32 height) { @@ -227,7 +165,6 @@ VCMQmResolution::ResetRates() _sumEncodedBytes = 0; _sumTargetRate = 0.0f; _sumIncomingFrameRate = 0.0f; - _sumFrameRateMM = 0.0f; _sumSeqRateMM = 0.0f; _sumPacketLoss = 0.0f; _frameCnt = 0; @@ -253,48 +190,32 @@ VCMQmResolution::Reset() } // Initialize rate control quantities after reset of encoder -WebRtc_Word32 -VCMQmResolution::Initialize(float bitRate, float userFrameRate, - WebRtc_UWord32 width, WebRtc_UWord32 height) -{ - if (userFrameRate == 0.0f || width == 0 || height == 0) - { - return VCM_PARAMETER_ERROR; - } - _targetBitRate = bitRate; - _userFrameRate = userFrameRate; +WebRtc_Word32 VCMQmResolution::Initialize(float bitRate, + float userFrameRate, + WebRtc_UWord32 width, + WebRtc_UWord32 height) { + if (userFrameRate == 0.0f || width == 0 || height == 0) { + return VCM_PARAMETER_ERROR; + } + _targetBitRate = bitRate; + _userFrameRate = userFrameRate; + // Native width and height. + _nativeWidth = width; + _nativeHeight = height; + UpdateCodecFrameSize(width, height); + // Initial buffer level. + _bufferLevel = INIT_BUFFER_LEVEL * _targetBitRate; + // Per-frame bandwidth. + _perFrameBandwidth = _targetBitRate / _userFrameRate; + _init = true; + return VCM_OK; +} - // Encoder width and height - _width = width; - _height = height; - - // Aspect ratio: used for selection of 1x2,2x1,2x2 - _aspectRatio = static_cast(_width) / static_cast(_height); - - // Set the imageType for the encoder width/height. - _imageType = GetImageType(_width, _height); - - // Initial buffer level - _bufferLevel = INIT_BUFFER_LEVEL * _targetBitRate; - - // Per-frame bandwidth - if ( _incomingFrameRate == 0 ) - { - _perFrameBandwidth = _targetBitRate / _userFrameRate; - _incomingFrameRate = _userFrameRate; - } - else - { - // Take average: this is due to delay in update of new encoder frame rate: - // userFrameRate is the new one, - // incomingFrameRate is the old one (based on previous ~ 1sec/RTCP report) - _perFrameBandwidth = 0.5 *( _targetBitRate / _userFrameRate + - _targetBitRate / _incomingFrameRate ); - } - _init = true; - - - return VCM_OK; +void VCMQmResolution::UpdateCodecFrameSize(uint32_t width, uint32_t height) { + _width = width; + _height = height; + // Set the imageType for the encoder width/height. + _imageType = GetImageType(_width, _height); } // Update after every encoded frame @@ -313,34 +234,15 @@ VCMQmResolution::UpdateEncodedSize(WebRtc_Word64 encodedSize, // per_frame_BW is updated when encoder is updated, every RTCP reports _bufferLevel += _perFrameBandwidth - encodedSizeKbits; - // Mismatch here is based on difference of actual encoded frame size and - // per-frame bandwidth, for delta frames - // This is a much stronger condition on rate mismatch than sumSeqRateMM - // Note: not used in this version - /* - const bool deltaFrame = (encodedFrameType != kVideoFrameKey && - encodedFrameType != kVideoFrameGolden); - - // Sum the frame mismatch: - if (deltaFrame) - { - _frameCntDelta++; - if (encodedSizeKbits > 0) - _sumFrameRateMM += - (float) (fabs(encodedSizeKbits - _perFrameBandwidth) / - encodedSizeKbits); - } - */ - // Counter for occurrences of low buffer level if (_bufferLevel <= PERC_BUFFER_THR * OPT_BUFFER_LEVEL * _targetBitRate) { _lowBufferCnt++; } - } // Update various quantities after SetTargetRates in MediaOpt +// TODO (marpan): use sent_video_rate_bps from mediaOPt for avgSentBitRate. void VCMQmResolution::UpdateRates(float targetBitRate, float avgSentBitRate, float incomingFrameRate, WebRtc_UWord8 packetLoss) @@ -382,11 +284,11 @@ VCMQmResolution::UpdateRates(float targetBitRate, float avgSentBitRate, { _perFrameBandwidth = _targetBitRate / _incomingFrameRate; } - } // Select the resolution factors: frame size and frame rate change: (QM modes) // Selection is for going back up in resolution, or going down in. +// TODO (marpan): break up into small routines/clean-up and update. WebRtc_Word32 VCMQmResolution::SelectResolution(VCMResolutionScale** qm) { @@ -406,11 +308,6 @@ VCMQmResolution::SelectResolution(VCMResolutionScale** qm) _qm->spatialHeightFact = 1; _qm->temporalFact = 1; - // Update native values - _nativeWidth = _contentMetrics->nativeWidth; - _nativeHeight = _contentMetrics->nativeHeight; - _nativeFrameRate = _contentMetrics->nativeFrameRate; - float avgTargetRate = 0.0f; float avgIncomingFrameRate = 0.0f; float ratioBufferLow = 0.0f; @@ -424,8 +321,6 @@ VCMQmResolution::SelectResolution(VCMResolutionScale** qm) { // Use seq-rate mismatch for now rateMisMatch = (float)_sumSeqRateMM / (float)_updateRateCnt; - //rateMisMatch = (float)_sumFrameRateMM / (float)_frameCntDelta; - // Average target and incoming frame rates avgTargetRate = (float)_sumTargetRate / (float)_updateRateCnt; avgIncomingFrameRate = (float)_sumIncomingFrameRate / @@ -504,7 +399,7 @@ VCMQmResolution::SelectResolution(VCMResolutionScale** qm) if (_stateDecFactorSpatial > 1) { // Check conditions on buffer level and rate_mismatch - if ( (avgTargetRate > estimatedTransRateUpS) && + if ((avgTargetRate > estimatedTransRateUpS) && (ratioBufferLow < MAX_BUFFER_LOW) && (rateMisMatch < MAX_RATE_MM)) { // width/height scaled back up: @@ -564,7 +459,6 @@ VCMQmResolution::SelectResolution(VCMResolutionScale** qm) { estimatedTransRateDown = LOSS_RATE_FAC * estimatedTransRateDown; } - if ((avgTargetRate < estimatedTransRateDown ) || (ratioBufferLow > MAX_BUFFER_LOW) || (rateMisMatch > MAX_RATE_MM)) @@ -657,14 +551,13 @@ VCMQmResolution::SelectResolution(VCMResolutionScale** qm) *qm = _qm; return VCM_OK; - - } WebRtc_Word32 VCMQmResolution::SelectSpatialDirectionMode(float transRate) { // Default is 1x2 (H) + _aspectRatio = static_cast(_width) / static_cast(_height); // For bit rates well below transitional rate, we select 2x2 if ( _targetBitRate < transRate * RATE_RED_SPATIAL_2X2 ) @@ -675,7 +568,6 @@ VCMQmResolution::SelectSpatialDirectionMode(float transRate) } // Otherwise check prediction errors, aspect ratio, horizontalness - float spatialErr = _contentMetrics->spatialPredErr; float spatialErrH = _contentMetrics->spatialPredErrH; float spatialErrV = _contentMetrics->spatialPredErrV; @@ -707,7 +599,6 @@ VCMQmResolution::SelectSpatialDirectionMode(float transRate) _qm->spatialHeightFact = 2; return VCM_OK; } - return VCM_OK; } @@ -764,11 +655,9 @@ VCMQmRobustness::AdjustFecFactor(WebRtc_UWord8 codeRateDelta, float totalRate, _prevTotalRate = totalRate; _prevRttTime = rttTime; _prevPacketLoss = packetLoss; - _prevCodeRateDelta = codeRateDelta; return adjustFec; - } // Set the UEP (unequal-protection) on/off for the FEC @@ -784,8 +673,6 @@ VCMQmRobustness::SetUepProtection(WebRtc_UWord8 codeRateDelta, float totalRate, return uepProtection; } - return uepProtection; } - } // end of namespace diff --git a/src/modules/video_coding/main/source/qm_select.h b/src/modules/video_coding/main/source/qm_select.h index 3fb9040361..761d40af3d 100644 --- a/src/modules/video_coding/main/source/qm_select.h +++ b/src/modules/video_coding/main/source/qm_select.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -71,15 +71,9 @@ public: // Compute spatial texture magnitude and level void Spatial(); - // Compute motion magnitude and level - void Motion(); - // Compute motion magnitude and level for NFD metric void MotionNFD(); - // Compute coherence magnitude and level - void Coherence(); - // Get the imageType (CIF, VGA, HD, etc) for the system width/height WebRtc_Word8 GetImageType(WebRtc_UWord32 width, WebRtc_UWord32 height); @@ -91,7 +85,6 @@ public: WebRtc_UWord32 _height; WebRtc_UWord32 _nativeWidth; WebRtc_UWord32 _nativeHeight; - WebRtc_UWord32 _nativeFrameRate; float _aspectRatio; // Image type for the current encoder system size. WebRtc_UWord8 _imageType; @@ -99,8 +92,6 @@ public: // Content L/M/H values. stationary flag VCMContFeature _motion; VCMContFeature _spatial; - VCMContFeature _coherence; - bool _stationaryMotion; bool _init; }; @@ -123,6 +114,9 @@ public: WebRtc_Word32 Initialize(float bitRate, float userFrameRate, WebRtc_UWord32 width, WebRtc_UWord32 height); + // Update the encoder frame size. + void UpdateCodecFrameSize(uint32_t width, uint32_t height); + // Update QM with actual bit rate (size of the latest encoded frame) // and frame type, after every encoded frame. void UpdateEncodedSize(WebRtc_Word64 encodedSize, @@ -152,7 +146,6 @@ private: float _sumTargetRate; float _sumIncomingFrameRate; float _sumSeqRateMM; - float _sumFrameRateMM; float _sumPacketLoss; WebRtc_Word64 _sumEncodedBytes; diff --git a/src/modules/video_processing/main/source/content_analysis.cc b/src/modules/video_processing/main/source/content_analysis.cc index 32ee09a18f..767dbbf7d0 100644 --- a/src/modules/video_processing/main/source/content_analysis.cc +++ b/src/modules/video_processing/main/source/content_analysis.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -27,10 +27,6 @@ _motionMagnitudeNZ(0.0f), _spatialPredErr(0.0f), _spatialPredErrH(0.0f), _spatialPredErrV(0.0f), -_sizeZeroMotion(0.0f), -_motionPredErr(0.0f), -_motionHorizontalness(0.0f), -_motionClusterDistortion(0.0f), _firstFrame(true), _CAInit(false), _cMetrics(NULL) @@ -329,19 +325,12 @@ VPMContentAnalysis::ContentMetrics() return NULL; } - _cMetrics->spatialPredErr = _spatialPredErr; _cMetrics->spatialPredErrH = _spatialPredErrH; _cMetrics->spatialPredErrV = _spatialPredErrV; - // normalized temporal difference (MAD) + // Motion metric: normalized temporal difference (MAD) _cMetrics->motionMagnitudeNZ = _motionMagnitudeNZ; - // Set to zero: not computed - _cMetrics->motionPredErr = _motionPredErr; - _cMetrics->sizeZeroMotion = _sizeZeroMotion; - _cMetrics->motionHorizontalness = _motionHorizontalness; - _cMetrics->motionClusterDistortion = _motionClusterDistortion; - return _cMetrics; } diff --git a/src/modules/video_processing/main/source/content_analysis.h b/src/modules/video_processing/main/source/content_analysis.h index 5051650291..a279edcf86 100644 --- a/src/modules/video_processing/main/source/content_analysis.h +++ b/src/modules/video_processing/main/source/content_analysis.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -78,11 +78,6 @@ private: float _spatialPredErr; // spatial class float _spatialPredErrH; // spatial class float _spatialPredErrV; // spatial class - float _sizeZeroMotion; // motion class - float _motionPredErr; // complexity class: - float _motionHorizontalness; // coherence class - float _motionClusterDistortion; // coherence class - bool _firstFrame; bool _CAInit; diff --git a/src/modules/video_processing/main/source/frame_preprocessor.cc b/src/modules/video_processing/main/source/frame_preprocessor.cc index fa13b2df7e..76fdac8384 100644 --- a/src/modules/video_processing/main/source/frame_preprocessor.cc +++ b/src/modules/video_processing/main/source/frame_preprocessor.cc @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -16,8 +16,6 @@ namespace webrtc { VPMFramePreprocessor::VPMFramePreprocessor(): _id(0), _contentMetrics(NULL), -_nativeHeight(0), -_nativeWidth(0), _maxFrameRate(0), _resampledFrame(), _enableCA(false) @@ -46,8 +44,6 @@ VPMFramePreprocessor::ChangeUniqueId(const WebRtc_Word32 id) void VPMFramePreprocessor::Reset() { - _nativeWidth = 0; - _nativeHeight = 0; _ca->Release(); _vd->Reset(); _contentMetrics = NULL; @@ -172,11 +168,6 @@ VPMFramePreprocessor::PreprocessFrame(const VideoFrame* frame, VideoFrame** proc } else { _contentMetrics = _ca->ComputeContentMetrics(&_resampledFrame); } - // Update native values: - _contentMetrics->nativeHeight = frame->Height(); - _contentMetrics->nativeWidth = frame->Width(); - // Max value as set by user - _contentMetrics->nativeFrameRate = _maxFrameRate; } return VPM_OK; } diff --git a/src/modules/video_processing/main/source/frame_preprocessor.h b/src/modules/video_processing/main/source/frame_preprocessor.h index cb5f97d4bb..3c07a47e54 100644 --- a/src/modules/video_processing/main/source/frame_preprocessor.h +++ b/src/modules/video_processing/main/source/frame_preprocessor.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source @@ -66,8 +66,6 @@ private: WebRtc_Word32 _id; VideoContentMetrics* _contentMetrics; - WebRtc_UWord32 _nativeHeight; - WebRtc_UWord32 _nativeWidth; WebRtc_UWord32 _maxFrameRate; VideoFrame _resampledFrame; VPMSpatialResampler* _spatialResampler;