diff --git a/src/modules/rtp_rtcp/interface/rtp_rtcp.h b/src/modules/rtp_rtcp/interface/rtp_rtcp.h index b460a03b2e..181706db24 100644 --- a/src/modules/rtp_rtcp/interface/rtp_rtcp.h +++ b/src/modules/rtp_rtcp/interface/rtp_rtcp.h @@ -948,6 +948,7 @@ public: WebRtc_UWord8& payloadTypeRED, WebRtc_UWord8& payloadTypeFEC) = 0; + /* * Set FEC code rate of key and delta frames * codeRate on a scale of 0 to 255 where 255 is 100% added packets, hence protect up to 50% packet loss @@ -957,6 +958,24 @@ public: virtual WebRtc_Word32 SetFECCodeRate(const WebRtc_UWord8 keyFrameCodeRate, const WebRtc_UWord8 deltaFrameCodeRate) = 0; + + /* + * Set FEC unequal protection (UEP) across packets, + * for key and delta frames. + * + * If keyUseUepProtection is true UEP is enabled for key frames. + * If deltaUseUepProtection is true UEP is enabled for delta frames. + * + * UEP skews the FEC protection towards being spent more on the + * important packets, at the cost of less FEC protection for the + * non-important packets. + * + * return -1 on failure else 0 + */ + virtual WebRtc_Word32 SetFECUepProtection(const bool keyUseUepProtection, + const bool deltaUseUepProtection) = 0; + + /* * Set method for requestion a new key frame * diff --git a/src/modules/rtp_rtcp/source/forward_error_correction.cc b/src/modules/rtp_rtcp/source/forward_error_correction.cc index 4cef2eda99..c6cbf20f7c 100644 --- a/src/modules/rtp_rtcp/source/forward_error_correction.cc +++ b/src/modules/rtp_rtcp/source/forward_error_correction.cc @@ -94,6 +94,7 @@ WebRtc_Word32 ForwardErrorCorrection::GenerateFEC(const ListWrapper& mediaPacketList, WebRtc_UWord8 protectionFactor, WebRtc_UWord32 numImportantPackets, + const bool useUnequalProtection, ListWrapper& fecPacketList) { if (mediaPacketList.Empty()) @@ -201,7 +202,7 @@ ForwardErrorCorrection::GenerateFEC(const ListWrapper& mediaPacketList, WebRtc_UWord8* packetMask = new WebRtc_UWord8[numFecPackets * numMaskBytes]; memset(packetMask, 0, numFecPackets * numMaskBytes); internal::GeneratePacketMasks(numMediaPackets, numFecPackets, - numImportantPackets, packetMask); + numImportantPackets, useUnequalProtection, packetMask); // -- Generate FEC bit strings -- WebRtc_UWord8 mediaPayloadLength[2]; diff --git a/src/modules/rtp_rtcp/source/forward_error_correction.h b/src/modules/rtp_rtcp/source/forward_error_correction.h index 275643b4a7..f05f32f54e 100644 --- a/src/modules/rtp_rtcp/source/forward_error_correction.h +++ b/src/modules/rtp_rtcp/source/forward_error_correction.h @@ -18,7 +18,10 @@ namespace webrtc { /** - * Performs codec-independent forward error correction. + * Performs codec-independent forward error correction (FEC), based on RFC 5109. + * Option exists to enable unequal protection (UEP) across packets. + * This is not to be confused with protection within packets + * (referred to as uneven level protection (ULP) in RFC 5109). */ class ForwardErrorCorrection { @@ -107,6 +110,10 @@ public: * be located at the start of the media packet list. * For codecs with data partitioning, the important * packets may correspond to first partition packets. + * \param[in] useUnequalProtection Parameter to enable/disable unequal protection + * (UEP) across packets. Enabling UEP will allocate more + * protection to the numImportantPackets from + * the start of the mediaPacketList. * \param[out] fecPacketList List of FEC packets, of type #Packet. Must be empty * on entry. The memory available through the list * will be valid until the next call to GenerateFEC(). @@ -116,6 +123,7 @@ public: WebRtc_Word32 GenerateFEC(const ListWrapper& mediaPacketList, WebRtc_UWord8 protectionFactor, WebRtc_UWord32 numImportantPackets, + const bool useUnequalProtection, ListWrapper& fecPacketList); /** diff --git a/src/modules/rtp_rtcp/source/forward_error_correction_internal.cc b/src/modules/rtp_rtcp/source/forward_error_correction_internal.cc index 1d2e6a29ea..6349ec262f 100644 --- a/src/modules/rtp_rtcp/source/forward_error_correction_internal.cc +++ b/src/modules/rtp_rtcp/source/forward_error_correction_internal.cc @@ -16,13 +16,6 @@ namespace { -// This parameter enables/disables unequal protection (UEP) across packets. -// This is not to be confused with protection within packets (referred to as ULP). -// One use case of UEP across packets is for codecs with data partitioning, -// e.g., VP8, H264 XP profile, where important packets would be first partition. -// TODO (marpan): Pass this parameter from MediaOpt (VCM). -const bool kUseUnequalProtection = true; - // Allow for two different modes of protection for residual packets. // The residual packets are the remaining packets beyond the important ones. enum ResidualProtectionMode @@ -299,6 +292,7 @@ void UnequalProtectionMask(const WebRtc_UWord16 numMediaPackets, void GeneratePacketMasks(const WebRtc_UWord32 numMediaPackets, const WebRtc_UWord32 numFecPackets, const WebRtc_UWord32 numImpPackets, + const bool useUnequalProtection, WebRtc_UWord8* packetMask) { assert(numMediaPackets <= sizeof(packetMaskTbl)/sizeof(*packetMaskTbl) && @@ -317,7 +311,7 @@ void GeneratePacketMasks(const WebRtc_UWord32 numMediaPackets, // Equal protection is also used for: (numImpPackets == 1 && numFecPackets == 1). // UEP=off would generally be more efficient than the UEP=on for this case. // TODO (marpan): check/test this condition. - if (!kUseUnequalProtection || numImpPackets == 0 || + if (!useUnequalProtection || numImpPackets == 0 || (numImpPackets == 1 && numFecPackets == 1)) { // Retrieve corresponding mask table directly: for equal-protection case. diff --git a/src/modules/rtp_rtcp/source/forward_error_correction_internal.h b/src/modules/rtp_rtcp/source/forward_error_correction_internal.h index 6cda8c111a..242704eb04 100644 --- a/src/modules/rtp_rtcp/source/forward_error_correction_internal.h +++ b/src/modules/rtp_rtcp/source/forward_error_correction_internal.h @@ -24,19 +24,24 @@ namespace internal { * corresponds to a number of mask bytes. The mask indicates which * media packets should be protected by the FEC packet. - * \param[in] numMediaPackets The number of media packets to protect. - * [1, maxMediaPackets]. - * \param[in] numFecPackets The number of FEC packets which will be generated. - * [1, numMediaPackets]. - * \param[in] numImpPackets The number of important packets. - * [0, numMediaPackets]. - * numImpPackets = 0 is the equal protection scenario. - * \param[out] packetMask A pointer to hold the packet mask array, of size - * numFecPackets * "number of mask bytes". + * \param[in] numMediaPackets The number of media packets to protect. + * [1, maxMediaPackets]. + * \param[in] numFecPackets The number of FEC packets which will + * be generated. [1, numMediaPackets]. + * \param[in] numImpPackets The number of important packets. + * [0, numMediaPackets]. + * numImpPackets = 0 is the equal + * protection scenario. + * \param[in] useUnequalProtection Enables unequal protection: allocates + * more protection to the numImpPackets. + * \param[out] packetMask A pointer to hold the packet mask array, + * of size: + * numFecPackets * "number of mask bytes". */ void GeneratePacketMasks(const WebRtc_UWord32 numMediaPackets, const WebRtc_UWord32 numFecPackets, const WebRtc_UWord32 numImpPackets, + const bool useUnequalProtection, WebRtc_UWord8* packetMask); diff --git a/src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc b/src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc index ae55a05157..78724da8fc 100644 --- a/src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc +++ b/src/modules/rtp_rtcp/source/rtp_rtcp_impl.cc @@ -1989,6 +1989,40 @@ ModuleRtpRtcpImpl::SetFECCodeRate(const WebRtc_UWord8 keyFrameCodeRate, } } +WebRtc_Word32 +ModuleRtpRtcpImpl::SetFECUepProtection(const bool keyUseUepProtection, + const bool deltaUseUepProtection) +{ + WEBRTC_TRACE(kTraceModuleCall, kTraceRtpRtcp, _id, + "SetFECUepProtection(%d, %d)", keyUseUepProtection, + deltaUseUepProtection); + + const bool defaultInstance(_childModules.Empty()?false:true); + if (defaultInstance) + { + // for default we need to update all child modules too + CriticalSectionScoped lock(_criticalSectionModulePtrs); + + ListItem* item = _childModules.First(); + while (item) + { + RtpRtcp* module = (RtpRtcp*)item->GetItem(); + if (module) + { + module->SetFECUepProtection(keyUseUepProtection, + deltaUseUepProtection); + } + item = _childModules.Next(item); + } + return 0; + + } else + { + return _rtpSender.SetFECUepProtection(keyUseUepProtection, + deltaUseUepProtection); + } +} + /* * Implementation of ModuleRtpRtcpPrivate */ diff --git a/src/modules/rtp_rtcp/source/rtp_rtcp_impl.h b/src/modules/rtp_rtcp/source/rtp_rtcp_impl.h index 3269db2273..0d42167506 100644 --- a/src/modules/rtp_rtcp/source/rtp_rtcp_impl.h +++ b/src/modules/rtp_rtcp/source/rtp_rtcp_impl.h @@ -446,7 +446,10 @@ public: virtual WebRtc_Word32 SetFECCodeRate(const WebRtc_UWord8 keyFrameCodeRate, - const WebRtc_UWord8 deltaFrameCodeRate); + const WebRtc_UWord8 deltaFrameCodeRate); + + virtual WebRtc_Word32 SetFECUepProtection(const bool keyUseUepProtection, + const bool deltaUseUepProtection); virtual WebRtc_Word32 SetH263InverseLogic(const bool enable); diff --git a/src/modules/rtp_rtcp/source/rtp_sender.cc b/src/modules/rtp_rtcp/source/rtp_sender.cc index 74767816de..08d445e1d0 100644 --- a/src/modules/rtp_rtcp/source/rtp_sender.cc +++ b/src/modules/rtp_rtcp/source/rtp_sender.cc @@ -1546,4 +1546,17 @@ RTPSender::SetFECCodeRate(const WebRtc_UWord8 keyFrameCodeRate, } return _video->SetFECCodeRate(keyFrameCodeRate, deltaFrameCodeRate); } + +WebRtc_Word32 +RTPSender::SetFECUepProtection(const bool keyUseUepProtection, + const bool deltaUseUepProtection) + +{ + if(_audioConfigured) + { + return -1; + } + return _video->SetFECUepProtection(keyUseUepProtection, + deltaUseUepProtection); +} } // namespace webrtc diff --git a/src/modules/rtp_rtcp/source/rtp_sender.h b/src/modules/rtp_rtcp/source/rtp_sender.h index 0e80cb57be..ebdd14c3f5 100644 --- a/src/modules/rtp_rtcp/source/rtp_sender.h +++ b/src/modules/rtp_rtcp/source/rtp_sender.h @@ -248,7 +248,10 @@ public: WebRtc_UWord8& payloadTypeFEC) const; WebRtc_Word32 SetFECCodeRate(const WebRtc_UWord8 keyFrameCodeRate, - const WebRtc_UWord8 deltaFrameCodeRate); + const WebRtc_UWord8 deltaFrameCodeRate); + + WebRtc_Word32 SetFECUepProtection(const bool keyUseUepProtection, + const bool deltaUseUepProtection); protected: WebRtc_Word32 CheckPayloadType(const WebRtc_Word8 payloadType, RtpVideoCodecTypes& videoType); diff --git a/src/modules/rtp_rtcp/source/rtp_sender_video.cc b/src/modules/rtp_rtcp/source/rtp_sender_video.cc index 5d052c5d28..4f3631aa36 100644 --- a/src/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/src/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -42,7 +42,10 @@ RTPSenderVideo::RTPSenderVideo(const WebRtc_Word32 id, _payloadTypeFEC(-1), _codeRateKey(0), _codeRateDelta(0), + _useUepProtectionKey(false), + _useUepProtectionDelta(false), _fecProtectionFactor(0), + _fecUseUepProtection(false), _numberFirstPartition(0), // H263 @@ -70,7 +73,10 @@ RTPSenderVideo::Init() _payloadTypeFEC = -1; _codeRateKey = 0; _codeRateDelta = 0; + _useUepProtectionKey = false; + _useUepProtectionDelta = false; _fecProtectionFactor = 0; + _fecUseUepProtection = false; _numberFirstPartition = 0; return 0; } @@ -195,8 +201,11 @@ RTPSenderVideo::SendVideoPacket(const FrameType frameType, ForwardErrorCorrection::kMaxMediaPackets; } - retVal = _fec.GenerateFEC(_mediaPacketListFec, _fecProtectionFactor, - _numberFirstPartition, fecPacketList); + retVal = _fec.GenerateFEC(_mediaPacketListFec, + _fecProtectionFactor, + _numberFirstPartition, + _fecUseUepProtection, + fecPacketList); while(!_rtpPacketListFec.Empty()) { WebRtc_UWord8 newDataBuffer[IP_PACKET_SIZE]; @@ -324,7 +333,8 @@ RTPSenderVideo::SetGenericFECStatus(const bool enable, _payloadTypeFEC = payloadTypeFEC; _codeRateKey = 0; _codeRateDelta = 0; - + _useUepProtectionKey = false; + _useUepProtectionDelta = false; return 0; } @@ -359,6 +369,15 @@ RTPSenderVideo::SetFECCodeRate(const WebRtc_UWord8 keyFrameCodeRate, return 0; } +WebRtc_Word32 +RTPSenderVideo::SetFECUepProtection(const bool keyUseUepProtection, + const bool deltaUseUepProtection) +{ + _useUepProtectionKey = keyUseUepProtection; + _useUepProtectionDelta = deltaUseUepProtection; + return 0; +} + WebRtc_Word32 RTPSenderVideo::SendVideo(const RtpVideoCodecTypes videoType, const FrameType frameType, @@ -378,9 +397,11 @@ RTPSenderVideo::SendVideo(const RtpVideoCodecTypes videoType, if (frameType == kVideoFrameKey) { _fecProtectionFactor = _codeRateKey; + _fecUseUepProtection = _useUepProtectionKey; } else { _fecProtectionFactor = _codeRateDelta; + _fecUseUepProtection = _useUepProtectionDelta; } // Default setting for number of first partition packets: diff --git a/src/modules/rtp_rtcp/source/rtp_sender_video.h b/src/modules/rtp_rtcp/source/rtp_sender_video.h index 078ec833e2..640f568dab 100644 --- a/src/modules/rtp_rtcp/source/rtp_sender_video.h +++ b/src/modules/rtp_rtcp/source/rtp_sender_video.h @@ -80,7 +80,10 @@ public: WebRtc_UWord8& payloadTypeFEC) const; WebRtc_Word32 SetFECCodeRate(const WebRtc_UWord8 keyFrameCodeRate, - const WebRtc_UWord8 deltaFrameCodeRate); + const WebRtc_UWord8 deltaFrameCodeRate); + + WebRtc_Word32 SetFECUepProtection(const bool keyUseUepProtection, + const bool deltaUseUepProtection); protected: virtual WebRtc_Word32 SendVideoPacket(const FrameType frameType, @@ -153,7 +156,10 @@ private: WebRtc_Word8 _payloadTypeFEC; WebRtc_UWord8 _codeRateKey; WebRtc_UWord8 _codeRateDelta; + bool _useUepProtectionKey; + bool _useUepProtectionDelta; WebRtc_UWord8 _fecProtectionFactor; + bool _fecUseUepProtection; WebRtc_UWord32 _numberFirstPartition; ListWrapper _mediaPacketListFec; ListWrapper _rtpPacketListFec; diff --git a/src/modules/rtp_rtcp/test/testFec/test_fec.cc b/src/modules/rtp_rtcp/test/testFec/test_fec.cc index 111682950d..066d0e5a38 100644 --- a/src/modules/rtp_rtcp/test/testFec/test_fec.cc +++ b/src/modules/rtp_rtcp/test/testFec/test_fec.cc @@ -28,15 +28,14 @@ int main() // Use same values as set in forward_correction.cc const WebRtc_UWord8 rtpHeaderSize = 12; - const bool kUEP = true; - const WebRtc_UWord32 kForceFecThr = 1; + + // FOR UEP + const bool kUseUnequalProtection = true; + WebRtc_UWord32 numImpPackets = 0; WebRtc_UWord32 id = 0; webrtc::ForwardErrorCorrection fec(id); - // FOR UEP test - WebRtc_UWord32 numImpPackets = 0; - webrtc::ListWrapper mediaPacketList; webrtc::ListWrapper fecPacketList; webrtc::ListWrapper toDecodeList; @@ -96,8 +95,11 @@ int main() WebRtc_UWord8 packetMask[numFecPackets * maskBytesPerFecPacket]; memset(packetMask, 0, numFecPackets * maskBytesPerFecPacket); - webrtc::internal::GeneratePacketMasks(numMediaPackets,numFecPackets, - numImpPackets, packetMask); + webrtc::internal::GeneratePacketMasks(numMediaPackets, + numFecPackets, + numImpPackets, + kUseUnequalProtection, + packetMask); #ifdef VERBOSE_OUTPUT printf("%u media packets, %u FEC packets, %u numImpPackets, " @@ -128,7 +130,8 @@ int main() #endif // Check for all zero rows or columns: indicates incorrect mask WebRtc_UWord32 rowLimit = numMediaPackets; - if (numFecPackets <= numImpPackets && kUEP == true) + if (numFecPackets <= numImpPackets && + kUseUnequalProtection == true) { rowLimit = numImpPackets; } @@ -195,7 +198,7 @@ int main() mediaPacket->data[1] |= 0x80; // Set the marker bit of the last packet. if (fec.GenerateFEC(mediaPacketList, protectionFactor, numImpPackets, - fecPacketList) != 0) + kUseUnequalProtection, fecPacketList) != 0) { printf("Error: GenerateFEC() failed\n"); return -1; diff --git a/src/modules/video_coding/main/interface/video_coding_defines.h b/src/modules/video_coding/main/interface/video_coding_defines.h index 921fb9060e..739440f287 100644 --- a/src/modules/video_coding/main/interface/video_coding_defines.h +++ b/src/modules/video_coding/main/interface/video_coding_defines.h @@ -138,13 +138,18 @@ protected: virtual ~VCMReceiveStatisticsCallback() {} }; -// Callback class used for telling the user about the requested amount of bit stream protection -// Key frame FEC rate, delta frame and whether NACK should be on or off. +// Callback class used for telling the user about the requested amount of +// bit stream protection: FEC rate for key and delta frame; +// whether the FEC uses unequal protection (UEP) across packets, +// for key and delta frame; +// and whether NACK should be on or off. class VCMProtectionCallback { public: virtual WebRtc_Word32 ProtectionRequest(const WebRtc_UWord8 deltaFECRate, const WebRtc_UWord8 keyFECRate, + const bool deltaUseUepProtection, + const bool keyUseUepProtection, const bool nack) = 0; protected: diff --git a/src/modules/video_coding/main/source/media_optimization.cc b/src/modules/video_coding/main/source/media_optimization.cc index 3b0351574d..5119254784 100644 --- a/src/modules/video_coding/main/source/media_optimization.cc +++ b/src/modules/video_coding/main/source/media_optimization.cc @@ -138,7 +138,6 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, // Update protection method with content metrics selectedMethod->UpdateContentMetrics(_content->ShortTermAvgData()); - // Update method will compute the robustness settings for the given // protection method and the overhead cost // the protection method is set by the user via SetVideoProtection. @@ -146,12 +145,18 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, // FEC protection settings _lossProtLogic->UpdateMethod(); - // Get the code rate for Key frames + // Get the FEC code rate for Key frames const WebRtc_UWord8 codeRateKeyRTP = selectedMethod->RequiredProtectionFactorK(); - // Get the code rate for Delta frames + // Get the FEC code rate for Delta frames const WebRtc_UWord8 codeRateDeltaRTP = selectedMethod->RequiredProtectionFactorD(); + // Get the FEC-UEP protection status for Key frames: UEP on/off + const bool useUepProtectionKeyRTP = selectedMethod->RequiredUepProtectionK(); + + // Get the FEC-UEP protection status for Delta frames: UEP on/off + const bool useUepProtectionDeltaRTP = selectedMethod->RequiredUepProtectionD(); + // Get the effective packet loss for ER packetLossEnc = selectedMethod->RequiredPacketLossER(); @@ -163,6 +168,8 @@ VCMMediaOptimization::SetTargetRates(WebRtc_UWord32 bitRate, { _videoProtectionCallback->ProtectionRequest(codeRateDeltaRTP, codeRateKeyRTP, + useUepProtectionDeltaRTP, + useUepProtectionKeyRTP, nackStatus); } } diff --git a/src/video_engine/main/source/vie_encoder.cc b/src/video_engine/main/source/vie_encoder.cc index 6cd5b6f0fd..81ad3c1779 100644 --- a/src/video_engine/main/source/vie_encoder.cc +++ b/src/video_engine/main/source/vie_encoder.cc @@ -811,11 +811,16 @@ ViEEncoder::SendData(const FrameType frameType, WebRtc_Word32 ViEEncoder::ProtectionRequest(const WebRtc_UWord8 deltaFECRate, const WebRtc_UWord8 keyFECRate, + const bool deltaUseUepProtection, + const bool keyUseUepProtection, const bool nack) { - WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceVideo, ViEId(_engineId, _channelId), - "%s: deltaFECRate: %u, keyFECRate: %u, nack: %d", __FUNCTION__, - deltaFECRate, keyFECRate, nack); + WEBRTC_TRACE(webrtc::kTraceStream, webrtc::kTraceVideo, + ViEId(_engineId, _channelId),"%s, " + "deltaFECRate: %u, keyFECRate: %u, deltaUseUepProtection: " + "%d, keyUseUepProtection: %d, nack: %d", __FUNCTION__, + deltaFECRate, keyFECRate, deltaUseUepProtection, + keyUseUepProtection, nack); if (_rtpRtcp.SetFECCodeRate(keyFECRate, deltaFECRate) != 0) { @@ -823,6 +828,13 @@ WebRtc_Word32 ViEEncoder::ProtectionRequest(const WebRtc_UWord8 deltaFECRate, ViEId(_engineId, _channelId), "%s: Could not update FEC code rate", __FUNCTION__); } + if (_rtpRtcp.SetFECUepProtection(keyUseUepProtection, + deltaUseUepProtection) != 0) + { + WEBRTC_TRACE(webrtc::kTraceError, webrtc::kTraceVideo, + ViEId(_engineId, _channelId), + "%s: Could not update FEC-UEP protection", __FUNCTION__); + } return 0; } diff --git a/src/video_engine/main/source/vie_encoder.h b/src/video_engine/main/source/vie_encoder.h index d821f3d3e5..da9715d60b 100644 --- a/src/video_engine/main/source/vie_encoder.h +++ b/src/video_engine/main/source/vie_encoder.h @@ -97,7 +97,10 @@ public: // Implements VideoProtectionCallback virtual WebRtc_Word32 ProtectionRequest(const WebRtc_UWord8 deltaFECRate, const WebRtc_UWord8 keyFECRate, + const bool deltaUseUepProtection, + const bool keyUseUepProtection, const bool nack); + // Implements VideoSendStatisticsCallback virtual WebRtc_Word32 SendStatistics(const WebRtc_UWord32 bitRate, const WebRtc_UWord32 frameRate);