Add CVO support to video_coding layer.

CVO is, instead of rotating frame on the capture side, to have renderer rotate the frame based on a new rtp header extension.

The change includes
1. encoder side needs to pass this from raw frame to the encoded frame.
2. decoder needs to copy it from rtp packet (only the last packet of a frame has this info) to decoded frame.

R=mflodman@webrtc.org
TBR=stefan@webrtc.org

BUG=4145

Review URL: https://webrtc-codereview.appspot.com/46429006

Cr-Commit-Position: refs/heads/master@{#8767}
git-svn-id: http://webrtc.googlecode.com/svn/trunk@8767 4adac7df-926f-26a2-2b94-8c16560cd09d

webrtc/modules/video_coding/main/source/encoded_frame.cc

@@ -16,26 +16,26 @@
 namespace webrtc {
 
 VCMEncodedFrame::VCMEncodedFrame()
-:
-webrtc::EncodedImage(),
-_renderTimeMs(-1),
-_payloadType(0),
-_missingFrame(false),
-_codec(kVideoCodecUnknown),
-_fragmentation()
-{
+    : webrtc::EncodedImage(),
+      _renderTimeMs(-1),
+      _payloadType(0),
+      _missingFrame(false),
+      _codec(kVideoCodecUnknown),
+      _fragmentation(),
+      _rotation(kVideoRotation_0),
+      _rotation_set(false) {
     _codecSpecificInfo.codecType = kVideoCodecUnknown;
 }
 
 VCMEncodedFrame::VCMEncodedFrame(const webrtc::EncodedImage& rhs)
-:
-webrtc::EncodedImage(rhs),
-_renderTimeMs(-1),
-_payloadType(0),
-_missingFrame(false),
-_codec(kVideoCodecUnknown),
-_fragmentation()
-{
+    : webrtc::EncodedImage(rhs),
+      _renderTimeMs(-1),
+      _payloadType(0),
+      _missingFrame(false),
+      _codec(kVideoCodecUnknown),
+      _fragmentation(),
+      _rotation(kVideoRotation_0),
+      _rotation_set(false) {
     _codecSpecificInfo.codecType = kVideoCodecUnknown;
     _buffer = NULL;
     _size = 0;
@@ -48,14 +48,15 @@ _fragmentation()
 }
 
 VCMEncodedFrame::VCMEncodedFrame(const VCMEncodedFrame& rhs)
-  :
-    webrtc::EncodedImage(rhs),
-    _renderTimeMs(rhs._renderTimeMs),
-    _payloadType(rhs._payloadType),
-    _missingFrame(rhs._missingFrame),
-    _codecSpecificInfo(rhs._codecSpecificInfo),
-    _codec(rhs._codec),
-    _fragmentation() {
+    : webrtc::EncodedImage(rhs),
+      _renderTimeMs(rhs._renderTimeMs),
+      _payloadType(rhs._payloadType),
+      _missingFrame(rhs._missingFrame),
+      _codecSpecificInfo(rhs._codecSpecificInfo),
+      _codec(rhs._codec),
+      _fragmentation(),
+      _rotation(rhs._rotation),
+      _rotation_set(rhs._rotation_set) {
   _buffer = NULL;
   _size = 0;
   _length = 0;
@@ -96,6 +97,8 @@ void VCMEncodedFrame::Reset()
     _length = 0;
     _codecSpecificInfo.codecType = kVideoCodecUnknown;
     _codec = kVideoCodecUnknown;
+    _rotation = kVideoRotation_0;
+    _rotation_set = false;
 }
 
 void VCMEncodedFrame::CopyCodecSpecific(const RTPVideoHeader* header)

webrtc/modules/video_coding/main/source/encoded_frame.h

@@ -70,6 +70,10 @@ public:
     */
     webrtc::FrameType FrameType() const {return ConvertFrameType(_frameType);}
     /**
+    *   Get frame rotation
+    */
+    VideoRotation rotation() const { return _rotation; }
+    /**
     *   True if this frame is complete, false otherwise
     */
     bool Complete() const { return _completeFrame; }
@@ -116,6 +120,12 @@ protected:
     CodecSpecificInfo             _codecSpecificInfo;
     webrtc::VideoCodecType        _codec;
     RTPFragmentationHeader        _fragmentation;
+    VideoRotation                 _rotation;
+
+    // Video rotation is only set along with the last packet for each frame
+    // (same as marker bit). This |_rotation_set| is only for debugging purpose
+    // to ensure we don't set it twice for a frame.
+    bool                          _rotation_set;
 };
 
 }  // namespace webrtc

webrtc/modules/video_coding/main/source/frame_buffer.cc

@@ -13,6 +13,7 @@
 #include <assert.h>
 #include <string.h>
 
+#include "webrtc/base/checks.h"
 #include "webrtc/modules/video_coding/main/source/packet.h"
 #include "webrtc/system_wrappers/interface/logging.h"
 
@@ -146,6 +147,18 @@ VCMFrameBuffer::InsertPacket(const VCMPacket& packet,
 
     _latestPacketTimeMs = timeInMs;
 
+    // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/
+    // ts_126114v120700p.pdf Section 7.4.5.
+    // The MTSI client shall add the payload bytes as defined in this clause
+    // onto the last RTP packet in each group of packets which make up a key
+    // frame (I-frame or IDR frame in H.264 (AVC), or an IRAP picture in H.265
+    // (HEVC)).
+    if (packet.markerBit) {
+      DCHECK(!_rotation_set);
+      _rotation = packet.codecSpecificHeader.rotation;
+      _rotation_set = true;
+    }
+
     if (_sessionInfo.complete()) {
       SetState(kStateComplete);
       return kCompleteSession;

webrtc/modules/video_coding/main/source/generic_decoder.cc

@@ -74,6 +74,7 @@ int32_t VCMDecodedFrameCallback::Decoded(I420VideoFrame& decodedImage)
     if (callback != NULL)
     {
         decodedImage.set_render_time_ms(frameInfo->renderTimeMs);
+        decodedImage.set_rotation(frameInfo->rotation);
         callback->FrameToRender(decodedImage);
     }
     return WEBRTC_VIDEO_CODEC_OK;
@@ -148,6 +149,7 @@ int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame,
 {
     _frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs;
     _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs();
+    _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation();
     _callback->Map(frame.TimeStamp(), &_frameInfos[_nextFrameInfoIdx]);
 
     _nextFrameInfoIdx = (_nextFrameInfoIdx + 1) % kDecoderFrameMemoryLength;

webrtc/modules/video_coding/main/source/generic_decoder.h

@@ -29,6 +29,7 @@ struct VCMFrameInformation
     int64_t     renderTimeMs;
     int64_t     decodeStartTimeMs;
     void*             userData;
+    VideoRotation rotation;
 };
 
 class VCMDecodedFrameCallback : public DecodedImageCallback

webrtc/modules/video_coding/main/source/generic_encoder.cc

@@ -60,9 +60,11 @@ VCMGenericEncoder::VCMGenericEncoder(VideoEncoder* encoder,
                                      bool internalSource)
     : encoder_(encoder),
       rate_observer_(rate_observer),
+      vcm_encoded_frame_callback_(nullptr),
       bit_rate_(0),
       frame_rate_(0),
-      internal_source_(internalSource) {
+      internal_source_(internalSource),
+      rotation_(kVideoRotation_0) {
 }
 
 VCMGenericEncoder::~VCMGenericEncoder()
@@ -75,6 +77,7 @@ int32_t VCMGenericEncoder::Release()
       rtc::CritScope lock(&rates_lock_);
       bit_rate_ = 0;
       frame_rate_ = 0;
+      vcm_encoded_frame_callback_ = nullptr;
     }
 
     return encoder_->Release();
@@ -106,6 +109,16 @@ VCMGenericEncoder::Encode(const I420VideoFrame& inputFrame,
   std::vector<VideoFrameType> video_frame_types(frameTypes.size(),
                                                 kDeltaFrame);
   VCMEncodedFrame::ConvertFrameTypes(frameTypes, &video_frame_types);
+
+  rotation_ = inputFrame.rotation();
+
+  if (vcm_encoded_frame_callback_) {
+    // Keep track of the current frame rotation and apply to the output of the
+    // encoder. There might not be exact as the encoder could have one frame
+    // delay but it should be close enough.
+    vcm_encoded_frame_callback_->SetRotation(rotation_);
+  }
+
   return encoder_->Encode(inputFrame, codecSpecificInfo, &video_frame_types);
 }
 
@@ -178,6 +191,7 @@ int32_t
 VCMGenericEncoder::RegisterEncodeCallback(VCMEncodedFrameCallback* VCMencodedFrameCallback)
 {
     VCMencodedFrameCallback->SetInternalSource(internal_source_);
+    vcm_encoded_frame_callback_ = VCMencodedFrameCallback;
     return encoder_->RegisterEncodeCompleteCallback(VCMencodedFrameCallback);
 }
 
@@ -191,14 +205,16 @@ VCMGenericEncoder::InternalSource() const
   * Callback Implementation
   ***************************/
 VCMEncodedFrameCallback::VCMEncodedFrameCallback(
-    EncodedImageCallback* post_encode_callback):
-_sendCallback(),
-_mediaOpt(NULL),
-_payloadType(0),
-_internalSource(false),
-post_encode_callback_(post_encode_callback)
+    EncodedImageCallback* post_encode_callback)
+    : _sendCallback(),
+      _mediaOpt(NULL),
+      _payloadType(0),
+      _internalSource(false),
+      _rotation(kVideoRotation_0),
+      post_encode_callback_(post_encode_callback)
 #ifdef DEBUG_ENCODER_BIT_STREAM
-, _bitStreamAfterEncoder(NULL)
+      ,
+      _bitStreamAfterEncoder(NULL)
 #endif
 {
 #ifdef DEBUG_ENCODER_BIT_STREAM
@@ -241,6 +257,7 @@ int32_t VCMEncodedFrameCallback::Encoded(
   memset(&rtpVideoHeader, 0, sizeof(RTPVideoHeader));
   RTPVideoHeader* rtpVideoHeaderPtr = &rtpVideoHeader;
   CopyCodecSpecific(codecSpecificInfo, &rtpVideoHeaderPtr);
+  rtpVideoHeader.rotation = _rotation;
 
   int32_t callbackReturn = _sendCallback->SendData(
       _payloadType, encodedImage, *fragmentationHeader, rtpVideoHeaderPtr);

webrtc/modules/video_coding/main/source/generic_encoder.h

@@ -54,11 +54,14 @@ public:
     void SetPayloadType(uint8_t payloadType) { _payloadType = payloadType; };
     void SetInternalSource(bool internalSource) { _internalSource = internalSource; };
 
+    void SetRotation(VideoRotation rotation) { _rotation = rotation; }
+
 private:
     VCMPacketizationCallback* _sendCallback;
     media_optimization::MediaOptimization* _mediaOpt;
     uint8_t _payloadType;
     bool _internalSource;
+    VideoRotation _rotation;
 
     EncodedImageCallback* post_encode_callback_;
 
@@ -136,10 +139,12 @@ public:
 private:
     VideoEncoder* const encoder_;
     VideoEncoderRateObserver* const rate_observer_;
+    VCMEncodedFrameCallback*  vcm_encoded_frame_callback_;
     uint32_t bit_rate_;
     uint32_t frame_rate_;
     const bool internal_source_;
     mutable rtc::CriticalSection rates_lock_;
+    VideoRotation rotation_;
 }; // end of VCMGenericEncoder class
 
 }  // namespace webrtc