Add rotation to EncodedImage and make sure it is passed through encoders.

This fix a potential race where the rotation information of a sent frame does not match the encoded frame.

BUG=webrtc:5783
TEST= Run ApprtcDemo on IOs and Android with and without capture to texture and both VP8 and H264.
R=magjed@webrtc.org, pbos@webrtc.org, tkchin@webrtc.org
TBR=tkchin_webrtc // For IOS changes.

Review URL: https://codereview.webrtc.org/1886113003 .

Cr-Commit-Position: refs/heads/master@{#12426}

webrtc/api/java/jni/androidmediaencoder_jni.cc

@@ -12,6 +12,9 @@
 // androidmediacodeccommon.h to avoid build errors.
 #include "webrtc/api/java/jni/androidmediaencoder_jni.h"
 
+#include <algorithm>
+#include <list>
+
 #include "third_party/libyuv/include/libyuv/convert.h"
 #include "third_party/libyuv/include/libyuv/convert_from.h"
 #include "third_party/libyuv/include/libyuv/video_common.h"
@@ -219,7 +222,6 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder,
   int frames_dropped_media_encoder_;  // Number of frames dropped by encoder.
   // Number of dropped frames caused by full queue.
   int consecutive_full_queue_frame_drops_;
-  int frames_in_queue_;  // Number of frames in encoder queue.
   int64_t stat_start_time_ms_;  // Start time for statistics.
   int current_frames_;  // Number of frames in the current statistics interval.
   int current_bytes_;  // Encoded bytes in the current statistics interval.
@@ -227,13 +229,31 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder,
   int current_encoding_time_ms_;  // Overall encoding time in the current second
   int64_t last_input_timestamp_ms_;  // Timestamp of last received yuv frame.
   int64_t last_output_timestamp_ms_;  // Timestamp of last encoded frame.
-  std::vector<int32_t> timestamps_;  // Video frames timestamp queue.
+
-  std::vector<int64_t> render_times_ms_;  // Video frames render time queue.
+  struct InputFrameInfo {
-  std::vector<int64_t> frame_rtc_times_ms_;  // Time when video frame is sent to
+    InputFrameInfo(int64_t encode_start_time,
-                                             // encoder input.
+                   int32_t frame_timestamp,
-  int32_t output_timestamp_;  // Last output frame timestamp from timestamps_ Q.
+                   int64_t frame_render_time_ms,
+                   webrtc::VideoRotation rotation)
+        : encode_start_time(encode_start_time),
+          frame_timestamp(frame_timestamp),
+          frame_render_time_ms(frame_render_time_ms),
+          rotation(rotation) {}
+    // Time when video frame is sent to encoder input.
+    const int64_t encode_start_time;
+
+    // Input frame information.
+    const int32_t frame_timestamp;
+    const int64_t frame_render_time_ms;
+    const webrtc::VideoRotation rotation;
+  };
+  std::list<InputFrameInfo> input_frame_infos_;
+  int32_t output_timestamp_;      // Last output frame timestamp from
+                                  // |input_frame_infos_|.
   int64_t output_render_time_ms_; // Last output frame render time from
-                                  // render_times_ms_ queue.
+                                  // |input_frame_infos_|.
+  webrtc::VideoRotation output_rotation_;  // Last output frame rotation from
+                                           // |input_frame_infos_|.
   // Frame size in bytes fed to MediaCodec.
   int yuv_size_;
   // True only when between a callback_->Encoded() call return a positive value
@@ -507,7 +527,6 @@ int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread(
   frames_encoded_ = 0;
   frames_dropped_media_encoder_ = 0;
   consecutive_full_queue_frame_drops_ = 0;
-  frames_in_queue_ = 0;
   current_timestamp_us_ = 0;
   stat_start_time_ms_ = GetCurrentTimeMs();
   current_frames_ = 0;
@@ -518,9 +537,7 @@ int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread(
   last_output_timestamp_ms_ = -1;
   output_timestamp_ = 0;
   output_render_time_ms_ = 0;
-  timestamps_.clear();
+  input_frame_infos_.clear();
-  render_times_ms_.clear();
-  frame_rtc_times_ms_.clear();
   drop_next_input_frame_ = false;
   use_surface_ = use_surface;
   picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF;
@@ -619,11 +636,10 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
       return WEBRTC_VIDEO_CODEC_ERROR;
   }
   if (frames_encoded_ < kMaxEncodedLogFrames) {
-    ALOGD << "Encoder frame in # " << (frames_received_ - 1) <<
+    ALOGD << "Encoder frame in # " << (frames_received_ - 1)
-        ". TS: " << (int)(current_timestamp_us_ / 1000) <<
+          << ". TS: " << (int)(current_timestamp_us_ / 1000)
-        ". Q: " << frames_in_queue_ <<
+          << ". Q: " << input_frame_infos_.size() << ". Fps: " << last_set_fps_
-        ". Fps: " << last_set_fps_ <<
+          << ". Kbps: " << last_set_bitrate_kbps_;
-        ". Kbps: " << last_set_bitrate_kbps_;
   }
 
   if (drop_next_input_frame_) {
@@ -639,8 +655,9 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
 
   // Check if we accumulated too many frames in encoder input buffers and drop
   // frame if so.
-  if (frames_in_queue_ > MAX_ENCODER_Q_SIZE) {
+  if (input_frame_infos_.size() > MAX_ENCODER_Q_SIZE) {
-    ALOGD << "Already " << frames_in_queue_ << " frames in the queue, dropping"
+    ALOGD << "Already " << input_frame_infos_.size()
+          << " frames in the queue, dropping"
           << ". TS: " << (int)(current_timestamp_us_ / 1000)
           << ". Fps: " << last_set_fps_
           << ". Consecutive drops: " << consecutive_full_queue_frame_drops_;
@@ -685,9 +702,7 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
 
-  // Save time when input frame is sent to the encoder input.
+  const int64_t time_before_calling_encode = GetCurrentTimeMs();
-  frame_rtc_times_ms_.push_back(GetCurrentTimeMs());
-
   const bool key_frame =
       frame_types->front() != webrtc::kVideoFrameDelta || send_key_frame;
   bool encode_status = true;
@@ -698,7 +713,6 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
     if (j_input_buffer_index == -1) {
       // Video codec falls behind - no input buffer available.
       ALOGW << "Encoder drop frame - no input buffers available";
-      frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin());
       current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
       frames_dropped_media_encoder_++;
       OnDroppedFrame();
@@ -720,13 +734,14 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
     return WEBRTC_VIDEO_CODEC_ERROR;
   }
 
+  // Save input image timestamps for later output.
+  input_frame_infos_.emplace_back(
+      time_before_calling_encode, input_frame.timestamp(),
+      input_frame.render_time_ms(), input_frame.rotation());
+
   last_input_timestamp_ms_ =
       current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec;
-  frames_in_queue_++;
 
-  // Save input image timestamps for later output
-  timestamps_.push_back(input_frame.timestamp());
-  render_times_ms_.push_back(input_frame.render_time_ms());
   current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
 
   if (!DeliverPendingOutputs(jni)) {
@@ -933,14 +948,14 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) {
     last_output_timestamp_ms_ =
         GetOutputBufferInfoPresentationTimestampUs(jni, j_output_buffer_info) /
         1000;
-    if (frames_in_queue_ > 0) {
+    if (!input_frame_infos_.empty()) {
-      output_timestamp_ = timestamps_.front();
+      const InputFrameInfo& frame_info = input_frame_infos_.front();
-      timestamps_.erase(timestamps_.begin());
+      output_timestamp_ = frame_info.frame_timestamp;
-      output_render_time_ms_ = render_times_ms_.front();
+      output_render_time_ms_ = frame_info.frame_render_time_ms;
-      render_times_ms_.erase(render_times_ms_.begin());
+      output_rotation_ = frame_info.rotation;
-      frame_encoding_time_ms = GetCurrentTimeMs() - frame_rtc_times_ms_.front();
+      frame_encoding_time_ms =
-      frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin());
+          GetCurrentTimeMs() - frame_info.encode_start_time;
-      frames_in_queue_--;
+      input_frame_infos_.pop_front();
     }
 
     // Extract payload.
@@ -969,6 +984,7 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) {
       image->_encodedHeight = height_;
       image->_timeStamp = output_timestamp_;
       image->capture_time_ms_ = output_render_time_ms_;
+      image->rotation_ = output_rotation_;
       image->_frameType =
           (key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta);
       image->_completeFrame = true;

webrtc/modules/video_coding/codecs/h264/h264_encoder_impl.cc

@@ -393,6 +393,7 @@ int32_t H264EncoderImpl::Encode(
   encoded_image_._timeStamp = frame.timestamp();
   encoded_image_.ntp_time_ms_ = frame.ntp_time_ms();
   encoded_image_.capture_time_ms_ = frame.render_time_ms();
+  encoded_image_.rotation_ = frame.rotation();
   encoded_image_._frameType = EVideoFrameType_to_FrameType(info.eFrameType);
 
   // Split encoded image up into fragments. This also updates |encoded_image_|.

webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.cc

@@ -118,8 +118,14 @@ struct FrameEncodeParams {
                     int32_t w,
                     int32_t h,
                     int64_t rtms,
-                    uint32_t ts)
+                    uint32_t ts,
-      : encoder(e), width(w), height(h), render_time_ms(rtms), timestamp(ts) {
+                    webrtc::VideoRotation r)
+      : encoder(e),
+        width(w),
+        height(h),
+        render_time_ms(rtms),
+        timestamp(ts),
+        rotation(r) {
     if (csi) {
       codec_specific_info = *csi;
     } else {
@@ -133,6 +139,7 @@ struct FrameEncodeParams {
   int32_t height;
   int64_t render_time_ms;
   uint32_t timestamp;
+  webrtc::VideoRotation rotation;
 };
 
 // We receive I420Frames as input, but we need to feed CVPixelBuffers into the
@@ -185,7 +192,8 @@ void VTCompressionOutputCallback(void* encoder,
   encode_params->encoder->OnEncodedFrame(
       status, info_flags, sample_buffer, encode_params->codec_specific_info,
       encode_params->width, encode_params->height,
-      encode_params->render_time_ms, encode_params->timestamp);
+      encode_params->render_time_ms, encode_params->timestamp,
+      encode_params->rotation);
 }
 
 }  // namespace internal
@@ -306,7 +314,7 @@ int H264VideoToolboxEncoder::Encode(
   std::unique_ptr<internal::FrameEncodeParams> encode_params;
   encode_params.reset(new internal::FrameEncodeParams(
       this, codec_specific_info, width_, height_, input_image.render_time_ms(),
-      input_image.timestamp()));
+      input_image.timestamp(), input_image.rotation()));
 
   // Update the bitrate if needed.
   SetBitrateBps(bitrate_adjuster_.GetAdjustedBitrateBps());
@@ -471,7 +479,8 @@ void H264VideoToolboxEncoder::OnEncodedFrame(
     int32_t width,
     int32_t height,
     int64_t render_time_ms,
-    uint32_t timestamp) {
+    uint32_t timestamp,
+    VideoRotation rotation) {
   if (status != noErr) {
     LOG(LS_ERROR) << "H264 encode failed.";
     return;
@@ -511,6 +520,7 @@ void H264VideoToolboxEncoder::OnEncodedFrame(
       is_keyframe ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta;
   frame.capture_time_ms_ = render_time_ms;
   frame._timeStamp = timestamp;
+  frame.rotation_ = rotation;
 
   int result = callback_->Encoded(frame, &codec_specific_info, header.get());
   if (result != 0) {

webrtc/modules/video_coding/codecs/h264/h264_video_toolbox_encoder.h

@@ -12,6 +12,7 @@
 #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_H264_H264_VIDEO_TOOLBOX_ENCODER_H_
 #define WEBRTC_MODULES_VIDEO_CODING_CODECS_H264_H264_VIDEO_TOOLBOX_ENCODER_H_
 
+#include "webrtc/common_video/rotation.h"
 #include "webrtc/modules/video_coding/codecs/h264/include/h264.h"
 #include "webrtc/modules/video_coding/include/bitrate_adjuster.h"
 
@@ -58,7 +59,8 @@ class H264VideoToolboxEncoder : public H264Encoder {
                       int32_t width,
                       int32_t height,
                       int64_t render_time_ms,
-                      uint32_t timestamp);
+                      uint32_t timestamp,
+                      VideoRotation rotation);
 
  private:
   int ResetCompressionSession();

webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc

@@ -1024,6 +1024,7 @@ int VP8EncoderImpl::GetEncodedPartitions(const VideoFrame& input_image,
     encoded_images_[encoder_idx]._timeStamp = input_image.timestamp();
     encoded_images_[encoder_idx].capture_time_ms_ =
         input_image.render_time_ms();
+    encoded_images_[encoder_idx].rotation_ = input_image.rotation();
 
     int qp = -1;
     vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp);

webrtc/modules/video_coding/codecs/vp9/vp9_impl.cc

@@ -692,6 +692,7 @@ int VP9EncoderImpl::GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt) {
     TRACE_COUNTER1("webrtc", "EncodedFrameSize", encoded_image_._length);
     encoded_image_._timeStamp = input_image_->timestamp();
     encoded_image_.capture_time_ms_ = input_image_->render_time_ms();
+    encoded_image_.rotation_ = input_image_->rotation();
     encoded_image_._encodedHeight = raw_->d_h;
     encoded_image_._encodedWidth = raw_->d_w;
     int qp = -1;

webrtc/modules/video_coding/generic_encoder.cc

@@ -101,7 +101,6 @@ VCMGenericEncoder::VCMGenericEncoder(
       vcm_encoded_frame_callback_(encoded_frame_callback),
       internal_source_(internal_source),
       encoder_params_({0, 0, 0, 0}),
-      rotation_(kVideoRotation_0),
       is_screenshare_(false) {}
 
 VCMGenericEncoder::~VCMGenericEncoder() {}
@@ -141,15 +140,6 @@ int32_t VCMGenericEncoder::Encode(const VideoFrame& frame,
   for (FrameType frame_type : frame_types)
     RTC_DCHECK(frame_type == kVideoFrameKey || frame_type == kVideoFrameDelta);
 
-  rotation_ = frame.rotation();
-
-  // Keep track of the current frame rotation and apply to the output of the
-  // encoder. There might not be exact as the encoder could have one frame delay
-  // but it should be close enough.
-  // TODO(pbos): Map from timestamp, this is racy (even if rotation_ is locked
-  // properly, which it isn't). More than one frame may be in the pipeline.
-  vcm_encoded_frame_callback_->SetRotation(rotation_);
-
   int32_t result = encoder_->Encode(frame, codec_specific, &frame_types);
 
   if (vcm_encoded_frame_callback_) {
@@ -228,7 +218,6 @@ VCMEncodedFrameCallback::VCMEncodedFrameCallback(
       media_opt_(nullptr),
       payload_type_(0),
       internal_source_(false),
-      rotation_(kVideoRotation_0),
       post_encode_callback_(post_encode_callback) {}
 
 VCMEncodedFrameCallback::~VCMEncodedFrameCallback() {}
@@ -254,7 +243,7 @@ int32_t VCMEncodedFrameCallback::Encoded(
   memset(&rtp_video_header, 0, sizeof(RTPVideoHeader));
   if (codec_specific)
     CopyCodecSpecific(codec_specific, &rtp_video_header);
-  rtp_video_header.rotation = rotation_;
+  rtp_video_header.rotation = encoded_image.rotation_;
 
   int32_t ret_val = send_callback_->SendData(
       payload_type_, encoded_image, fragmentation_header, &rtp_video_header);

webrtc/modules/video_coding/generic_encoder.h

@@ -48,7 +48,6 @@ class VCMEncodedFrameCallback : public EncodedImageCallback {
   void SetInternalSource(bool internal_source) {
     internal_source_ = internal_source;
   }
-  void SetRotation(VideoRotation rotation) { rotation_ = rotation; }
   void SignalLastEncoderImplementationUsed(
       const char* encoder_implementation_name);
 
@@ -57,7 +56,6 @@ class VCMEncodedFrameCallback : public EncodedImageCallback {
   media_optimization::MediaOptimization* media_opt_;
   uint8_t payload_type_;
   bool internal_source_;
-  VideoRotation rotation_;
 
   EncodedImageCallback* post_encode_callback_;
 };
@@ -96,7 +94,6 @@ class VCMGenericEncoder {
   const bool internal_source_;
   rtc::CriticalSection params_lock_;
   EncoderParameters encoder_params_ GUARDED_BY(params_lock_);
-  VideoRotation rotation_;
   bool is_screenshare_;
 };
 

webrtc/modules/video_coding/utility/quality_scaler.cc

@@ -144,9 +144,15 @@ const VideoFrame& QualityScaler::GetScaledFrame(const VideoFrame& frame) {
   if (scaler_.Scale(frame, &scaled_frame_) != 0)
     return frame;
 
+  // TODO(perkj): Refactor the scaler to not own |scaled_frame|. VideoFrame are
+  // just thin wrappers so instead the scaler should return a
+  // rtc::scoped_refptr<VideoFrameBuffer> and a new VideoFrame be created with
+  // the meta data from |frame|. That way we would not have to set all these
+  // meta data.
   scaled_frame_.set_ntp_time_ms(frame.ntp_time_ms());
   scaled_frame_.set_timestamp(frame.timestamp());
   scaled_frame_.set_render_time_ms(frame.render_time_ms());
+  scaled_frame_.set_rotation(frame.rotation());
 
   return scaled_frame_;
 }

webrtc/test/call_test.cc

@@ -275,6 +275,10 @@ void CallTest::CreateVideoStreams() {
   }
 }
 
+void CallTest::SetFakeVideoCaptureRotation(VideoRotation rotation) {
+  frame_generator_capturer_->SetFakeRotation(rotation);
+}
+
 void CallTest::CreateAudioStreams() {
   audio_send_stream_ = sender_call_->CreateAudioSendStream(audio_send_config_);
   for (size_t i = 0; i < audio_receive_configs_.size(); ++i) {

webrtc/test/call_test.h

@@ -80,6 +80,7 @@ class CallTest : public ::testing::Test {
   void Start();
   void Stop();
   void DestroyStreams();
+  void SetFakeVideoCaptureRotation(VideoRotation rotation);
 
   Clock* const clock_;
 

webrtc/test/frame_generator_capturer.cc

@@ -80,6 +80,11 @@ FrameGeneratorCapturer::~FrameGeneratorCapturer() {
   thread_.Stop();
 }
 
+void FrameGeneratorCapturer::SetFakeRotation(VideoRotation rotation) {
+  rtc::CritScope cs(&lock_);
+  fake_rotation_ = rotation;
+}
+
 bool FrameGeneratorCapturer::Init() {
   // This check is added because frame_generator_ might be file based and should
   // not crash because a file moved.
@@ -104,6 +109,7 @@ void FrameGeneratorCapturer::InsertFrame() {
     if (sending_) {
       VideoFrame* frame = frame_generator_->NextFrame();
       frame->set_ntp_time_ms(clock_->CurrentNtpInMilliseconds());
+      frame->set_rotation(fake_rotation_);
       if (first_frame_capture_time_ == -1) {
         first_frame_capture_time_ = frame->ntp_time_ms();
       }

webrtc/test/frame_generator_capturer.h

@@ -15,6 +15,7 @@
 #include "webrtc/base/criticalsection.h"
 #include "webrtc/base/platform_thread.h"
 #include "webrtc/base/scoped_ptr.h"
+#include "webrtc/common_video/rotation.h"
 #include "webrtc/test/video_capturer.h"
 #include "webrtc/typedefs.h"
 
@@ -46,6 +47,7 @@ class FrameGeneratorCapturer : public VideoCapturer {
   void Start() override;
   void Stop() override;
   void ForceFrame();
+  void SetFakeRotation(VideoRotation rotation);
 
   int64_t first_frame_capture_time() const { return first_frame_capture_time_; }
 
@@ -68,6 +70,7 @@ class FrameGeneratorCapturer : public VideoCapturer {
   rtc::scoped_ptr<FrameGenerator> frame_generator_;
 
   int target_fps_;
+  VideoRotation fake_rotation_ = kVideoRotation_0;
 
   int64_t first_frame_capture_time_;
 };

webrtc/video/end_to_end_tests.cc

@@ -266,107 +266,101 @@ TEST_F(EndToEndTest, TransmitsFirstFrame) {
   DestroyStreams();
 }
 
-TEST_F(EndToEndTest, SendsAndReceivesVP9) {
+class CodecObserver : public test::EndToEndTest,
-  class VP9Observer : public test::EndToEndTest,
                       public rtc::VideoSinkInterface<VideoFrame> {
-   public:
+ public:
-    VP9Observer()
+  CodecObserver(int no_frames_to_wait_for,
-        : EndToEndTest(2 * kDefaultTimeoutMs),
+                VideoRotation rotation_to_test,
-          encoder_(VideoEncoder::Create(VideoEncoder::kVp9)),
+                const std::string& payload_name,
-          decoder_(VP9Decoder::Create()),
+                webrtc::VideoEncoder* encoder,
-          frame_counter_(0) {}
+                webrtc::VideoDecoder* decoder)
+      : EndToEndTest(2 * webrtc::EndToEndTest::kDefaultTimeoutMs),
+        no_frames_to_wait_for_(no_frames_to_wait_for),
+        expected_rotation_(rotation_to_test),
+        payload_name_(payload_name),
+        encoder_(encoder),
+        decoder_(decoder),
+        frame_counter_(0) {}
 
-    void PerformTest() override {
+  void PerformTest() override {
-      EXPECT_TRUE(Wait())
+    EXPECT_TRUE(Wait())
-          << "Timed out while waiting for enough frames to be decoded.";
+        << "Timed out while waiting for enough frames to be decoded.";
-    }
+  }
 
-    void ModifyVideoConfigs(
+  void ModifyVideoConfigs(
-        VideoSendStream::Config* send_config,
+      VideoSendStream::Config* send_config,
-        std::vector<VideoReceiveStream::Config>* receive_configs,
+      std::vector<VideoReceiveStream::Config>* receive_configs,
-        VideoEncoderConfig* encoder_config) override {
+      VideoEncoderConfig* encoder_config) override {
-      send_config->encoder_settings.encoder = encoder_.get();
+    send_config->encoder_settings.encoder = encoder_.get();
-      send_config->encoder_settings.payload_name = "VP9";
+    send_config->encoder_settings.payload_name = payload_name_;
-      send_config->encoder_settings.payload_type = 124;
+    send_config->encoder_settings.payload_type = 126;
-      encoder_config->streams[0].min_bitrate_bps = 50000;
+    encoder_config->streams[0].min_bitrate_bps = 50000;
-      encoder_config->streams[0].target_bitrate_bps =
+    encoder_config->streams[0].target_bitrate_bps =
-          encoder_config->streams[0].max_bitrate_bps = 2000000;
+        encoder_config->streams[0].max_bitrate_bps = 2000000;
 
-      (*receive_configs)[0].renderer = this;
+    (*receive_configs)[0].renderer = this;
-      (*receive_configs)[0].decoders.resize(1);
+    (*receive_configs)[0].decoders.resize(1);
-      (*receive_configs)[0].decoders[0].payload_type =
+    (*receive_configs)[0].decoders[0].payload_type =
-          send_config->encoder_settings.payload_type;
+        send_config->encoder_settings.payload_type;
-      (*receive_configs)[0].decoders[0].payload_name =
+    (*receive_configs)[0].decoders[0].payload_name =
-          send_config->encoder_settings.payload_name;
+        send_config->encoder_settings.payload_name;
-      (*receive_configs)[0].decoders[0].decoder = decoder_.get();
+    (*receive_configs)[0].decoders[0].decoder = decoder_.get();
-    }
+  }
 
-    void OnFrame(const VideoFrame& video_frame) override {
+  void OnFrame(const VideoFrame& video_frame) override {
-      const int kRequiredFrames = 500;
+    EXPECT_EQ(expected_rotation_, video_frame.rotation());
-      if (++frame_counter_ == kRequiredFrames)
+    if (++frame_counter_ == no_frames_to_wait_for_)
-        observation_complete_.Set();
+      observation_complete_.Set();
-    }
+  }
 
-   private:
+  void OnFrameGeneratorCapturerCreated(
-    std::unique_ptr<webrtc::VideoEncoder> encoder_;
+      test::FrameGeneratorCapturer* frame_generator_capturer) override {
-    std::unique_ptr<webrtc::VideoDecoder> decoder_;
+    frame_generator_capturer->SetFakeRotation(expected_rotation_);
-    int frame_counter_;
+  }
-  } test;
 
+ private:
+  int no_frames_to_wait_for_;
+  VideoRotation expected_rotation_;
+  std::string payload_name_;
+  std::unique_ptr<webrtc::VideoEncoder> encoder_;
+  std::unique_ptr<webrtc::VideoDecoder> decoder_;
+  int frame_counter_;
+};
+
+TEST_F(EndToEndTest, SendsAndReceivesVP8Rotation90) {
+  CodecObserver test(5, kVideoRotation_90, "VP8",
+                     VideoEncoder::Create(VideoEncoder::kVp8),
+                     VP8Decoder::Create());
+  RunBaseTest(&test);
+}
+
+TEST_F(EndToEndTest, SendsAndReceivesVP9) {
+  CodecObserver test(500, kVideoRotation_0, "VP9",
+                     VideoEncoder::Create(VideoEncoder::kVp9),
+                     VP9Decoder::Create());
+  RunBaseTest(&test);
+}
+
+TEST_F(EndToEndTest, SendsAndReceivesVP9VideoRotation90) {
+  CodecObserver test(5, kVideoRotation_90, "VP9",
+                     VideoEncoder::Create(VideoEncoder::kVp9),
+                     VP9Decoder::Create());
   RunBaseTest(&test);
 }
 
 #if defined(WEBRTC_END_TO_END_H264_TESTS)
 
 TEST_F(EndToEndTest, SendsAndReceivesH264) {
-  class H264Observer : public test::EndToEndTest,
+  CodecObserver test(500, kVideoRotation_0, "H264",
-                       public rtc::VideoSinkInterface<VideoFrame> {
+                     VideoEncoder::Create(VideoEncoder::kH264),
-   public:
+                     H264Decoder::Create());
-    H264Observer()
+  RunBaseTest(&test);
-        : EndToEndTest(2 * kDefaultTimeoutMs),
+}
-          encoder_(VideoEncoder::Create(VideoEncoder::kH264)),
-          decoder_(H264Decoder::Create()),
-          frame_counter_(0) {}
-
-    void PerformTest() override {
-      EXPECT_TRUE(Wait())
-          << "Timed out while waiting for enough frames to be decoded.";
-    }
-
-    void ModifyVideoConfigs(
-        VideoSendStream::Config* send_config,
-        std::vector<VideoReceiveStream::Config>* receive_configs,
-        VideoEncoderConfig* encoder_config) override {
-      send_config->rtp.nack.rtp_history_ms =
-          (*receive_configs)[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
-      send_config->encoder_settings.encoder = encoder_.get();
-      send_config->encoder_settings.payload_name = "H264";
-      send_config->encoder_settings.payload_type = 126;
-      encoder_config->streams[0].min_bitrate_bps = 50000;
-      encoder_config->streams[0].target_bitrate_bps =
-          encoder_config->streams[0].max_bitrate_bps = 2000000;
-
-      (*receive_configs)[0].renderer = this;
-      (*receive_configs)[0].decoders.resize(1);
-      (*receive_configs)[0].decoders[0].payload_type =
-          send_config->encoder_settings.payload_type;
-      (*receive_configs)[0].decoders[0].payload_name =
-          send_config->encoder_settings.payload_name;
-      (*receive_configs)[0].decoders[0].decoder = decoder_.get();
-    }
-
-    void OnFrame(const VideoFrame& video_frame) override {
-      const int kRequiredFrames = 500;
-      if (++frame_counter_ == kRequiredFrames)
-        observation_complete_.Set();
-    }
-
-   private:
-    std::unique_ptr<webrtc::VideoEncoder> encoder_;
-    std::unique_ptr<webrtc::VideoDecoder> decoder_;
-    int frame_counter_;
-  } test;
 
+TEST_F(EndToEndTest, SendsAndReceivesH264VideoRotation90) {
+  CodecObserver test(5, kVideoRotation_90, "H264",
+                     VideoEncoder::Create(VideoEncoder::kH264),
+                     H264Decoder::Create());
   RunBaseTest(&test);
 }
 

webrtc/video_frame.h

@@ -157,6 +157,7 @@ class EncodedImage {
   static size_t GetBufferPaddingBytes(VideoCodecType codec_type);
 
   EncodedImage() : EncodedImage(nullptr, 0, 0) {}
+
   EncodedImage(uint8_t* buffer, size_t length, size_t size)
       : _buffer(buffer), _length(length), _size(size) {}
 
@@ -182,6 +183,7 @@ class EncodedImage {
   uint8_t* _buffer;
   size_t _length;
   size_t _size;
+  VideoRotation rotation_ = kVideoRotation_0;
   bool _completeFrame = false;
   AdaptReason adapt_reason_;
   int qp_ = -1;  // Quantizer value.