Add concept of whether video renderer supports rotation.

Rotation is best done when rendered in GPU, added the shader code which rotates the frame. For renderers which don't support rotation, the rotation will be done before sending down the frame to render. By default, assume renderer can't do rotation.

Tested with peerconnection_client on windows, AppRTCDemo on Mac.

BUG=4145
R=glaznev@webrtc.org, pthatcher@webrtc.org

Committed: https://code.google.com/p/webrtc/source/detail?r=8660

Committed: https://code.google.com/p/webrtc/source/detail?r=8661

Review URL: https://webrtc-codereview.appspot.com/43569004

Cr-Commit-Position: refs/heads/master@{#8705}
git-svn-id: http://webrtc.googlecode.com/svn/trunk@8705 4adac7df-926f-26a2-2b94-8c16560cd09d
This commit is contained in:
guoweis@webrtc.org
2015-03-12 21:37:26 +00:00
parent 04cd69887d
commit 00c509ad1c
23 changed files with 379 additions and 119 deletions

View File

@ -41,8 +41,7 @@ public class VideoCapturerAndroidTest extends ActivityTestCase {
private int framesRendered = 0;
private Object frameLock = 0;
@Override
public void setSize(int width, int height) {
private void setSize(int width, int height) {
}
@Override

View File

@ -528,10 +528,14 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
}
}
@Override
public void setSize(final int width, final int height) {
private void setSize(final int width, final int height) {
if (width == videoWidth && height == videoHeight) {
return;
}
Log.d(TAG, "ID: " + id + ". YuvImageRenderer.setSize: " +
width + " x " + height);
videoWidth = width;
videoHeight = height;
int[] strides = { width, width / 2, width / 2 };
@ -550,6 +554,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
@Override
public synchronized void renderFrame(I420Frame frame) {
setSize(frame.width, frame.height);
long now = System.nanoTime();
framesReceived++;
// Skip rendering of this frame if setSize() was not called.

View File

@ -696,21 +696,23 @@ class VideoRendererWrapper : public VideoRendererInterface {
virtual ~VideoRendererWrapper() {}
void SetSize(int width, int height) override {
ScopedLocalRefFrame local_ref_frame(AttachCurrentThreadIfNeeded());
const bool kNotReserved = false; // What does this param mean??
renderer_->SetSize(width, height, kNotReserved);
}
void RenderFrame(const cricket::VideoFrame* frame) override {
// This wraps VideoRenderer which still has SetSize.
void RenderFrame(const cricket::VideoFrame* video_frame) override {
ScopedLocalRefFrame local_ref_frame(AttachCurrentThreadIfNeeded());
const cricket::VideoFrame* frame =
video_frame->GetCopyWithRotationApplied();
if (width_ != frame->GetWidth() || height_ != frame->GetHeight()) {
width_ = frame->GetWidth();
height_ = frame->GetHeight();
renderer_->SetSize(width_, height_, 0);
}
renderer_->RenderFrame(frame);
}
private:
explicit VideoRendererWrapper(cricket::VideoRenderer* renderer)
: renderer_(renderer) {}
: renderer_(renderer), width_(0), height_(0) {}
int width_, height_;
scoped_ptr<cricket::VideoRenderer> renderer_;
};
@ -720,8 +722,6 @@ class JavaVideoRendererWrapper : public VideoRendererInterface {
public:
JavaVideoRendererWrapper(JNIEnv* jni, jobject j_callbacks)
: j_callbacks_(jni, j_callbacks),
j_set_size_id_(GetMethodID(
jni, GetObjectClass(jni, j_callbacks), "setSize", "(II)V")),
j_render_frame_id_(GetMethodID(
jni, GetObjectClass(jni, j_callbacks), "renderFrame",
"(Lorg/webrtc/VideoRenderer$I420Frame;)V")),
@ -738,14 +738,13 @@ class JavaVideoRendererWrapper : public VideoRendererInterface {
virtual ~JavaVideoRendererWrapper() {}
void SetSize(int width, int height) override {
void RenderFrame(const cricket::VideoFrame* video_frame) override {
ScopedLocalRefFrame local_ref_frame(jni());
jni()->CallVoidMethod(*j_callbacks_, j_set_size_id_, width, height);
CHECK_EXCEPTION(jni());
}
void RenderFrame(const cricket::VideoFrame* frame) override {
ScopedLocalRefFrame local_ref_frame(jni());
// TODO(guoweis): Remove once the java implementation supports rotation.
const cricket::VideoFrame* frame =
video_frame->GetCopyWithRotationApplied();
if (frame->GetNativeHandle() != NULL) {
jobject j_frame = CricketToJavaTextureFrame(frame);
jni()->CallVoidMethod(*j_callbacks_, j_render_frame_id_, j_frame);
@ -798,7 +797,6 @@ class JavaVideoRendererWrapper : public VideoRendererInterface {
}
ScopedGlobalRef<jobject> j_callbacks_;
jmethodID j_set_size_id_;
jmethodID j_render_frame_id_;
ScopedGlobalRef<jclass> j_frame_class_;
jmethodID j_i420_frame_ctor_id_;

View File

@ -144,7 +144,8 @@ public class VideoRenderer {
/** The real meat of VideoRendererInterface. */
public static interface Callbacks {
public void setSize(int width, int height);
// |frame| might have pending rotation and implementation of Callbacks
// should handle that by applying rotation during rendering.
public void renderFrame(I420Frame frame);
}

View File

@ -59,7 +59,6 @@ public class PeerConnectionTest {
private int expectedIceCandidates = 0;
private int expectedErrors = 0;
private int expectedRenegotiations = 0;
private int expectedSetSize = 0;
private int previouslySeenWidth = 0;
private int previouslySeenHeight = 0;
private int expectedFramesDelivered = 0;
@ -113,19 +112,8 @@ public class PeerConnectionTest {
gotIceCandidates.add(candidate);
}
public synchronized void expectSetSize() {
if (RENDER_TO_GUI) {
// When new frames are delivered to the GUI renderer we don't get
// notified of frame size info.
return;
}
++expectedSetSize;
}
@Override
public synchronized void setSize(int width, int height) {
private synchronized void setSize(int width, int height) {
assertFalse(RENDER_TO_GUI);
assertTrue(--expectedSetSize >= 0);
// Because different camera devices (fake & physical) produce different
// resolutions, we only sanity-check the set sizes,
assertTrue(width > 0);
@ -146,6 +134,7 @@ public class PeerConnectionTest {
@Override
public synchronized void renderFrame(VideoRenderer.I420Frame frame) {
setSize(frame.width, frame.height);
--expectedFramesDelivered;
}
@ -315,9 +304,6 @@ public class PeerConnectionTest {
stillWaitingForExpectations.add(
"expectedRemoveStreamLabels: " + expectedRemoveStreamLabels.size());
}
if (expectedSetSize != 0) {
stillWaitingForExpectations.add("expectedSetSize");
}
if (expectedFramesDelivered > 0) {
stillWaitingForExpectations.add(
"expectedFramesDelivered: " + expectedFramesDelivered);
@ -436,8 +422,7 @@ public class PeerConnectionTest {
public int height = -1;
public int numFramesDelivered = 0;
@Override
public void setSize(int width, int height) {
private void setSize(int width, int height) {
assertEquals(this.width, -1);
assertEquals(this.height, -1);
this.width = width;
@ -542,7 +527,6 @@ public class PeerConnectionTest {
VideoSource videoSource = factory.createVideoSource(
VideoCapturer.create(""), new MediaConstraints());
offeringExpectations.expectSetSize();
offeringExpectations.expectRenegotiationNeeded();
WeakReference<MediaStream> oLMS = addTracksToPC(
factory, offeringPC, videoSource, "offeredMediaStream",
@ -574,7 +558,6 @@ public class PeerConnectionTest {
assertTrue(sdpLatch.await());
assertNull(sdpLatch.getSdp());
answeringExpectations.expectSetSize();
answeringExpectations.expectRenegotiationNeeded();
WeakReference<MediaStream> aLMS = addTracksToPC(
factory, answeringPC, videoSource, "answeredMediaStream",
@ -636,8 +619,6 @@ public class PeerConnectionTest {
// chosen arbitrarily).
offeringExpectations.expectFramesDelivered(10);
answeringExpectations.expectFramesDelivered(10);
offeringExpectations.expectSetSize();
answeringExpectations.expectSetSize();
}
offeringExpectations.expectStateChange(DataChannel.State.OPEN);

View File

@ -115,9 +115,27 @@ class MediaStreamTrackInterface : public rtc::RefCountInterface,
// Interface for rendering VideoFrames from a VideoTrack
class VideoRendererInterface {
public:
virtual void SetSize(int width, int height) = 0;
// TODO(guoweis): Remove this function. Obsolete. The implementation of
// VideoRendererInterface should be able to handle different frame size as
// well as pending rotation. If it can't apply the frame rotation by itself,
// it should call |frame|.GetCopyWithRotationApplied() to get a frame that has
// the rotation applied.
virtual void SetSize(int width, int height) {}
// |frame| may have pending rotation. For clients which can't apply rotation,
// |frame|->GetCopyWithRotationApplied() will return a frame that has the
// rotation applied.
virtual void RenderFrame(const cricket::VideoFrame* frame) = 0;
// TODO(guoweis): Remove this function. This is added as a temporary solution
// until chrome renderers can apply rotation.
// Whether the VideoRenderer has the ability to rotate the frame before being
// displayed. The rotation of a frame is carried by
// VideoFrame.GetVideoRotation() and is the clockwise angle the frames must be
// rotated in order to display the frames correctly. If returning false, the
// frame's rotation must be applied before being delivered by RenderFrame.
virtual bool CanApplyRotation() { return false; }
protected:
// The destructor is protected to prevent deletion via the interface.
// This is so that we allow reference counted classes, where the destructor

View File

@ -38,19 +38,23 @@ class RTCVideoRendererNativeAdapter : public VideoRendererInterface {
public:
RTCVideoRendererNativeAdapter(RTCVideoRendererAdapter* adapter) {
_adapter = adapter;
_size = CGSizeZero;
}
void SetSize(int width, int height) override {
[_adapter.videoRenderer setSize:CGSizeMake(width, height)];
}
void RenderFrame(const cricket::VideoFrame* frame) override {
void RenderFrame(const cricket::VideoFrame* videoFrame) override {
const cricket::VideoFrame* frame = videoFrame->GetCopyWithRotationApplied();
CGSize currentSize = CGSizeMake(frame->GetWidth(), frame->GetHeight());
if (!CGSizeEqualToSize(_size, currentSize)) {
_size = currentSize;
[_adapter.videoRenderer setSize:_size];
}
RTCI420Frame* i420Frame = [[RTCI420Frame alloc] initWithVideoFrame:frame];
[_adapter.videoRenderer renderFrame:i420Frame];
}
private:
__weak RTCVideoRendererAdapter* _adapter;
CGSize _size;
};
}

View File

@ -35,34 +35,55 @@ namespace webrtc {
class FakeVideoTrackRenderer : public VideoRendererInterface {
public:
explicit FakeVideoTrackRenderer(VideoTrackInterface* video_track)
: video_track_(video_track) {
FakeVideoTrackRenderer(VideoTrackInterface* video_track)
: video_track_(video_track),
can_apply_rotation_(true),
last_frame_(NULL) {
video_track_->AddRenderer(this);
}
FakeVideoTrackRenderer(VideoTrackInterface* video_track,
bool can_apply_rotation)
: video_track_(video_track),
can_apply_rotation_(can_apply_rotation),
last_frame_(NULL) {
video_track_->AddRenderer(this);
}
~FakeVideoTrackRenderer() {
video_track_->RemoveRenderer(this);
}
// Implements VideoRendererInterface
virtual void SetSize(int width, int height) {
fake_renderer_.SetSize(width, height, 0);
}
virtual void RenderFrame(const cricket::VideoFrame* video_frame) override {
last_frame_ = const_cast<cricket::VideoFrame*>(video_frame);
const cricket::VideoFrame* frame =
can_apply_rotation_ ? video_frame
: video_frame->GetCopyWithRotationApplied();
if (!fake_renderer_.SetSize(static_cast<int>(frame->GetWidth()),
static_cast<int>(frame->GetHeight()), 0)) {
return;
}
virtual void RenderFrame(const cricket::VideoFrame* frame) {
fake_renderer_.RenderFrame(frame);
}
virtual bool CanApplyRotation() override { return can_apply_rotation_; }
int errors() const { return fake_renderer_.errors(); }
int width() const { return fake_renderer_.width(); }
int height() const { return fake_renderer_.height(); }
int num_set_sizes() const { return fake_renderer_.num_set_sizes(); }
int num_rendered_frames() const {
return fake_renderer_.num_rendered_frames();
}
const cricket::VideoFrame* last_frame() const { return last_frame_; }
private:
cricket::FakeVideoRenderer fake_renderer_;
rtc::scoped_refptr<VideoTrackInterface> video_track_;
bool can_apply_rotation_;
// Weak reference for frame pointer comparison only.
cricket::VideoFrame* last_frame_;
};
} // namespace webrtc

View File

@ -43,57 +43,132 @@ using webrtc::VideoSource;
using webrtc::VideoTrack;
using webrtc::VideoTrackInterface;
namespace {
class WebRtcVideoTestFrame : public cricket::WebRtcVideoFrame {
public:
using cricket::WebRtcVideoFrame::SetRotation;
};
} // namespace
class VideoTrackTest : public testing::Test {
public:
VideoTrackTest() {
static const char kVideoTrackId[] = "track_id";
channel_manager_.reset(new cricket::ChannelManager(
new cricket::FakeMediaEngine(), new cricket::FakeDeviceManager(),
rtc::Thread::Current()));
EXPECT_TRUE(channel_manager_->Init());
video_track_ = VideoTrack::Create(
kVideoTrackId,
VideoSource::Create(channel_manager_.get(),
new webrtc::RemoteVideoCapturer(), NULL));
}
protected:
rtc::scoped_ptr<cricket::ChannelManager> channel_manager_;
rtc::scoped_refptr<VideoTrackInterface> video_track_;
};
// Test adding renderers to a video track and render to them by providing
// frames to the source.
TEST(VideoTrack, RenderVideo) {
static const char kVideoTrackId[] = "track_id";
rtc::scoped_ptr<cricket::ChannelManager> channel_manager_;
channel_manager_.reset(
new cricket::ChannelManager(new cricket::FakeMediaEngine(),
new cricket::FakeDeviceManager(),
rtc::Thread::Current()));
ASSERT_TRUE(channel_manager_->Init());
rtc::scoped_refptr<VideoTrackInterface> video_track(
VideoTrack::Create(kVideoTrackId,
VideoSource::Create(channel_manager_.get(),
new webrtc::RemoteVideoCapturer(),
NULL)));
// FakeVideoTrackRenderer register itself to |video_track|
TEST_F(VideoTrackTest, RenderVideo) {
// FakeVideoTrackRenderer register itself to |video_track_|
rtc::scoped_ptr<FakeVideoTrackRenderer> renderer_1(
new FakeVideoTrackRenderer(video_track.get()));
new FakeVideoTrackRenderer(video_track_.get()));
cricket::VideoRenderer* render_input = video_track->GetSource()->FrameInput();
ASSERT_FALSE(render_input == NULL);
cricket::VideoRenderer* renderer_input =
video_track_->GetSource()->FrameInput();
ASSERT_FALSE(renderer_input == NULL);
cricket::WebRtcVideoFrame frame;
frame.InitToBlack(123, 123, 1, 1, 0, 0);
render_input->RenderFrame(&frame);
renderer_input->RenderFrame(&frame);
EXPECT_EQ(1, renderer_1->num_rendered_frames());
EXPECT_EQ(1, renderer_1->num_set_sizes());
EXPECT_EQ(123, renderer_1->width());
EXPECT_EQ(123, renderer_1->height());
// FakeVideoTrackRenderer register itself to |video_track|
// FakeVideoTrackRenderer register itself to |video_track_|
rtc::scoped_ptr<FakeVideoTrackRenderer> renderer_2(
new FakeVideoTrackRenderer(video_track.get()));
new FakeVideoTrackRenderer(video_track_.get()));
render_input->RenderFrame(&frame);
renderer_input->RenderFrame(&frame);
EXPECT_EQ(1, renderer_1->num_set_sizes());
EXPECT_EQ(123, renderer_1->width());
EXPECT_EQ(123, renderer_1->height());
EXPECT_EQ(1, renderer_2->num_set_sizes());
EXPECT_EQ(123, renderer_2->width());
EXPECT_EQ(123, renderer_2->height());
EXPECT_EQ(2, renderer_1->num_rendered_frames());
EXPECT_EQ(1, renderer_2->num_rendered_frames());
video_track->RemoveRenderer(renderer_1.get());
render_input->RenderFrame(&frame);
video_track_->RemoveRenderer(renderer_1.get());
renderer_input->RenderFrame(&frame);
EXPECT_EQ(2, renderer_1->num_rendered_frames());
EXPECT_EQ(2, renderer_2->num_rendered_frames());
}
// Test adding renderers which support and don't support rotation and receive
// the right frame.
TEST_F(VideoTrackTest, RenderVideoWithPendingRotation) {
const size_t kWidth = 800;
const size_t kHeight = 400;
// Add a renderer which supports rotation.
rtc::scoped_ptr<FakeVideoTrackRenderer> rotating_renderer(
new FakeVideoTrackRenderer(video_track_.get(), true));
cricket::VideoRenderer* renderer_input =
video_track_->GetSource()->FrameInput();
ASSERT_FALSE(renderer_input == NULL);
// Create a frame with rotation 90 degree.
WebRtcVideoTestFrame frame;
frame.InitToBlack(kWidth, kHeight, 1, 1, 0, 0);
frame.SetRotation(webrtc::kVideoRotation_90);
// rotating_renderer should see the frame unrotated.
renderer_input->RenderFrame(&frame);
EXPECT_EQ(1, rotating_renderer->num_rendered_frames());
EXPECT_EQ(kWidth, rotating_renderer->width());
EXPECT_EQ(kHeight, rotating_renderer->height());
EXPECT_EQ(&frame, rotating_renderer->last_frame());
// Add 2nd renderer which doesn't support rotation.
rtc::scoped_ptr<FakeVideoTrackRenderer> non_rotating_renderer(
new FakeVideoTrackRenderer(video_track_.get(), false));
// Render the same 90 degree frame.
renderer_input->RenderFrame(&frame);
// rotating_renderer should see the same frame.
EXPECT_EQ(kWidth, rotating_renderer->width());
EXPECT_EQ(kHeight, rotating_renderer->height());
EXPECT_EQ(&frame, rotating_renderer->last_frame());
// non_rotating_renderer should see the frame rotated.
EXPECT_EQ(kHeight, non_rotating_renderer->width());
EXPECT_EQ(kWidth, non_rotating_renderer->height());
EXPECT_NE(&frame, non_rotating_renderer->last_frame());
// Render the same 90 degree frame the 3rd time.
renderer_input->RenderFrame(&frame);
// Now render a frame without rotation.
frame.SetRotation(webrtc::kVideoRotation_0);
renderer_input->RenderFrame(&frame);
// rotating_renderer should still only have 1 setsize.
EXPECT_EQ(kWidth, rotating_renderer->width());
EXPECT_EQ(kHeight, rotating_renderer->height());
EXPECT_EQ(&frame, rotating_renderer->last_frame());
// render_2 should have a new size but should have the same frame.
EXPECT_EQ(kWidth, non_rotating_renderer->width());
EXPECT_EQ(kHeight, non_rotating_renderer->height());
EXPECT_EQ(&frame, non_rotating_renderer->last_frame());
}

View File

@ -26,19 +26,20 @@
*/
#include "talk/app/webrtc/videotrackrenderers.h"
#include "talk/media/base/videoframe.h"
namespace webrtc {
VideoTrackRenderers::VideoTrackRenderers()
: width_(0),
height_(0),
enabled_(true) {
VideoTrackRenderers::VideoTrackRenderers() : enabled_(true) {
}
VideoTrackRenderers::~VideoTrackRenderers() {
}
void VideoTrackRenderers::AddRenderer(VideoRendererInterface* renderer) {
if (!renderer) {
return;
}
rtc::CritScope cs(&critical_section_);
std::vector<RenderObserver>::iterator it = renderers_.begin();
for (; it != renderers_.end(); ++it) {
@ -65,14 +66,6 @@ void VideoTrackRenderers::SetEnabled(bool enable) {
}
bool VideoTrackRenderers::SetSize(int width, int height, int reserved) {
rtc::CritScope cs(&critical_section_);
width_ = width;
height_ = height;
std::vector<RenderObserver>::iterator it = renderers_.begin();
for (; it != renderers_.end(); ++it) {
it->renderer_->SetSize(width, height);
it->size_set_ = true;
}
return true;
}
@ -81,13 +74,14 @@ bool VideoTrackRenderers::RenderFrame(const cricket::VideoFrame* frame) {
if (!enabled_) {
return true;
}
std::vector<RenderObserver>::iterator it = renderers_.begin();
for (; it != renderers_.end(); ++it) {
if (!it->size_set_) {
it->renderer_->SetSize(width_, height_);
it->size_set_ = true;
if (it->can_apply_rotation_) {
it->renderer_->RenderFrame(frame);
} else {
it->renderer_->RenderFrame(frame->GetCopyWithRotationApplied());
}
it->renderer_->RenderFrame(frame);
}
return true;
}

View File

@ -33,6 +33,7 @@
#include "talk/app/webrtc/mediastreaminterface.h"
#include "talk/media/base/videorenderer.h"
#include "webrtc/base/criticalsection.h"
#include "webrtc/base/scoped_ptr.h"
namespace webrtc {
@ -58,14 +59,11 @@ class VideoTrackRenderers : public cricket::VideoRenderer {
struct RenderObserver {
explicit RenderObserver(VideoRendererInterface* renderer)
: renderer_(renderer),
size_set_(false) {
}
can_apply_rotation_(renderer->CanApplyRotation()) {}
VideoRendererInterface* renderer_;
bool size_set_;
bool can_apply_rotation_;
};
int width_;
int height_;
bool enabled_;
std::vector<RenderObserver> renderers_;