zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Delete AndroidVideoCapturer::FrameFactory.

Browse Source 
Splits VideoCapturer::OnFrameCaptured into helper methods,
which enables use of the VideoAdaptation logic without
using a frame factory.

Refactors AndroidVideoCapturer to make adaptation decision
earlier, so we can crop and rotate using
NV12ToI420Rotate.

BUG=webrtc:5682

Review-Url: https://codereview.webrtc.org/1973873003
Cr-Commit-Position: refs/heads/master@{#12895}
This commit is contained in:
nisse
2016-05-25 08:47:01 -07:00
committed by Commit bot
parent 612c25e7af
commit 47ac4620c8
14 changed files with 438 additions and 394 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
webrtc/api/androidvideocapturer.cc
View File

@ -19,103 +19,10 @@
namespace webrtc {
// A hack for avoiding deep frame copies in
// cricket::VideoCapturer.SignalFrameCaptured() using a custom FrameFactory.
// A frame is injected using UpdateCapturedFrame(), and converted into a
// cricket::VideoFrame with CreateAliasedFrame(). UpdateCapturedFrame() should
// be called before CreateAliasedFrame() for every frame.
// TODO(magjed): Add an interface cricket::VideoCapturer::OnFrameCaptured()
// for ref counted I420 frames instead of this hack.
class AndroidVideoCapturer::FrameFactory : public cricket::VideoFrameFactory {
public:
explicit FrameFactory(
const rtc::scoped_refptr<AndroidVideoCapturerDelegate>& delegate)
: delegate_(delegate) {
// Create a CapturedFrame that only contains header information, not the
// actual pixel data.
captured_frame_.pixel_height = 1;
captured_frame_.pixel_width = 1;
captured_frame_.data = nullptr;
captured_frame_.data_size = cricket::CapturedFrame::kUnknownDataSize;
captured_frame_.fourcc = static_cast<uint32_t>(cricket::FOURCC_ANY);
}
void UpdateCapturedFrame(
const rtc::scoped_refptr<webrtc::VideoFrameBuffer>& buffer,
int rotation,
int64_t time_stamp_in_ns) {
RTC_DCHECK(rotation == 0 || rotation == 90 || rotation == 180 ||
rotation == 270);
buffer_ = buffer;
captured_frame_.width = buffer->width();
captured_frame_.height = buffer->height();
captured_frame_.time_stamp = time_stamp_in_ns;
captured_frame_.rotation = static_cast<webrtc::VideoRotation>(rotation);
}
void ClearCapturedFrame() {
buffer_ = nullptr;
captured_frame_.width = 0;
captured_frame_.height = 0;
captured_frame_.time_stamp = 0;
}
const cricket::CapturedFrame* GetCapturedFrame() const {
return &captured_frame_;
}
cricket::VideoFrame* CreateAliasedFrame(
const cricket::CapturedFrame* captured_frame,
int dst_width,
int dst_height) const override {
// Check that captured_frame is actually our frame.
RTC_CHECK(captured_frame == &captured_frame_);
RTC_CHECK(buffer_->native_handle() == nullptr);
std::unique_ptr<cricket::VideoFrame> frame(new cricket::WebRtcVideoFrame(
ShallowCenterCrop(buffer_, dst_width, dst_height),
captured_frame->time_stamp, captured_frame->rotation));
// Caller takes ownership.
// TODO(magjed): Change CreateAliasedFrame() to return a std::unique_ptr.
return apply_rotation_ ? frame->GetCopyWithRotationApplied()->Copy()
: frame.release();
}
cricket::VideoFrame* CreateAliasedFrame(
const cricket::CapturedFrame* input_frame,
int cropped_input_width,
int cropped_input_height,
int output_width,
int output_height) const override {
if (buffer_->native_handle() != nullptr) {
rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled_buffer(
static_cast<webrtc_jni::AndroidTextureBuffer*>(buffer_.get())
->CropScaleAndRotate(cropped_input_width, cropped_input_height,
output_width, output_height,
apply_rotation_ ? input_frame->rotation
: webrtc::kVideoRotation_0));
return new cricket::WebRtcVideoFrame(
scaled_buffer, input_frame->time_stamp,
apply_rotation_ ? webrtc::kVideoRotation_0 : input_frame->rotation);
}
return VideoFrameFactory::CreateAliasedFrame(input_frame,
cropped_input_width,
cropped_input_height,
output_width,
output_height);
}
private:
rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer_;
cricket::CapturedFrame captured_frame_;
rtc::scoped_refptr<AndroidVideoCapturerDelegate> delegate_;
};
AndroidVideoCapturer::AndroidVideoCapturer(
const rtc::scoped_refptr<AndroidVideoCapturerDelegate>& delegate)
: running_(false),
delegate_(delegate),
frame_factory_(NULL),
current_state_(cricket::CS_STOPPED) {
thread_checker_.DetachFromThread();
SetSupportedFormats(delegate_->GetSupportedFormats());
@ -133,9 +40,6 @@ cricket::CaptureState AndroidVideoCapturer::Start(
LOG(LS_INFO) << " AndroidVideoCapturer::Start " << capture_format.width << "x"
<< capture_format.height << "@" << fps;
frame_factory_ = new AndroidVideoCapturer::FrameFactory(delegate_.get());
set_frame_factory(frame_factory_);
running_ = true;
delegate_->Start(capture_format.width, capture_format.height, fps, this);
SetCaptureFormat(&capture_format);
@ -176,19 +80,6 @@ void AndroidVideoCapturer::OnCapturerStarted(bool success) {
SetCaptureState(new_state);
}
void AndroidVideoCapturer::OnIncomingFrame(
const rtc::scoped_refptr<webrtc::VideoFrameBuffer>& buffer,
int rotation,
int64_t time_stamp) {
// NOTE: The frame_factory hack isn't thread safe. It works because
// all calls to this method are from the same Java thread. In
// addition, calls are currently syncronized on the caller's
// AndroidVideoCapturerJni:capturer_lock_.
frame_factory_->UpdateCapturedFrame(buffer, rotation, time_stamp);
SignalFrameCaptured(this, frame_factory_->GetCapturedFrame());
frame_factory_->ClearCapturedFrame();
}
void AndroidVideoCapturer::OnOutputFormatRequest(
int width, int height, int fps) {
RTC_CHECK(thread_checker_.CalledOnValidThread());

15
webrtc/api/androidvideocapturer.h
View File

@ -48,13 +48,6 @@ class AndroidVideoCapturer : public cricket::VideoCapturer {
// Called from JNI when the capturer has been started.
void OnCapturerStarted(bool success);
// Called from JNI when a new frame has been captured.
// Argument |buffer| is intentionally by value, for use with rtc::Bind.
void OnIncomingFrame(
const rtc::scoped_refptr<webrtc::VideoFrameBuffer>& buffer,
int rotation,
int64_t time_stamp);
// Called from JNI to request a new video format.
void OnOutputFormatRequest(int width, int height, int fps);
@ -64,6 +57,11 @@ class AndroidVideoCapturer : public cricket::VideoCapturer {
bool GetBestCaptureFormat(const cricket::VideoFormat& desired,
cricket::VideoFormat* best_format) override;
// Expose these protected methods as public, to be used by the
// AndroidVideoCapturerJni.
using VideoCapturer::AdaptFrame;
using VideoCapturer::OnFrame;
private:
// cricket::VideoCapturer implementation.
// Video frames will be delivered using
@ -80,9 +78,6 @@ class AndroidVideoCapturer : public cricket::VideoCapturer {
rtc::ThreadChecker thread_checker_;
class FrameFactory;
FrameFactory* frame_factory_; // Owned by cricket::VideoCapturer.
cricket::CaptureState current_state_;
};

6
webrtc/api/java/jni/androidmediaencoder_jni.cc
View File

@ -683,7 +683,7 @@ int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled_buffer(
static_cast<AndroidTextureBuffer*>(
frame.video_frame_buffer().get())->CropScaleAndRotate(
frame.width(), frame.height(),
frame.width(), frame.height(), 0, 0,
scaled_resolution.width, scaled_resolution.height,
webrtc::kVideoRotation_0));
input_frame.set_video_frame_buffer(scaled_buffer);
@ -824,9 +824,7 @@ bool MediaCodecVideoEncoder::EncodeTextureOnCodecThread(JNIEnv* jni,
RTC_CHECK(use_surface_);
NativeHandleImpl* handle = static_cast<NativeHandleImpl*>(
frame.video_frame_buffer()->native_handle());
jfloatArray sampling_matrix = jni->NewFloatArray(16);
jni->SetFloatArrayRegion(sampling_matrix, 0, 16, handle->sampling_matrix);
jfloatArray sampling_matrix = handle->sampling_matrix.ToJava(jni);
bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
j_encode_texture_method_,
key_frame,

132
webrtc/api/java/jni/androidvideocapturer_jni.cc
View File

@ -13,6 +13,7 @@
#include "webrtc/api/java/jni/native_handle_impl.h"
#include "webrtc/api/java/jni/surfacetexturehelper_jni.h"
#include "third_party/libyuv/include/libyuv/convert.h"
#include "third_party/libyuv/include/libyuv/scale.h"
#include "webrtc/base/bind.h"
namespace webrtc_jni {
@ -169,25 +170,79 @@ void AndroidVideoCapturerJni::OnMemoryBufferFrame(void* video_frame,
int height,
int rotation,
int64_t timestamp_ns) {
const uint8_t* y_plane = static_cast<uint8_t*>(video_frame);
const uint8_t* vu_plane = y_plane + width * height;
rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer =
buffer_pool_.CreateBuffer(width, height);
libyuv::NV21ToI420(
y_plane, width,
vu_plane, width,
buffer->MutableData(webrtc::kYPlane), buffer->stride(webrtc::kYPlane),
buffer->MutableData(webrtc::kUPlane), buffer->stride(webrtc::kUPlane),
buffer->MutableData(webrtc::kVPlane), buffer->stride(webrtc::kVPlane),
width, height);
RTC_DCHECK(rotation == 0 || rotation == 90 || rotation == 180 ||
rotation == 270);
rtc::CritScope cs(&capturer_lock_);
if (!capturer_) {
LOG(LS_WARNING) << "OnMemoryBufferFrame() called for closed capturer.";
int adapted_width;
int adapted_height;
int crop_width;
int crop_height;
int crop_x;
int crop_y;
if (!capturer_->AdaptFrame(width, height, timestamp_ns,
&adapted_width, &adapted_height,
&crop_width, &crop_height, &crop_x, &crop_y)) {
return;
}
capturer_->OnIncomingFrame(buffer, rotation, timestamp_ns);
int rotated_width = crop_width;
int rotated_height = crop_height;
if (capturer_->apply_rotation() && (rotation == 90 || rotation == 270)) {
std::swap(adapted_width, adapted_height);
std::swap(rotated_width, rotated_height);
}
rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer =
pre_scale_pool_.CreateBuffer(rotated_width, rotated_height);
const uint8_t* y_plane = static_cast<const uint8_t*>(video_frame);
const uint8_t* uv_plane = y_plane + width * height;
// Can only crop at even pixels.
crop_x &= ~1;
crop_y &= ~1;
int uv_width = (width + 1) / 2;
libyuv::NV12ToI420Rotate(
y_plane + width * crop_y + crop_x, width,
uv_plane + uv_width * crop_y + crop_x, width,
buffer->MutableData(webrtc::kYPlane), buffer->stride(webrtc::kYPlane),
// Swap U and V, since we have NV21, not NV12.
buffer->MutableData(webrtc::kVPlane), buffer->stride(webrtc::kVPlane),
buffer->MutableData(webrtc::kUPlane), buffer->stride(webrtc::kUPlane),
crop_width, crop_height, static_cast<libyuv::RotationMode>(
capturer_->apply_rotation() ? rotation : 0));
if (adapted_width != rotated_width || adapted_height != rotated_height) {
rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled =
post_scale_pool_.CreateBuffer(adapted_width, adapted_height);
// TODO(nisse): This should be done by some Scale method in
// I420Buffer, but we can't do that right now, since
// I420BufferPool uses a wrapper object.
if (libyuv::I420Scale(buffer->DataY(), buffer->StrideY(),
buffer->DataU(), buffer->StrideU(),
buffer->DataV(), buffer->StrideV(),
rotated_width, rotated_height,
scaled->MutableDataY(), scaled->StrideY(),
scaled->MutableDataU(), scaled->StrideU(),
scaled->MutableDataV(), scaled->StrideV(),
adapted_width, adapted_height,
libyuv::kFilterBox) < 0) {
LOG(LS_WARNING) << "I420Scale failed";
return;
}
buffer = scaled;
}
// TODO(nisse): Use microsecond time instead.
capturer_->OnFrame(cricket::WebRtcVideoFrame(
buffer, timestamp_ns,
capturer_->apply_rotation()
? webrtc::kVideoRotation_0
: static_cast<webrtc::VideoRotation>(rotation)),
width, height);
}
void AndroidVideoCapturerJni::OnTextureFrame(int width,
@ -195,15 +250,48 @@ void AndroidVideoCapturerJni::OnTextureFrame(int width,
int rotation,
int64_t timestamp_ns,
const NativeHandleImpl& handle) {
rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer(
surface_texture_helper_->CreateTextureFrame(width, height, handle));
RTC_DCHECK(rotation == 0 || rotation == 90 || rotation == 180 ||
rotation == 270);
rtc::CritScope cs(&capturer_lock_);
if (!capturer_) {
LOG(LS_WARNING) << "OnTextureFrame() called for closed capturer.";
int adapted_width;
int adapted_height;
int crop_width;
int crop_height;
int crop_x;
int crop_y;
if (!capturer_->AdaptFrame(width, height, timestamp_ns,
&adapted_width, &adapted_height,
&crop_width, &crop_height, &crop_x, &crop_y)) {
return;
}
capturer_->OnIncomingFrame(buffer, rotation, timestamp_ns);
Matrix matrix = handle.sampling_matrix;
matrix.Crop(crop_width / static_cast<float>(width),
crop_height / static_cast<float>(height),
crop_x / static_cast<float>(width),
crop_y / static_cast<float>(height));
if (capturer_->apply_rotation()) {
if (rotation == webrtc::kVideoRotation_90 ||
rotation == webrtc::kVideoRotation_270) {
std::swap(adapted_width, adapted_height);
}
matrix.Rotate(static_cast<webrtc::VideoRotation>(rotation));
}
// TODO(nisse): Use microsecond time instead.
capturer_->OnFrame(
cricket::WebRtcVideoFrame(
surface_texture_helper_->CreateTextureFrame(
adapted_width, adapted_height,
NativeHandleImpl(handle.oes_texture_id, matrix)),
timestamp_ns, capturer_->apply_rotation()
? webrtc::kVideoRotation_0
: static_cast<webrtc::VideoRotation>(rotation)),
width, height);
}
void AndroidVideoCapturerJni::OnOutputFormatRequest(int width,

3
webrtc/api/java/jni/androidvideocapturer_jni.h
View File

@ -77,7 +77,8 @@ class AndroidVideoCapturerJni : public webrtc::AndroidVideoCapturerDelegate {
const ScopedGlobalRef<jclass> j_observer_class_;
// Used on the Java thread running the camera.
webrtc::I420BufferPool buffer_pool_;
webrtc::I420BufferPool pre_scale_pool_;
webrtc::I420BufferPool post_scale_pool_;
rtc::scoped_refptr<SurfaceTextureHelper> surface_texture_helper_;
rtc::ThreadChecker thread_checker_;

118
webrtc/api/java/jni/native_handle_impl.cc
View File

@ -21,9 +21,24 @@
using webrtc::NativeHandleBuffer;
namespace {
namespace webrtc_jni {
void RotateMatrix(float a[16], webrtc::VideoRotation rotation) {
Matrix::Matrix(JNIEnv* jni, jfloatArray a) {
RTC_CHECK_EQ(16, jni->GetArrayLength(a));
jfloat* ptr = jni->GetFloatArrayElements(a, nullptr);
for (int i = 0; i < 16; ++i) {
elem_[i] = ptr[i];
}
jni->ReleaseFloatArrayElements(a, ptr, 0);
}
jfloatArray Matrix::ToJava(JNIEnv* jni) {
jfloatArray matrix = jni->NewFloatArray(16);
jni->SetFloatArrayRegion(matrix, 0, 16, elem_);
return matrix;
}
void Matrix::Rotate(webrtc::VideoRotation rotation) {
// Texture coordinates are in the range 0 to 1. The transformation of the last
// row in each rotation matrix is needed for proper translation, e.g, to
// mirror x, we don't replace x by -x, but by 1-x.
@ -32,35 +47,36 @@ void RotateMatrix(float a[16], webrtc::VideoRotation rotation) {
break;
case webrtc::kVideoRotation_90: {
const float ROTATE_90[16] =
{ a[4], a[5], a[6], a[7],
-a[0], -a[1], -a[2], -a[3],
a[8], a[9], a[10], a[11],
a[0] + a[12], a[1] + a[13], a[2] + a[14], a[3] + a[15]};
memcpy(a, ROTATE_90, sizeof(ROTATE_90));
{ elem_[4], elem_[5], elem_[6], elem_[7],
-elem_[0], -elem_[1], -elem_[2], -elem_[3],
elem_[8], elem_[9], elem_[10], elem_[11],
elem_[0] + elem_[12], elem_[1] + elem_[13],
elem_[2] + elem_[14], elem_[3] + elem_[15]};
memcpy(elem_, ROTATE_90, sizeof(elem_));
} break;
case webrtc::kVideoRotation_180: {
const float ROTATE_180[16] =
{ -a[0], -a[1], -a[2], -a[3],
-a[4], -a[5], -a[6], -a[7],
a[8], a[9], a[10], a[11],
a[0] + a[4] + a[12], a[1] +a[5] + a[13], a[2] + a[6] + a[14],
a[3] + a[11]+ a[15]};
memcpy(a, ROTATE_180, sizeof(ROTATE_180));
}
break;
{ -elem_[0], -elem_[1], -elem_[2], -elem_[3],
-elem_[4], -elem_[5], -elem_[6], -elem_[7],
elem_[8], elem_[9], elem_[10], elem_[11],
elem_[0] + elem_[4] + elem_[12], elem_[1] + elem_[5] + elem_[13],
elem_[2] + elem_[6] + elem_[14], elem_[3] + elem_[11]+ elem_[15]};
memcpy(elem_, ROTATE_180, sizeof(elem_));
} break;
case webrtc::kVideoRotation_270: {
const float ROTATE_270[16] =
{ -a[4], -a[5], -a[6], -a[7],
a[0], a[1], a[2], a[3],
a[8], a[9], a[10], a[11],
a[4] + a[12], a[5] + a[13], a[6] + a[14], a[7] + a[15]};
memcpy(a, ROTATE_270, sizeof(ROTATE_270));
{ -elem_[4], -elem_[5], -elem_[6], -elem_[7],
elem_[0], elem_[1], elem_[2], elem_[3],
elem_[8], elem_[9], elem_[10], elem_[11],
elem_[4] + elem_[12], elem_[5] + elem_[13],
elem_[6] + elem_[14], elem_[7] + elem_[15]};
memcpy(elem_, ROTATE_270, sizeof(elem_));
} break;
}
}
// Calculates result = a * b, in column-major order.
void MultiplyMatrix(const float a[16], const float b[16], float result[16]) {
void Matrix::Multiply(const float a[16], const float b[16], float result[16]) {
for (int i = 0; i < 4; ++i) {
for (int j = 0; j < 4; ++j) {
float sum = 0;
@ -75,40 +91,30 @@ void MultiplyMatrix(const float a[16], const float b[16], float result[16]) {
// Center crop by keeping xFraction of the width and yFraction of the height,
// so e.g. cropping from 640x480 to 640x360 would use
// xFraction=1, yFraction=360/480.
void CropMatrix(float a[16], float xFraction, float yFraction) {
// Move cropped area to the center of the frame by offsetting half the
// removed area.
const float xOffset = (1 - xFraction) / 2;
const float yOffset = (1 - yFraction) / 2;
const float crop_matrix[16] = {
xFraction, 0, 0, 0,
0, yFraction, 0, 0,
0, 0, 1, 0,
xOffset, yOffset, 0, 1};
float mul_result[16];
MultiplyMatrix(crop_matrix, a, mul_result);
memcpy(a, mul_result, sizeof(mul_result));
void Matrix::Crop(float xFraction,
float yFraction,
float xOffset,
float yOffset) {
const float crop_matrix[16] =
{xFraction, 0, 0, 0,
0, yFraction, 0, 0,
0, 0, 1, 0,
xOffset, yOffset, 0, 1};
const Matrix old = *this;
Multiply(crop_matrix, old.elem_, this->elem_);
}
} // anonymouse namespace
namespace webrtc_jni {
// Aligning pointer to 64 bytes for improved performance, e.g. use SIMD.
static const int kBufferAlignment = 64;
NativeHandleImpl::NativeHandleImpl(int id, const Matrix& matrix)
: oes_texture_id(id), sampling_matrix(matrix) {}
NativeHandleImpl::NativeHandleImpl(JNIEnv* jni,
jint j_oes_texture_id,
jfloatArray j_transform_matrix)
: oes_texture_id(j_oes_texture_id) {
RTC_CHECK_EQ(16, jni->GetArrayLength(j_transform_matrix));
jfloat* transform_matrix_ptr =
jni->GetFloatArrayElements(j_transform_matrix, nullptr);
for (int i = 0; i < 16; ++i) {
sampling_matrix[i] = transform_matrix_ptr[i];
}
jni->ReleaseFloatArrayElements(j_transform_matrix, transform_matrix_ptr, 0);
}
: oes_texture_id(j_oes_texture_id),
sampling_matrix(jni, j_transform_matrix) {}
AndroidTextureBuffer::AndroidTextureBuffer(
int width,
@ -162,11 +168,7 @@ AndroidTextureBuffer::NativeToI420Buffer() {
jobject byte_buffer = jni->NewDirectByteBuffer(y_data, size);
// TODO(nisse): Keep java transform matrix around.
jfloatArray sampling_matrix = jni->NewFloatArray(16);
jni->SetFloatArrayRegion(sampling_matrix, 0, 16,
native_handle_.sampling_matrix);
jfloatArray sampling_matrix = native_handle_.sampling_matrix.ToJava(jni);
jni->CallVoidMethod(surface_texture_helper_,
transform_mid,
byte_buffer, width(), height(), stride,
@ -179,6 +181,8 @@ AndroidTextureBuffer::NativeToI420Buffer() {
rtc::scoped_refptr<AndroidTextureBuffer>
AndroidTextureBuffer::CropScaleAndRotate(int cropped_width,
int cropped_height,
int crop_x,
int crop_y,
int dst_width,
int dst_height,
webrtc::VideoRotation rotation) {
@ -198,11 +202,13 @@ AndroidTextureBuffer::CropScaleAndRotate(int cropped_width,
surface_texture_helper_, rtc::KeepRefUntilDone(this)));
if (cropped_width != width() || cropped_height != height()) {
CropMatrix(buffer->native_handle_.sampling_matrix,
cropped_width / static_cast<float>(width()),
cropped_height / static_cast<float>(height()));
buffer->native_handle_.sampling_matrix.Crop(
cropped_width / static_cast<float>(width()),
cropped_height / static_cast<float>(height()),
crop_x / static_cast<float>(width()),
crop_y / static_cast<float>(height()));
}
RotateMatrix(buffer->native_handle_.sampling_matrix, rotation);
buffer->native_handle_.sampling_matrix.Rotate(rotation);
return buffer;
}

27
webrtc/api/java/jni/native_handle_impl.h
View File

@ -18,14 +18,37 @@
namespace webrtc_jni {
// Open gl texture matrix, in column-major order. Operations are
// in-place.
class Matrix {
public:
Matrix(JNIEnv* jni, jfloatArray a);
jfloatArray ToJava(JNIEnv* jni);
// Crop arguments are relative to original size.
void Crop(float cropped_width,
float cropped_height,
float crop_x,
float crop_y);
void Rotate(webrtc::VideoRotation rotation);
private:
static void Multiply(const float a[16], const float b[16], float result[16]);
float elem_[16];
};
// Wrapper for texture object.
struct NativeHandleImpl {
NativeHandleImpl(JNIEnv* jni,
jint j_oes_texture_id,
jfloatArray j_transform_matrix);
NativeHandleImpl(int id, const Matrix& matrix);
const int oes_texture_id;
float sampling_matrix[16];
Matrix sampling_matrix;
};
class AndroidTextureBuffer : public webrtc::NativeHandleBuffer {
@ -42,6 +65,8 @@ class AndroidTextureBuffer : public webrtc::NativeHandleBuffer {
rtc::scoped_refptr<AndroidTextureBuffer> CropScaleAndRotate(
int cropped_width,
int cropped_height,
int crop_x,
int crop_y,
int dst_width,
int dst_height,
webrtc::VideoRotation rotation);

4
webrtc/api/java/jni/peerconnection_jni.cc
View File

@ -794,8 +794,8 @@ class JavaVideoRendererWrapper
jobject CricketToJavaTextureFrame(const cricket::VideoFrame* frame) {
NativeHandleImpl* handle = reinterpret_cast<NativeHandleImpl*>(
frame->video_frame_buffer()->native_handle());
jfloatArray sampling_matrix = jni()->NewFloatArray(16);
jni()->SetFloatArrayRegion(sampling_matrix, 0, 16, handle->sampling_matrix);
jfloatArray sampling_matrix = handle->sampling_matrix.ToJava(jni());
return jni()->NewObject(
*j_frame_class_, j_texture_frame_ctor_id_,
frame->width(), frame->height(),

10
webrtc/media/base/videoadapter.cc
View File

@ -153,7 +153,7 @@ bool VideoAdapter::KeepFrame(int64_t in_timestamp_ns) {
return true;
}
void VideoAdapter::AdaptFrameResolution(int in_width,
bool VideoAdapter::AdaptFrameResolution(int in_width,
int in_height,
int64_t in_timestamp_ns,
int* cropped_width,
@ -189,11 +189,7 @@ void VideoAdapter::AdaptFrameResolution(int in_width,
}
// Drop frame.
*cropped_width = 0;
*cropped_height = 0;
*out_width = 0;
*out_height = 0;
return;
return false;
}
// Calculate how the input should be cropped.
@ -250,6 +246,8 @@ void VideoAdapter::AdaptFrameResolution(int in_width,
previous_width_ = *out_width;
previous_height_ = *out_height;
return true;
}
void VideoAdapter::OnOutputFormatRequest(const VideoFormat& format) {

10
webrtc/media/base/videoadapter.h
View File

@ -27,11 +27,11 @@ class VideoAdapter {
VideoAdapter();
virtual ~VideoAdapter();
// Return the adapted resolution given the input resolution. The input
// resolution should first be cropped to the specified resolution, and then
// scaled to the final output resolution. The output resolution will be 0x0 if
// the frame should be dropped.
void AdaptFrameResolution(int in_width,
// Return the adapted resolution and cropping parameters given the
// input resolution. The input frame should first be cropped, then
// scaled to the final output resolution. Returns true if the frame
// should be adapted, and false if it should be dropped.
bool AdaptFrameResolution(int in_width,
int in_height,
int64_t in_timestamp_ns,
int* cropped_width,

283
webrtc/media/base/videoadapter_unittest.cc
View File

@ -71,11 +71,10 @@ class VideoAdapterTest : public testing::Test {
int cropped_height;
int out_width;
int out_height;
video_adapter_->AdaptFrameResolution(in_width, in_height,
captured_frame->time_stamp,
&cropped_width, &cropped_height,
&out_width, &out_height);
if (out_width != 0 && out_height != 0) {
if (video_adapter_->AdaptFrameResolution(in_width, in_height,
captured_frame->time_stamp,
&cropped_width, &cropped_height,
&out_width, &out_height)) {
cropped_width_ = cropped_width;
cropped_height_ = cropped_height;
out_width_ = out_width;
@ -390,9 +389,10 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionHighLimit) {
output_format.width *= 10;
output_format.height *= 10;
adapter_.OnOutputFormatRequest(output_format);
adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height,
0, &cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(
capture_format_.width, capture_format_.height, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(capture_format_.width, cropped_width_);
EXPECT_EQ(capture_format_.height, cropped_height_);
EXPECT_EQ(capture_format_.width, out_width_);
@ -403,9 +403,10 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionHighLimit) {
// cropping or resolution change.
TEST_F(VideoAdapterTest, AdaptFrameResolutionIdentical) {
adapter_.OnOutputFormatRequest(capture_format_);
adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height,
0, &cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(
capture_format_.width, capture_format_.height, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(capture_format_.width, cropped_width_);
EXPECT_EQ(capture_format_.height, cropped_height_);
EXPECT_EQ(capture_format_.width, out_width_);
@ -419,9 +420,10 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionQuarter) {
request_format.width /= 2;
request_format.height /= 2;
adapter_.OnOutputFormatRequest(request_format);
adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height,
0, &cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(
capture_format_.width, capture_format_.height, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(capture_format_.width, cropped_width_);
EXPECT_EQ(capture_format_.height, cropped_height_);
EXPECT_EQ(request_format.width, out_width_);
@ -434,11 +436,10 @@ TEST_F(VideoAdapterTest, AdaptFrameResolutionDrop) {
output_format.width = 0;
output_format.height = 0;
adapter_.OnOutputFormatRequest(output_format);
adapter_.AdaptFrameResolution(capture_format_.width, capture_format_.height,
0, &cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(0, out_width_);
EXPECT_EQ(0, out_height_);
EXPECT_FALSE(adapter_.AdaptFrameResolution(
capture_format_.width, capture_format_.height, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
}
// Adapt the frame resolution to be a quarter of the capture resolution at the
@ -503,9 +504,9 @@ TEST_F(VideoAdapterTest, DropAllFrames) {
TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
VideoFormat format(640, 400, 0, 0);
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(400, cropped_height_);
EXPECT_EQ(640, out_width_);
@ -514,9 +515,9 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
// Format request 640x400.
format.height = 400;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(400, cropped_height_);
EXPECT_EQ(640, out_width_);
@ -527,9 +528,9 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
format.width = 1280;
format.height = 720;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(640, out_width_);
@ -539,19 +540,17 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
format.width = 0;
format.height = 0;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(0, out_width_);
EXPECT_EQ(0, out_height_);
EXPECT_FALSE(adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
// Request 320x200. Expect scaling, but no cropping.
format.width = 320;
format.height = 200;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(400, cropped_height_);
EXPECT_EQ(320, out_width_);
@ -563,9 +562,9 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
format.width = 424;
format.height = 265;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(400, cropped_height_);
EXPECT_EQ(320, out_width_);
@ -575,9 +574,9 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
format.width = 640 * 3 / 8;
format.height = 400 * 3 / 8;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(400, cropped_height_);
EXPECT_EQ(640 * 3 / 8, out_width_);
@ -587,9 +586,9 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
format.width = 320;
format.height = 200;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(400, cropped_height_);
EXPECT_EQ(320, out_width_);
@ -599,9 +598,9 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
format.width = 480;
format.height = 300;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 400, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(400, cropped_height_);
EXPECT_EQ(480, out_width_);
@ -611,9 +610,9 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequest) {
TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) {
// Start at HD.
VideoFormat format(1280, 720, 0, 0);
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(1280, out_width_);
@ -623,9 +622,9 @@ TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) {
format.width = 640;
format.height = 360;
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(640, out_width_);
@ -633,9 +632,9 @@ TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) {
// Now, the camera reopens at VGA.
// Both the frame and the output format should be 640x360.
adapter_.AdaptFrameResolution(640, 360, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 360, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(640, out_width_);
@ -644,9 +643,9 @@ TEST_F(VideoAdapterTest, TestViewRequestPlusCameraSwitch) {
// And another view request comes in for 640x360, which should have no
// real impact.
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 360, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 360, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(640, out_width_);
@ -658,9 +657,9 @@ TEST_F(VideoAdapterTest, TestVGAWidth) {
VideoFormat format(640, 360, 0, FOURCC_I420);
adapter_.OnOutputFormatRequest(format);
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
// Expect cropping.
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
@ -668,17 +667,17 @@ TEST_F(VideoAdapterTest, TestVGAWidth) {
EXPECT_EQ(360, out_height_);
// But if frames come in at 640x360, we shouldn't adapt them down.
adapter_.AdaptFrameResolution(640, 360, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 360, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(640, out_width_);
EXPECT_EQ(360, out_height_);
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(640, out_width_);
@ -686,9 +685,9 @@ TEST_F(VideoAdapterTest, TestVGAWidth) {
}
TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(1280, out_width_);
@ -697,9 +696,9 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
// Adapt down one step.
adapter_.OnResolutionRequest(rtc::Optional<int>(1280 * 720 - 1),
rtc::Optional<int>());
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(960, out_width_);
@ -708,9 +707,9 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
// Adapt down one step more.
adapter_.OnResolutionRequest(rtc::Optional<int>(960 * 540 - 1),
rtc::Optional<int>());
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(640, out_width_);
@ -719,9 +718,9 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
// Adapt down one step more.
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 - 1),
rtc::Optional<int>());
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(480, out_width_);
@ -730,9 +729,9 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
// Adapt up one step.
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(480 * 270));
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(640, out_width_);
@ -741,9 +740,9 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
// Adapt up one step more.
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(640 * 360));
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(960, out_width_);
@ -752,9 +751,9 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
// Adapt up one step more.
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(960 * 720));
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(1280, out_width_);
@ -762,28 +761,26 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInSmallSteps) {
}
TEST_F(VideoAdapterTest, TestOnResolutionRequestMaxZero) {
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(1280, out_width_);
EXPECT_EQ(720, out_height_);
adapter_.OnResolutionRequest(rtc::Optional<int>(0), rtc::Optional<int>());
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_EQ(0, out_width_);
EXPECT_EQ(0, out_height_);
EXPECT_FALSE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
}
TEST_F(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) {
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 - 1),
rtc::Optional<int>());
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(480, out_width_);
@ -791,9 +788,9 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) {
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(960 * 720));
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(1280, out_width_);
@ -803,9 +800,9 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestInLargeSteps) {
TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) {
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 - 1),
rtc::Optional<int>());
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(480, out_width_);
@ -813,9 +810,9 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) {
VideoFormat new_format(640, 360, 0, FOURCC_I420);
adapter_.OnOutputFormatRequest(new_format);
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(480, out_width_);
@ -823,9 +820,9 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) {
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(960 * 720));
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(640, out_width_);
@ -833,9 +830,9 @@ TEST_F(VideoAdapterTest, TestOnOutputFormatRequestCapsMaxResolution) {
}
TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) {
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(1280, out_width_);
@ -843,18 +840,18 @@ TEST_F(VideoAdapterTest, TestOnResolutionRequestReset) {
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 - 1),
rtc::Optional<int>());
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(480, out_width_);
EXPECT_EQ(270, out_height_);
adapter_.OnResolutionRequest(rtc::Optional<int>(), rtc::Optional<int>());
adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(1280, 720, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(1280, cropped_width_);
EXPECT_EQ(720, cropped_height_);
EXPECT_EQ(1280, out_width_);
@ -865,9 +862,9 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) {
// Ask for 640x360 (16:9 aspect).
adapter_.OnOutputFormatRequest(VideoFormat(640, 360, 0, FOURCC_I420));
// Send 640x480 (4:3 aspect).
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
// Expect cropping to 16:9 format and no scaling.
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
@ -878,9 +875,9 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) {
adapter_.OnResolutionRequest(rtc::Optional<int>(640 * 360 - 1),
rtc::Optional<int>());
// Expect cropping to 16:9 format and 3/4 scaling.
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(480, out_width_);
@ -890,9 +887,9 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) {
adapter_.OnResolutionRequest(rtc::Optional<int>(480 * 270 - 1),
rtc::Optional<int>());
// Expect cropping to 16:9 format and 1/2 scaling.
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(320, out_width_);
@ -902,9 +899,9 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) {
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(320 * 180));
// Expect cropping to 16:9 format and 3/4 scaling.
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(480, out_width_);
@ -914,9 +911,9 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) {
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(480 * 270));
// Expect cropping to 16:9 format and no scaling.
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(640, out_width_);
@ -926,9 +923,9 @@ TEST_F(VideoAdapterTest, TestCroppingWithResolutionRequest) {
adapter_.OnResolutionRequest(rtc::Optional<int>(),
rtc::Optional<int>(640 * 360));
// Expect cropping to 16:9 format and no scaling.
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
EXPECT_EQ(640, cropped_width_);
EXPECT_EQ(360, cropped_height_);
EXPECT_EQ(640, out_width_);
@ -943,9 +940,9 @@ TEST_F(VideoAdapterTest, TestCroppingOddResolution) {
rtc::Optional<int>());
// Send 640x480 (4:3 aspect).
adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_);
EXPECT_TRUE(adapter_.AdaptFrameResolution(640, 480, 0,
&cropped_width_, &cropped_height_,
&out_width_, &out_height_));
// Instead of getting the exact aspect ratio with cropped resolution 640x360,
// the resolution should be adjusted to get a perfect scale factor instead.

77
webrtc/media/base/videocapturer.cc
View File

@ -214,26 +214,54 @@ void VideoCapturer::OnSinkWantsChanged(const rtc::VideoSinkWants& wants) {
}
}
void VideoCapturer::OnFrameCaptured(VideoCapturer*,
const CapturedFrame* captured_frame) {
bool VideoCapturer::AdaptFrame(int width,
int height,
// TODO(nisse): Switch to us unit.
int64_t capture_time_ns,
int* out_width,
int* out_height,
int* crop_width,
int* crop_height,
int* crop_x,
int* crop_y) {
if (!broadcaster_.frame_wanted()) {
return;
return false;
}
int cropped_width = captured_frame->width;
int cropped_height = captured_frame->height;
int out_width = captured_frame->width;
int out_height = captured_frame->height;
if (enable_video_adapter_ && !IsScreencast()) {
video_adapter_.AdaptFrameResolution(
captured_frame->width, captured_frame->height,
captured_frame->time_stamp,
&cropped_width, &cropped_height,
&out_width, &out_height);
if (out_width == 0 || out_height == 0) {
if (!video_adapter_.AdaptFrameResolution(
width, height, capture_time_ns,
crop_width, crop_height, out_width, out_height)) {
// VideoAdapter dropped the frame.
return;
return false;
}
*crop_x = (width - *crop_width) / 2;
*crop_y = (height - *crop_height) / 2;
} else {
*out_width = width;
*out_height = height;
*crop_width = width;
*crop_height = height;
*crop_x = 0;
*crop_y = 0;
}
return true;
}
void VideoCapturer::OnFrameCaptured(VideoCapturer*,
const CapturedFrame* captured_frame) {
int out_width;
int out_height;
int crop_width;
int crop_height;
int crop_x;
int crop_y;
if (!AdaptFrame(captured_frame->width, captured_frame->height,
captured_frame->time_stamp,
&out_width, &out_height,
&crop_width, &crop_height, &crop_x, &crop_y)) {
return;
}
if (!frame_factory_) {
@ -241,9 +269,10 @@ void VideoCapturer::OnFrameCaptured(VideoCapturer*,
return;
}
// TODO(nisse): Reorganize frame factory methods.
// TODO(nisse): Reorganize frame factory methods. crop_x and crop_y
// are ignored for now.
std::unique_ptr<VideoFrame> adapted_frame(frame_factory_->CreateAliasedFrame(
captured_frame, cropped_width, cropped_height, out_width, out_height));
captured_frame, crop_width, crop_height, out_width, out_height));
if (!adapted_frame) {
// TODO(fbarchard): LOG more information about captured frame attributes.
@ -253,12 +282,14 @@ void VideoCapturer::OnFrameCaptured(VideoCapturer*,
return;
}
OnFrame(this, adapted_frame.get());
UpdateInputSize(captured_frame);
OnFrame(*adapted_frame, captured_frame->width, captured_frame->height);
}
void VideoCapturer::OnFrame(VideoCapturer* capturer, const VideoFrame* frame) {
broadcaster_.OnFrame(*frame);
void VideoCapturer::OnFrame(const VideoFrame& frame,
int orig_width,
int orig_height) {
broadcaster_.OnFrame(frame);
UpdateInputSize(orig_width, orig_height);
}
void VideoCapturer::SetCaptureState(CaptureState state) {
@ -399,13 +430,13 @@ bool VideoCapturer::ShouldFilterFormat(const VideoFormat& format) const {
format.height > max_format_->height;
}
void VideoCapturer::UpdateInputSize(const CapturedFrame* captured_frame) {
void VideoCapturer::UpdateInputSize(int width, int height) {
// Update stats protected from fetches from different thread.
rtc::CritScope cs(&frame_stats_crit_);
input_size_valid_ = true;
input_width_ = captured_frame->width;
input_height_ = captured_frame->height;
input_width_ = width;
input_height_ = height;
}
} // namespace cricket

26
webrtc/media/base/videocapturer.h
View File

@ -168,7 +168,7 @@ class VideoCapturer : public sigslot::has_slots<>,
return capture_state_;
}
virtual bool GetApplyRotation() { return apply_rotation_; }
virtual bool apply_rotation() { return apply_rotation_; }
// Returns true if the capturer is screencasting. This can be used to
// implement screencast specific behavior.
@ -222,14 +222,28 @@ class VideoCapturer : public sigslot::has_slots<>,
// when a sink changes its VideoSinkWants by calling AddOrUpdateSink.
virtual void OnSinkWantsChanged(const rtc::VideoSinkWants& wants);
// Reports the appropriate frame size after adaptation. Returns true
// if a frame is wanted. Returns false if there are no interested
// sinks, or if the VideoAdapter decides to drop the frame.
bool AdaptFrame(int width,
int height,
int64_t capture_time_ns,
int* out_width,
int* out_height,
int* crop_width,
int* crop_height,
int* crop_x,
int* crop_y);
// Callback attached to SignalFrameCaptured where SignalVideoFrames is called.
void OnFrameCaptured(VideoCapturer* video_capturer,
const CapturedFrame* captured_frame);
// Called when a frame has been captured and converted to a VideoFrame.
// OnFrame can be called directly by an implementation that does not use
// SignalFrameCaptured or OnFrameCaptured.
void OnFrame(VideoCapturer* capturer, const VideoFrame* frame);
// Called when a frame has been captured and converted to a
// VideoFrame. OnFrame can be called directly by an implementation
// that does not use SignalFrameCaptured or OnFrameCaptured. The
// orig_width and orig_height are used only to produce stats.
void OnFrame(const VideoFrame& frame, int orig_width, int orig_height);
VideoAdapter* video_adapter() { return &video_adapter_; }
@ -268,7 +282,7 @@ class VideoCapturer : public sigslot::has_slots<>,
// Returns true if format doesn't fulfill all applied restrictions.
bool ShouldFilterFormat(const VideoFormat& format) const;
void UpdateInputSize(const CapturedFrame* captured_frame);
void UpdateInputSize(int width, int height);
rtc::ThreadChecker thread_checker_;
std::string id_;

12
webrtc/media/engine/webrtcvideoengine2_unittest.cc
View File

@ -259,12 +259,12 @@ TEST_F(WebRtcVideoEngine2Test, CVOSetHeaderExtensionBeforeCapturer) {
channel->SetSource(kSsrc, &capturer);
// Verify capturer has turned off applying rotation.
EXPECT_FALSE(capturer.GetApplyRotation());
EXPECT_FALSE(capturer.apply_rotation());
// Verify removing header extension turns on applying rotation.
parameters.extensions.clear();
EXPECT_TRUE(channel->SetSendParameters(parameters));
EXPECT_TRUE(capturer.GetApplyRotation());
EXPECT_TRUE(capturer.apply_rotation());
}
TEST_F(WebRtcVideoEngine2Test, CVOSetHeaderExtensionBeforeAddSendStream) {
@ -290,7 +290,7 @@ TEST_F(WebRtcVideoEngine2Test, CVOSetHeaderExtensionBeforeAddSendStream) {
channel->SetSource(kSsrc, &capturer);
// Verify capturer has turned off applying rotation.
EXPECT_FALSE(capturer.GetApplyRotation());
EXPECT_FALSE(capturer.apply_rotation());
}
TEST_F(WebRtcVideoEngine2Test, CVOSetHeaderExtensionAfterCapturer) {
@ -311,7 +311,7 @@ TEST_F(WebRtcVideoEngine2Test, CVOSetHeaderExtensionAfterCapturer) {
channel->SetSource(kSsrc, &capturer);
// Verify capturer has turned on applying rotation.
EXPECT_TRUE(capturer.GetApplyRotation());
EXPECT_TRUE(capturer.apply_rotation());
// Add CVO extension.
const int id = 1;
@ -322,12 +322,12 @@ TEST_F(WebRtcVideoEngine2Test, CVOSetHeaderExtensionAfterCapturer) {
EXPECT_TRUE(channel->SetSendParameters(parameters));
// Verify capturer has turned off applying rotation.
EXPECT_FALSE(capturer.GetApplyRotation());
EXPECT_FALSE(capturer.apply_rotation());
// Verify removing header extension turns on applying rotation.
parameters.extensions.clear();
EXPECT_TRUE(channel->SetSendParameters(parameters));
EXPECT_TRUE(capturer.GetApplyRotation());
EXPECT_TRUE(capturer.apply_rotation());
}
TEST_F(WebRtcVideoEngine2Test, SetSendFailsBeforeSettingCodecs) {