zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[iOS] Fix incorrectly oriented frames when rapidly switching between cameras.

Browse Source 
During a call, with both phones in horizontal or landscape mode, rapidly switching between the front and back camera sometimes causes the remote video to be shown upside down.

There seems to be a race condition when setting the rotation based on the orientation of the device and which camera we're using.

So use the active input's camera to check instead of the client state.

BUG=webrtc:7898

Review-Url: https://codereview.webrtc.org/2964703002
Cr-Commit-Position: refs/heads/master@{#19139}
This commit is contained in:
jtteh
2017-07-25 09:35:55 -07:00
committed by Commit Bot
parent 96b69bdbee
commit 61b0ed039d
7 changed files with 822 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
webrtc/sdk/objc/Framework/Classes/Video/AVCaptureSession+Device.h Normal file
View File

@ -0,0 +1,23 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import <AVFoundation/AVFoundation.h>
#import <CoreMedia/CoreMedia.h>
NS_ASSUME_NONNULL_BEGIN
@interface AVCaptureSession (Device)
// Check the image's EXIF for the camera the image came from.
+ (AVCaptureDevicePosition)devicePositionForSampleBuffer:(CMSampleBufferRef)sampleBuffer;
@end
NS_ASSUME_NONNULL_END

51
webrtc/sdk/objc/Framework/Classes/Video/AVCaptureSession+Device.mm Normal file
View File

@ -0,0 +1,51 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#import "AVCaptureSession+Device.h"
BOOL CFStringContainsString(CFStringRef theString, CFStringRef stringToFind) {
return CFStringFindWithOptions(theString,
stringToFind,
CFRangeMake(0, CFStringGetLength(theString)),
kCFCompareCaseInsensitive,
nil);
}
@implementation AVCaptureSession (Device)
+ (AVCaptureDevicePosition)devicePositionForSampleBuffer:(CMSampleBufferRef)sampleBuffer {
// Check the image's EXIF for the camera the image came from.
AVCaptureDevicePosition cameraPosition = AVCaptureDevicePositionUnspecified;
CFDictionaryRef attachments = CMCopyDictionaryOfAttachments(
kCFAllocatorDefault, sampleBuffer, kCMAttachmentMode_ShouldPropagate);
if (attachments) {
int size = CFDictionaryGetCount(attachments);
if (size > 0) {
CFDictionaryRef cfExifDictVal = nil;
if (CFDictionaryGetValueIfPresent(
attachments, (const void *)CFSTR("{Exif}"), (const void **)&cfExifDictVal)) {
CFStringRef cfLensModelStrVal;
if (CFDictionaryGetValueIfPresent(cfExifDictVal,
(const void *)CFSTR("LensModel"),
(const void **)&cfLensModelStrVal)) {
if (CFStringContainsString(cfLensModelStrVal, CFSTR("front"))) {
cameraPosition = AVCaptureDevicePositionFront;
} else if (CFStringContainsString(cfLensModelStrVal, CFSTR("back"))) {
cameraPosition = AVCaptureDevicePositionBack;
}
}
}
}
CFRelease(attachments);
}
return cameraPosition;
}
@end

125
webrtc/sdk/objc/Framework/Classes/Video/RTCAVFoundationVideoCapturerInternal.mm
View File

@ -16,6 +16,7 @@
#import "WebRTC/UIDevice+RTCDevice.h"
#endif
#import "AVCaptureSession+Device.h"
#import "RTCDispatcher+Private.h"
#import "WebRTC/RTCLogging.h"
@ -28,11 +29,13 @@
AVCaptureVideoDataOutput *_videoDataOutput;
// The cricket::VideoCapturer that owns this class. Should never be NULL.
webrtc::AVFoundationVideoCapturer *_capturer;
webrtc::VideoRotation _rotation;
BOOL _hasRetriedOnFatalError;
BOOL _isRunning;
BOOL _hasStarted;
rtc::CriticalSection _crit;
#if TARGET_OS_IPHONE
UIDeviceOrientation _orientation;
#endif
}
@synthesize captureSession = _captureSession;
@ -57,6 +60,7 @@
}
NSNotificationCenter *center = [NSNotificationCenter defaultCenter];
#if TARGET_OS_IPHONE
_orientation = UIDeviceOrientationPortrait;
[center addObserver:self
selector:@selector(deviceOrientationDidChange:)
name:UIDeviceOrientationDidChangeNotification
@ -161,14 +165,6 @@
[RTCDispatcher
dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
#if TARGET_OS_IPHONE
// Default to portrait orientation on iPhone. This will be reset in
// updateOrientation unless orientation is unknown/faceup/facedown.
_rotation = webrtc::kVideoRotation_90;
#else
// No rotation on Mac.
_rotation = webrtc::kVideoRotation_0;
#endif
[self updateOrientation];
#if TARGET_OS_IPHONE
[[UIDevice currentDevice] beginGeneratingDeviceOrientationNotifications];
@ -218,7 +214,47 @@
if (!self.hasStarted) {
return;
}
_capturer->CaptureSampleBuffer(sampleBuffer, _rotation);
#if TARGET_OS_IPHONE
// Default to portrait orientation on iPhone.
webrtc::VideoRotation rotation = webrtc::kVideoRotation_90;
BOOL usingFrontCamera;
// Check the image's EXIF for the camera the image came from as the image could have been
// delayed as we set alwaysDiscardsLateVideoFrames to NO.
AVCaptureDevicePosition cameraPosition =
[AVCaptureSession devicePositionForSampleBuffer:sampleBuffer];
if (cameraPosition != AVCaptureDevicePositionUnspecified) {
usingFrontCamera = AVCaptureDevicePositionFront == cameraPosition;
} else {
AVCaptureDeviceInput *deviceInput =
(AVCaptureDeviceInput *)((AVCaptureInputPort *)connection.inputPorts.firstObject).input;
usingFrontCamera = AVCaptureDevicePositionFront == deviceInput.device.position;
}
switch (_orientation) {
case UIDeviceOrientationPortrait:
rotation = webrtc::kVideoRotation_90;
break;
case UIDeviceOrientationPortraitUpsideDown:
rotation = webrtc::kVideoRotation_270;
break;
case UIDeviceOrientationLandscapeLeft:
rotation = usingFrontCamera ? webrtc::kVideoRotation_180 : webrtc::kVideoRotation_0;
break;
case UIDeviceOrientationLandscapeRight:
rotation = usingFrontCamera ? webrtc::kVideoRotation_0 : webrtc::kVideoRotation_180;
break;
case UIDeviceOrientationFaceUp:
case UIDeviceOrientationFaceDown:
case UIDeviceOrientationUnknown:
// Ignore.
break;
}
#else
// No rotation on Mac.
webrtc::VideoRotation rotation = webrtc::kVideoRotation_0;
#endif
_capturer->CaptureSampleBuffer(sampleBuffer, rotation);
}
- (void)captureOutput:(AVCaptureOutput *)captureOutput
@ -448,56 +484,35 @@
// Called from capture session queue.
- (void)updateOrientation {
#if TARGET_OS_IPHONE
switch ([UIDevice currentDevice].orientation) {
case UIDeviceOrientationPortrait:
_rotation = webrtc::kVideoRotation_90;
break;
case UIDeviceOrientationPortraitUpsideDown:
_rotation = webrtc::kVideoRotation_270;
break;
case UIDeviceOrientationLandscapeLeft:
_rotation =
_capturer->GetUseBackCamera() ? webrtc::kVideoRotation_0 : webrtc::kVideoRotation_180;
break;
case UIDeviceOrientationLandscapeRight:
_rotation =
_capturer->GetUseBackCamera() ? webrtc::kVideoRotation_180 : webrtc::kVideoRotation_0;
break;
case UIDeviceOrientationFaceUp:
case UIDeviceOrientationFaceDown:
case UIDeviceOrientationUnknown:
// Ignore.
break;
}
_orientation = [UIDevice currentDevice].orientation;
#endif
}
// Update the current session input to match what's stored in _useBackCamera.
- (void)updateSessionInputForUseBackCamera:(BOOL)useBackCamera {
[RTCDispatcher dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
[_captureSession beginConfiguration];
AVCaptureDeviceInput *oldInput = _backCameraInput;
AVCaptureDeviceInput *newInput = _frontCameraInput;
if (useBackCamera) {
oldInput = _frontCameraInput;
newInput = _backCameraInput;
}
if (oldInput) {
// Ok to remove this even if it's not attached. Will be no-op.
[_captureSession removeInput:oldInput];
}
if (newInput) {
[_captureSession addInput:newInput];
}
[self updateOrientation];
AVCaptureDevice *newDevice = newInput.device;
const cricket::VideoFormat *format =
_capturer->GetCaptureFormat();
webrtc::SetFormatForCaptureDevice(
newDevice, _captureSession, *format);
[_captureSession commitConfiguration];
}];
[RTCDispatcher
dispatchAsyncOnType:RTCDispatcherTypeCaptureSession
block:^{
[_captureSession beginConfiguration];
AVCaptureDeviceInput *oldInput = _backCameraInput;
AVCaptureDeviceInput *newInput = _frontCameraInput;
if (useBackCamera) {
oldInput = _frontCameraInput;
newInput = _backCameraInput;
}
if (oldInput) {
// Ok to remove this even if it's not attached. Will be no-op.
[_captureSession removeInput:oldInput];
}
if (newInput) {
[_captureSession addInput:newInput];
}
[self updateOrientation];
AVCaptureDevice *newDevice = newInput.device;
const cricket::VideoFormat *format = _capturer->GetCaptureFormat();
webrtc::SetFormatForCaptureDevice(newDevice, _captureSession, *format);
[_captureSession commitConfiguration];
}];
}
@end