From 8245a8542944b1b1c41dba6f61aa2d72cfe79536 Mon Sep 17 00:00:00 2001 From: magjed Date: Wed, 26 Apr 2017 02:02:10 -0700 Subject: [PATCH] ObjC: Split out I420 texture uploading into separate class This CL splits out the code in RTCI420Shader.mm for uploading byte buffers to textures into its own class RTCI420TextureCache. The purpose is to prepare for allowing clients to inject their own shaders. RTCI420TextureCache will be used in the generic code, while the actual shaders in RTCI420Shader will be customizable by the client. BUG=webrtc:7473 Review-Url: https://codereview.webrtc.org/2842453002 Cr-Commit-Position: refs/heads/master@{#17882} --- webrtc/sdk/BUILD.gn | 2 + .../objc/Framework/Classes/RTCI420Shader.mm | 145 +++-------------- .../Framework/Classes/RTCI420TextureCache.h | 24 +++ .../Framework/Classes/RTCI420TextureCache.mm | 149 ++++++++++++++++++ .../objc/Framework/Classes/RTCOpenGLDefines.h | 2 + 5 files changed, 197 insertions(+), 125 deletions(-) create mode 100644 webrtc/sdk/objc/Framework/Classes/RTCI420TextureCache.h create mode 100644 webrtc/sdk/objc/Framework/Classes/RTCI420TextureCache.mm diff --git a/webrtc/sdk/BUILD.gn b/webrtc/sdk/BUILD.gn index a352f9c65a..8aabcf14ee 100644 --- a/webrtc/sdk/BUILD.gn +++ b/webrtc/sdk/BUILD.gn @@ -110,6 +110,8 @@ if (is_ios || is_mac) { "objc/Framework/Classes/RTCDataChannelConfiguration+Private.h", "objc/Framework/Classes/RTCDataChannelConfiguration.mm", "objc/Framework/Classes/RTCI420Shader.mm", + "objc/Framework/Classes/RTCI420TextureCache.h", + "objc/Framework/Classes/RTCI420TextureCache.mm", "objc/Framework/Classes/RTCIceCandidate+Private.h", "objc/Framework/Classes/RTCIceCandidate.mm", "objc/Framework/Classes/RTCIceServer+Private.h", diff --git a/webrtc/sdk/objc/Framework/Classes/RTCI420Shader.mm b/webrtc/sdk/objc/Framework/Classes/RTCI420Shader.mm index 3c3ffa72f0..83ecbe4d67 100644 --- a/webrtc/sdk/objc/Framework/Classes/RTCI420Shader.mm +++ b/webrtc/sdk/objc/Framework/Classes/RTCI420Shader.mm @@ -10,22 +10,13 @@ #import "RTCShader.h" -#include - +#import "RTCI420TextureCache.h" #import "RTCShader+Private.h" #import "WebRTC/RTCLogging.h" #import "WebRTC/RTCVideoFrame.h" #include "webrtc/base/optional.h" -// |kNumTextures| must not exceed 8, which is the limit in OpenGLES2. Two sets -// of 3 textures are used here, one for each of the Y, U and V planes. Having -// two sets alleviates CPU blockage in the event that the GPU is asked to render -// to a texture that is already in use. -static const GLsizei kNumTextureSets = 2; -static const GLsizei kNumTexturesPerSet = 3; -static const GLsizei kNumTextures = kNumTexturesPerSet * kNumTextureSets; - // Fragment shader converts YUV values from input textures into a final RGB // pixel. The conversion formula is from http://www.fourcc.org/fccyvrgb.php. static const char kI420FragmentShaderSource[] = @@ -50,10 +41,8 @@ static const char kI420FragmentShaderSource[] = " }\n"; @implementation RTCI420Shader { - BOOL _hasUnpackRowLength; - GLint _currentTextureSet; + RTCI420TextureCache* textureCache; // Handles for OpenGL constructs. - GLuint _textures[kNumTextures]; GLuint _i420Program; GLuint _vertexArray; GLuint _vertexBuffer; @@ -63,20 +52,13 @@ static const char kI420FragmentShaderSource[] = // Store current rotation and only upload new vertex data when rotation // changes. rtc::Optional _currentRotation; - // Used to create a non-padded plane for GPU upload when we receive padded - // frames. - std::vector _planeBuffer; } - (instancetype)initWithContext:(GlContextType *)context { if (self = [super init]) { -#if TARGET_OS_IPHONE - _hasUnpackRowLength = (context.API == kEAGLRenderingAPIOpenGLES3); -#else - _hasUnpackRowLength = YES; -#endif + textureCache = [[RTCI420TextureCache alloc] initWithContext:context]; glPixelStorei(GL_UNPACK_ALIGNMENT, 1); - if (![self setupI420Program] || ![self setupTextures] || + if (![self setupI420Program] || !RTCSetupVerticesForProgram(_i420Program, &_vertexBuffer, &_vertexArray)) { RTCLog(@"Failed to initialize RTCI420Shader."); self = nil; @@ -87,7 +69,6 @@ static const char kI420FragmentShaderSource[] = - (void)dealloc { glDeleteProgram(_i420Program); - glDeleteTextures(kNumTextures, _textures); glDeleteBuffers(1, &_vertexBuffer); glDeleteVertexArrays(1, &_vertexArray); } @@ -104,27 +85,27 @@ static const char kI420FragmentShaderSource[] = return (_ySampler >= 0 && _uSampler >= 0 && _vSampler >= 0); } -- (BOOL)setupTextures { - glGenTextures(kNumTextures, _textures); - // Set parameters for each of the textures we created. - for (GLsizei i = 0; i < kNumTextures; i++) { - glBindTexture(GL_TEXTURE_2D, _textures[i]); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - } - return YES; -} - - (BOOL)drawFrame:(RTCVideoFrame*)frame { glUseProgram(_i420Program); - if (![self updateTextureDataForFrame:frame]) { - return NO; - } + + [textureCache uploadFrameToTextures:frame]; + #if !TARGET_OS_IPHONE glBindVertexArray(_vertexArray); #endif + + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, textureCache.yTexture); + glUniform1i(_ySampler, 0); + + glActiveTexture(GL_TEXTURE1); + glBindTexture(GL_TEXTURE_2D, textureCache.uTexture); + glUniform1i(_uSampler, 1); + + glActiveTexture(GL_TEXTURE2); + glBindTexture(GL_TEXTURE_2D, textureCache.vTexture); + glUniform1i(_vSampler, 2); + glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); if (!_currentRotation || frame.rotation != *_currentRotation) { _currentRotation = rtc::Optional(frame.rotation); @@ -135,90 +116,4 @@ static const char kI420FragmentShaderSource[] = return YES; } -- (void)uploadPlane:(const uint8_t *)plane - sampler:(GLint)sampler - offset:(GLint)offset - width:(size_t)width - height:(size_t)height - stride:(int32_t)stride { - glActiveTexture(static_cast(GL_TEXTURE0 + offset)); - glBindTexture(GL_TEXTURE_2D, _textures[offset]); - - // When setting texture sampler uniforms, the texture index is used not - // the texture handle. - glUniform1i(sampler, offset); - const uint8_t *uploadPlane = plane; - if ((size_t)stride != width) { - if (_hasUnpackRowLength) { - // GLES3 allows us to specify stride. - glPixelStorei(GL_UNPACK_ROW_LENGTH, stride); - glTexImage2D(GL_TEXTURE_2D, - 0, - RTC_PIXEL_FORMAT, - static_cast(width), - static_cast(height), - 0, - RTC_PIXEL_FORMAT, - GL_UNSIGNED_BYTE, - uploadPlane); - glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); - return; - } else { - // Make an unpadded copy and upload that instead. Quick profiling showed - // that this is faster than uploading row by row using glTexSubImage2D. - uint8_t *unpaddedPlane = _planeBuffer.data(); - for (size_t y = 0; y < height; ++y) { - memcpy(unpaddedPlane + y * width, plane + y * stride, width); - } - uploadPlane = unpaddedPlane; - } - } - glTexImage2D(GL_TEXTURE_2D, - 0, - RTC_PIXEL_FORMAT, - static_cast(width), - static_cast(height), - 0, - RTC_PIXEL_FORMAT, - GL_UNSIGNED_BYTE, - uploadPlane); -} - -- (BOOL)updateTextureDataForFrame:(RTCVideoFrame *)frame { - GLint textureOffset = _currentTextureSet * 3; - NSAssert(textureOffset + 3 <= kNumTextures, @"invalid offset"); - - const int chromaWidth = (frame.width + 1) / 2; - const int chromaHeight = (frame.height + 1) / 2; - if (frame.strideY != frame.width || - frame.strideU != chromaWidth || - frame.strideV != chromaWidth) { - _planeBuffer.resize(frame.width * frame.height); - } - - [self uploadPlane:frame.dataY - sampler:_ySampler - offset:textureOffset - width:frame.width - height:frame.height - stride:frame.strideY]; - - [self uploadPlane:frame.dataU - sampler:_uSampler - offset:textureOffset + 1 - width:chromaWidth - height:chromaHeight - stride:frame.strideU]; - - [self uploadPlane:frame.dataV - sampler:_vSampler - offset:textureOffset + 2 - width:chromaWidth - height:chromaHeight - stride:frame.strideV]; - - _currentTextureSet = (_currentTextureSet + 1) % kNumTextureSets; - return YES; -} - @end diff --git a/webrtc/sdk/objc/Framework/Classes/RTCI420TextureCache.h b/webrtc/sdk/objc/Framework/Classes/RTCI420TextureCache.h new file mode 100644 index 0000000000..5ee657c61f --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/RTCI420TextureCache.h @@ -0,0 +1,24 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCOpenGLDefines.h" +#import "WebRTC/RTCVideoFrame.h" + +@interface RTCI420TextureCache : NSObject + +@property(nonatomic, readonly) GLuint yTexture; +@property(nonatomic, readonly) GLuint uTexture; +@property(nonatomic, readonly) GLuint vTexture; + +- (instancetype)initWithContext:(GlContextType *)context; + +- (void)uploadFrameToTextures:(RTCVideoFrame *)frame; + +@end diff --git a/webrtc/sdk/objc/Framework/Classes/RTCI420TextureCache.mm b/webrtc/sdk/objc/Framework/Classes/RTCI420TextureCache.mm new file mode 100644 index 0000000000..d263c75e93 --- /dev/null +++ b/webrtc/sdk/objc/Framework/Classes/RTCI420TextureCache.mm @@ -0,0 +1,149 @@ +/* + * Copyright 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#import "RTCI420TextureCache.h" + +#import "RTCShader+Private.h" + +#include + +// Two sets of 3 textures are used here, one for each of the Y, U and V planes. Having two sets +// alleviates CPU blockage in the event that the GPU is asked to render to a texture that is already +// in use. +static const GLsizei kNumTextureSets = 2; +static const GLsizei kNumTexturesPerSet = 3; +static const GLsizei kNumTextures = kNumTexturesPerSet * kNumTextureSets; + +@implementation RTCI420TextureCache { + BOOL _hasUnpackRowLength; + GLint _currentTextureSet; + // Handles for OpenGL constructs. + GLuint _textures[kNumTextures]; + // Used to create a non-padded plane for GPU upload when we receive padded frames. + std::vector _planeBuffer; +} + +- (GLuint)yTexture { + return _textures[_currentTextureSet * kNumTexturesPerSet]; +} + +- (GLuint)uTexture { + return _textures[_currentTextureSet * kNumTexturesPerSet + 1]; +} + +- (GLuint)vTexture { + return _textures[_currentTextureSet * kNumTexturesPerSet + 2]; +} + +- (instancetype)initWithContext:(GlContextType *)context { + if (self = [super init]) { +#if TARGET_OS_IPHONE + _hasUnpackRowLength = (context.API == kEAGLRenderingAPIOpenGLES3); +#else + _hasUnpackRowLength = YES; +#endif + glPixelStorei(GL_UNPACK_ALIGNMENT, 1); + + [self setupTextures]; + } + return self; +} + +- (void)dealloc { + glDeleteTextures(kNumTextures, _textures); +} + +- (void)setupTextures { + glGenTextures(kNumTextures, _textures); + // Set parameters for each of the textures we created. + for (GLsizei i = 0; i < kNumTextures; i++) { + glBindTexture(GL_TEXTURE_2D, _textures[i]); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + } +} + +- (void)uploadPlane:(const uint8_t *)plane + texture:(GLuint)texture + width:(size_t)width + height:(size_t)height + stride:(int32_t)stride { + glBindTexture(GL_TEXTURE_2D, texture); + + const uint8_t *uploadPlane = plane; + if ((size_t)stride != width) { + if (_hasUnpackRowLength) { + // GLES3 allows us to specify stride. + glPixelStorei(GL_UNPACK_ROW_LENGTH, stride); + glTexImage2D(GL_TEXTURE_2D, + 0, + RTC_PIXEL_FORMAT, + static_cast(width), + static_cast(height), + 0, + RTC_PIXEL_FORMAT, + GL_UNSIGNED_BYTE, + uploadPlane); + glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); + return; + } else { + // Make an unpadded copy and upload that instead. Quick profiling showed + // that this is faster than uploading row by row using glTexSubImage2D. + uint8_t *unpaddedPlane = _planeBuffer.data(); + for (size_t y = 0; y < height; ++y) { + memcpy(unpaddedPlane + y * width, plane + y * stride, width); + } + uploadPlane = unpaddedPlane; + } + } + glTexImage2D(GL_TEXTURE_2D, + 0, + RTC_PIXEL_FORMAT, + static_cast(width), + static_cast(height), + 0, + RTC_PIXEL_FORMAT, + GL_UNSIGNED_BYTE, + uploadPlane); +} + +- (void)uploadFrameToTextures:(RTCVideoFrame *)frame { + _currentTextureSet = (_currentTextureSet + 1) % kNumTextureSets; + + const int chromaWidth = (frame.width + 1) / 2; + const int chromaHeight = (frame.height + 1) / 2; + if (frame.strideY != frame.width || + frame.strideU != chromaWidth || + frame.strideV != chromaWidth) { + _planeBuffer.resize(frame.width * frame.height); + } + + [self uploadPlane:frame.dataY + texture:self.yTexture + width:frame.width + height:frame.height + stride:frame.strideY]; + + [self uploadPlane:frame.dataU + texture:self.uTexture + width:chromaWidth + height:chromaHeight + stride:frame.strideU]; + + [self uploadPlane:frame.dataV + texture:self.vTexture + width:chromaWidth + height:chromaHeight + stride:frame.strideV]; +} + +@end diff --git a/webrtc/sdk/objc/Framework/Classes/RTCOpenGLDefines.h b/webrtc/sdk/objc/Framework/Classes/RTCOpenGLDefines.h index 1ef642bb81..4088535861 100644 --- a/webrtc/sdk/objc/Framework/Classes/RTCOpenGLDefines.h +++ b/webrtc/sdk/objc/Framework/Classes/RTCOpenGLDefines.h @@ -8,6 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ +#import + #if TARGET_OS_IPHONE #define RTC_PIXEL_FORMAT GL_LUMINANCE #define SHADER_VERSION