/* * Copyright 2015 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ package org.webrtc; import android.graphics.Matrix; import android.opengl.GLES20; import java.nio.ByteBuffer; import org.webrtc.VideoFrame.I420Buffer; import org.webrtc.VideoFrame.TextureBuffer; /** * Class for converting OES textures to a YUV ByteBuffer. It can be constructed on any thread, but * should only be operated from a single thread with an active EGL context. */ public class YuvConverter { private static final String FRAGMENT_SHADER = // Difference in texture coordinate corresponding to one // sub-pixel in the x direction. "uniform vec2 xUnit;\n" // Color conversion coefficients, including constant term + "uniform vec4 coeffs;\n" + "\n" + "void main() {\n" // Since the alpha read from the texture is always 1, this could // be written as a mat4 x vec4 multiply. However, that seems to // give a worse framerate, possibly because the additional // multiplies by 1.0 consume resources. TODO(nisse): Could also // try to do it as a vec3 x mat3x4, followed by an add in of a // constant vector. + " gl_FragColor.r = coeffs.a + dot(coeffs.rgb,\n" + " sample(tc - 1.5 * xUnit).rgb);\n" + " gl_FragColor.g = coeffs.a + dot(coeffs.rgb,\n" + " sample(tc - 0.5 * xUnit).rgb);\n" + " gl_FragColor.b = coeffs.a + dot(coeffs.rgb,\n" + " sample(tc + 0.5 * xUnit).rgb);\n" + " gl_FragColor.a = coeffs.a + dot(coeffs.rgb,\n" + " sample(tc + 1.5 * xUnit).rgb);\n" + "}\n"; private static class ShaderCallbacks implements GlGenericDrawer.ShaderCallbacks { // Y'UV444 to RGB888, see https://en.wikipedia.org/wiki/YUV#Y.27UV444_to_RGB888_conversion. We // use the ITU-R coefficients for U and V. private static final float[] yCoeffs = new float[] {0.2987856f, 0.5871095f, 0.1141049f, 0.0f}; private static final float[] uCoeffs = new float[] {-0.168805420f, -0.3317003f, 0.5005057f, 0.5f}; private static final float[] vCoeffs = new float[] {0.4997964f, -0.4184672f, -0.0813292f, 0.5f}; private int xUnitLoc; private int coeffsLoc; private float[] coeffs; private float stepSize; public void setPlaneY() { coeffs = yCoeffs; stepSize = 1.0f; } public void setPlaneU() { coeffs = uCoeffs; stepSize = 2.0f; } public void setPlaneV() { coeffs = vCoeffs; stepSize = 2.0f; } @Override public void onNewShader(GlShader shader) { xUnitLoc = shader.getUniformLocation("xUnit"); coeffsLoc = shader.getUniformLocation("coeffs"); } @Override public void onPrepareShader(GlShader shader, float[] texMatrix, int frameWidth, int frameHeight, int viewportWidth, int viewportHeight) { GLES20.glUniform4fv(coeffsLoc, /* count= */ 1, coeffs, /* offset= */ 0); // Matrix * (1;0;0;0) / (width / stepSize). Note that OpenGL uses column major order. GLES20.glUniform2f( xUnitLoc, stepSize * texMatrix[0] / frameWidth, stepSize * texMatrix[1] / frameWidth); } } private final ThreadUtils.ThreadChecker threadChecker = new ThreadUtils.ThreadChecker(); private final GlTextureFrameBuffer i420TextureFrameBuffer = new GlTextureFrameBuffer(GLES20.GL_RGBA); private boolean released = false; private final ShaderCallbacks shaderCallbacks = new ShaderCallbacks(); private final GlGenericDrawer drawer = new GlGenericDrawer(FRAGMENT_SHADER, shaderCallbacks); /** * This class should be constructed on a thread that has an active EGL context. */ public YuvConverter() { threadChecker.detachThread(); } /** Converts the texture buffer to I420. */ public I420Buffer convert(TextureBuffer inputTextureBuffer) { threadChecker.checkIsOnValidThread(); if (released) { throw new IllegalStateException("YuvConverter.convert called on released object"); } // We draw into a buffer laid out like // // +---------+ // | | // | Y | // | | // | | // +----+----+ // | U | V | // | | | // +----+----+ // // In memory, we use the same stride for all of Y, U and V. The // U data starts at offset |height| * |stride| from the Y data, // and the V data starts at at offset |stride/2| from the U // data, with rows of U and V data alternating. // // Now, it would have made sense to allocate a pixel buffer with // a single byte per pixel (EGL10.EGL_COLOR_BUFFER_TYPE, // EGL10.EGL_LUMINANCE_BUFFER,), but that seems to be // unsupported by devices. So do the following hack: Allocate an // RGBA buffer, of width |stride|/4. To render each of these // large pixels, sample the texture at 4 different x coordinates // and store the results in the four components. // // Since the V data needs to start on a boundary of such a // larger pixel, it is not sufficient that |stride| is even, it // has to be a multiple of 8 pixels. final int frameWidth = inputTextureBuffer.getWidth(); final int frameHeight = inputTextureBuffer.getHeight(); final int stride = ((frameWidth + 7) / 8) * 8; final int uvHeight = (frameHeight + 1) / 2; // Total height of the combined memory layout. final int totalHeight = frameHeight + uvHeight; final ByteBuffer i420ByteBuffer = JniCommon.nativeAllocateByteBuffer(stride * totalHeight); // Viewport width is divided by four since we are squeezing in four color bytes in each RGBA // pixel. final int viewportWidth = stride / 4; // Produce a frame buffer starting at top-left corner, not bottom-left. final Matrix renderMatrix = new Matrix(); renderMatrix.preTranslate(0.5f, 0.5f); renderMatrix.preScale(1f, -1f); renderMatrix.preTranslate(-0.5f, -0.5f); i420TextureFrameBuffer.setSize(viewportWidth, totalHeight); // Bind our framebuffer. GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, i420TextureFrameBuffer.getFrameBufferId()); GlUtil.checkNoGLES2Error("glBindFramebuffer"); // Draw Y. shaderCallbacks.setPlaneY(); VideoFrameDrawer.drawTexture(drawer, inputTextureBuffer, renderMatrix, frameWidth, frameHeight, /* viewportX= */ 0, /* viewportY= */ 0, viewportWidth, /* viewportHeight= */ frameHeight); // Draw U. shaderCallbacks.setPlaneU(); VideoFrameDrawer.drawTexture(drawer, inputTextureBuffer, renderMatrix, frameWidth, frameHeight, /* viewportX= */ 0, /* viewportY= */ frameHeight, viewportWidth / 2, /* viewportHeight= */ uvHeight); // Draw V. shaderCallbacks.setPlaneV(); VideoFrameDrawer.drawTexture(drawer, inputTextureBuffer, renderMatrix, frameWidth, frameHeight, /* viewportX= */ viewportWidth / 2, /* viewportY= */ frameHeight, viewportWidth / 2, /* viewportHeight= */ uvHeight); GLES20.glReadPixels(0, 0, i420TextureFrameBuffer.getWidth(), i420TextureFrameBuffer.getHeight(), GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, i420ByteBuffer); GlUtil.checkNoGLES2Error("YuvConverter.convert"); // Restore normal framebuffer. GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); // Prepare Y, U, and V ByteBuffer slices. final int yPos = 0; final int uPos = yPos + stride * frameHeight; // Rows of U and V alternate in the buffer, so V data starts after the first row of U. final int vPos = uPos + stride / 2; i420ByteBuffer.position(yPos); i420ByteBuffer.limit(yPos + stride * frameHeight); final ByteBuffer dataY = i420ByteBuffer.slice(); i420ByteBuffer.position(uPos); // The last row does not have padding. final int uvSize = stride * (uvHeight - 1) + stride / 2; i420ByteBuffer.limit(uPos + uvSize); final ByteBuffer dataU = i420ByteBuffer.slice(); i420ByteBuffer.position(vPos); i420ByteBuffer.limit(vPos + uvSize); final ByteBuffer dataV = i420ByteBuffer.slice(); return JavaI420Buffer.wrap(frameWidth, frameHeight, dataY, stride, dataU, stride, dataV, stride, () -> { JniCommon.nativeFreeByteBuffer(i420ByteBuffer); }); } public void release() { threadChecker.checkIsOnValidThread(); released = true; drawer.release(); i420TextureFrameBuffer.release(); } }