安卓下多线程OpenGL共享Context (四)

     之前的方案假定Java层更新纹理时使用的是RGB或RBGA格式的数据，但是在播放视频这种应用场景下，解码器解码出来的数据如果是YUV格式，渲染起来就比较麻烦了。一种方式是使用CPU进行YUV转RGB，然后再进行渲染，但是这种方式性能极差；另一种方式是使用GPU进行转换，利用GPU的并行计算能力加速转换。我们需要编写Shader来实现。如前文所述，Unity只需要Java层的纹理ID，当使用Shader进行YUV转RGB时，怎么实现更新该纹理的数据呢？答案是Render to Texture (参见[1])。具体做法是，创建一个FrameBuffer，调用glFramebufferTexture2D将纹理与FrameBuffer关联起来，这样在FrameBuffer上进行的绘制，就会被写入到该纹理中。Java代码如下：

public void setupGL(int width, int height) {
    // 创建纹理
    int tempBuffer[] = new int[1];
    GLES20.glGenTextures(1, tempBuffer, 0);
    mTextureId = tempBuffer[0];
    if (mTextureId == 0) {
        glLogE("setupGL, glGenTextures for render texture failed");
        return;
    }
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mTextureId);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
    GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, width, height, 0, GLES20.GL_RGBA,
            GLES20.GL_UNSIGNED_BYTE, null);
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);

    // 创建YUV纹理
    GLES20.glGenTextures(3, mYuvTextures, 0);
    if (mYuvTextures[0] == 0 || mYuvTextures[1] == 0 || mYuvTextures[2] == 0) {
        MyLog.e(TAG, "setupGL, glGenTextures for yuv texture failed");
        return;
    }
    for (int yuvTexture : mYuvTextures) {
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTexture);
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
    }
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);

    // 创建帧缓冲区
    GLES20.glGenFramebuffers(1, tempBuffer, 0);
    mFramebuffer = tempBuffer[0];
    if (mFramebuffer == 0) {
        glLogE("setupGL, glGenFramebuffers failed");
        return;
    }
    GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, mFramebuffer);
    GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
            GLES20.GL_TEXTURE_2D, mTextureId, 0);
    int errCode = GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER);
    if (errCode != GLES20.GL_FRAMEBUFFER_COMPLETE) {
        glLogE("setupGL, glCheckFramebufferStatus failed, errCode=0x" + Integer.toHexString(errCode));
        return;
    }
    GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
}

     绘制时，先绑定FrameBuffer，再进行绘制操作，Java代码如下：

public void updateTexture() {
    GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, mFramebuffer);
    GLES20.glViewport(0, 0, mWidth, mHeight);
    GLES20.glClearColor(0, 0, 0, 1);
    GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
    // 此处添加绘制操作
    GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
}

      为了实现YUV数据的渲染，需要编写Shader。此处，我们渲染的YUV数据格式为YUV420 (YUV420格式的具体介绍，请读者自行百度)，Y、U、V通道数据分别存放在三个缓冲区中。将YUV数据分别赋给三个纹理，然后指定Shader的顶点坐标和纹理坐标，绘制一个矩形即可 (参见[2])。

      首先，在setupGL函数中为YUV生成三个纹理，Java代码如下：

// 创建YUV纹理
GLES20.glGenTextures(3, mYuvTextures, 0);
if (mYuvTextures[0] == 0 || mYuvTextures[1] == 0 || mYuvTextures[2] == 0) {
    MyLog.e(TAG, "setupGL, glGenTextures for yuv texture failed");
    return;
}
for (int yuvTexture : mYuvTextures) {
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTexture);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
    GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
}
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);

绘制时，将YUV数据分别赋给三个纹理，并将三个纹理分别与GLES20.GL_TEXTURE0，GLES20.GL_TEXTURE1，GLES20.GL_TEXTURE2绑定，Java代码如下：

for (int i = 0; i < 3; ++i) {
    GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mYuvTextures[i]);
    int w = i == 0 ? yuvFrame.yuvStrides[0] : (yuvFrame.yuvStrides[0] / 2);
    int h = i == 0 ? yuvFrame.height : (yuvFrame.height / 2);
    GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, w, h, 0,
            GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, yuvFrame.yuvPlanes[i]);
}

　　然后，我们需要创建Program，为Program创建Vertex Shader和Fragment Shader (参见[2])。Java代码如下：

public static final String VERTEX_SHADER_STRING = "attribute vec4 in_pos;
"
        + "attribute vec2 in_tc;
"
        + "varying vec2 out_tc;
"
        + "void main() {
"
        + "    gl_Position = in_pos;
"
        + "    out_tc = in_tc;
"
        + "}
";

public static final String FRAGMENT_SHADER_STRING = "precision mediump float;
"
        + "uniform sampler2D tex_y;
"
        + "uniform sampler2D tex_u;
"
        + "uniform sampler2D tex_v;
"
        + "varying vec2 out_tc;
"
        + "void main() {
"
        + "    vec4 c = vec4((texture2D(tex_y, out_tc).r - 16./255.) * 1.164);
"
        + "    vec4 U = vec4(texture2D(tex_u, out_tc).r - 128./255.);
"
        + "    vec4 V = vec4(texture2D(tex_v, out_tc).r - 128./255.);
"
        + "    c += V * vec4(1.596, -0.813, 0, 0);
"
        + "    c += U * vec4(0, -0.392, 2.017, 0);
"
        + "    c.a = 1.0;
"
        + "    gl_FragColor = c;
"
        + "}
";

protected final void addShaderTo(int type, String source, int program) throws RuntimeException {
    int shader = GLES20.glCreateShader(type);
    if (shader == 0) {
        throw new RuntimeException("Create shader failed, err=" + GLES10.glGetError());
    }
    GLES20.glShaderSource(shader, source);
    GLES20.glCompileShader(shader);
    int[] result = new int[]{GLES20.GL_FALSE};
    GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, result, 0);
    if (result[0] != GLES20.GL_TRUE) {
        GLES20.glDeleteShader(shader);
        throw new RuntimeException("Compile shader failed, err=" + GLES10.glGetError());
    }
    GLES20.glAttachShader(program, shader);
    GLES20.glDeleteShader(shader);
}

public void setupGL(int width, int height) {
    ...
    // 创建Program
    mProgram = GLES20.glCreateProgram();
    addShaderTo(GLES20.GL_VERTEX_SHADER, EglRender.VERTEX_SHADER_STRING, mProgram);
    addShaderTo(GLES20.GL_FRAGMENT_SHADER, EglRender.FRAGMENT_SHADER_STRING, mProgram);
    GLES20.glLinkProgram(mProgram);
    GLES20.glGetProgramiv(mProgram, GLES20.GL_LINK_STATUS, tempBuffer, 0);
    if (tempBuffer[0] != GLES20.GL_TRUE) {
        glLogE("setupGL, create program failed");
        return;
    }
    GLES20.glUseProgram(mProgram);
    int y_tex = GLES20.glGetUniformLocation(mProgram, "tex_y");
    GLES20.glUniform1i(y_tex, 0);
    int u_tex = GLES20.glGetUniformLocation(mProgram, "tex_u");
    GLES20.glUniform1i(u_tex, 1);
    int v_tex = GLES20.glGetUniformLocation(mProgram, "tex_v");
    GLES20.glUniform1i(v_tex, 2);
    mVertexLocation = GLES20.glGetAttribLocation(mProgram, "in_pos");
    mTextureLocation = GLES20.glGetAttribLocation(mProgram, "in_tc");
    GLES20.glUseProgram(0);
}

完整绘制代码如下：

public void updateTexture() {
    GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, mFramebuffer);

    GLES20.glViewport(0, 0, mWidth, mHeight);
    GLES20.glClearColor(0, 0, 0, 1);
    GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
    GLES20.glDisable(GLES20.GL_CULL_FACE);
    GLES20.glDisable(GLES20.GL_DEPTH_TEST);
    GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, 0);
    GLES20.glBindBuffer(GLES20.GL_ELEMENT_ARRAY_BUFFER, 0);

    GLES20.glUseProgram(mProgram);
    // 更新YUV数据
    for (int i = 0; i < 3; ++i) {
        GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, mYuvTextures[i]);
        int w = i == 0 ? yuvFrame.yuvStrides[0] : (yuvFrame.yuvStrides[0] / 2);
        int h = i == 0 ? yuvFrame.height : (yuvFrame.height / 2);
        GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, w, h, 0,
                GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, yuvFrame.yuvPlanes[i]);
    }
    // 设置顶点坐标和纹理坐标
    GLES20.glEnableVertexAttribArray(mVertexLocation);
    mVertexCoord.position(0);
    GLES20.glVertexAttribPointer(mVertexLocation, 2, GLES20.GL_FLOAT, false, 0, mVertexCoord);
    GLES20.glEnableVertexAttribArray(mTextureLocation);
    mTextureCoord.position(0);
    GLES20.glVertexAttribPointer(mTextureLocation, 2, GLES20.GL_FLOAT, false, 0, mTextureCoord);
    // 绘制矩形
    GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);

    GLES20.glDisableVertexAttribArray(mVertexLocation);
    GLES20.glDisableVertexAttribArray(mTextureLocation);
    for (int i = 0; i < 3; ++i) {
        GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
    }

    GLES20.glUseProgram(0);
    GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
    GLES20.glClear(0);
}

其中，9～10行是必需的，否则，glDrawArrays调用会失败，报GL_INVALID_OPERATION错误 (参见[3])。当绘制完成之后，需要通知Unity3D，在C#中调用GL.InvalidateState，否则会影响Unity的绘制 (参见[4])。C#代码如下：

void Update () {
    mPluginTexture.Call ("updateTexture");
    GL.InvalidateState ();
}

其中，Update函数是Unity在每次刷新帧时回调，我们在该回调中调用Java层的updateTexture函数更新纹理数据，然后调用GL.InvalidateState，通知Unity重置OpenGL状态。笔者一开始采用当Java解码出一帧时，便更新纹理数据进行绘制，然后通知C#调用GL.InvalidateState。但是这种方式存在两个问题，一是当App退到后台，绘制操作仍会进行，只是绘制会失败；二是画面更新一段时间之后，便会卡住，过很久才会恢复。具体原因并未查出。后来，改用从C#的Update回调更新纹理，这两个问题得以解决。

总结：

     Unity官方给出的Plugin绘制方式是通过在C#层调用GL.IssuePluginEvent，C++层接收从Unity Render线程过来的回调，在该回调中更新纹理 (参见[5])。该方案需要编写JNI和C++，实现起来比较麻烦。本文给出的方案全部在Java层即可实现。供感兴趣的读者参考。

[参考文献]

[1] Tutorial 14 : Render To Texture

[2] 最简单的视音频播放示例6：OpenGL播放YUV420P（通过Texture，使用Shader）

[3] robertcastle/UnityFBO

[4] GL.InvalidateState

[5] Low-level Native Plugin Interface