From 1e762b1f935c9d4a06af6dd56121590ca81d48b1 Mon Sep 17 00:00:00 2001 From: mbansal Date: Tue, 16 Aug 2011 10:01:20 -0400 Subject: Updates to allow the viewfinder to slide across the UI with device panning. Also, rendering updates for color conversion using GPU. 1) Refactored the SurfaceTextureRenderer and WarpRenderer classes to inherit from a new base class Renderer. 2) Fixed a minor bug in the Blend class and added a helper function to create the mosaic in middle frame coordinate system [no changes in run-time behavior]. 3) Added a flip-flop gPreviewFBO to allow panning the viewfinder across the UI when the user pans the device. 4) Fixed the size of the viewfinder and preview mosaic to match the UI layout height. 5) The viewfinder in the preview stage is now aligned with the viewfinder in the capture stage. 6) Added new class YVURenderer that converts from RGB to YVU data using GLSL shaders. 7) Modified the rendering pipeline to use GPU for RGB to YVU conversion. The processing thread now only performs conversion from YVUA interlaced to YVU planar format. 8) Updates following Wei-Ta's code-review comments. Change-Id: I6c18df3b57d1c2d78d55da8a8f3b9cd970c8d903 --- jni/Android.mk | 2 + jni/feature_mos/src/mosaic/Blend.cpp | 23 +- jni/feature_mos/src/mosaic/Blend.h | 1 + jni/feature_mos/src/mosaic_renderer/Renderer.cpp | 216 +++++++++++++ jni/feature_mos/src/mosaic_renderer/Renderer.h | 65 ++++ .../src/mosaic_renderer/SurfaceTextureRenderer.cpp | 297 +++-------------- .../src/mosaic_renderer/SurfaceTextureRenderer.h | 45 +-- .../src/mosaic_renderer/WarpRenderer.cpp | 303 +++--------------- jni/feature_mos/src/mosaic_renderer/WarpRenderer.h | 44 +-- .../src/mosaic_renderer/YVURenderer.cpp | 149 +++++++++ jni/feature_mos/src/mosaic_renderer/YVURenderer.h | 35 ++ jni/feature_mos_jni.cpp | 47 ++- jni/mosaic_renderer_jni.cpp | 356 +++++++++++++++------ jni/mosaic_renderer_jni.h | 20 +- 14 files changed, 903 insertions(+), 700 deletions(-) create mode 100755 jni/feature_mos/src/mosaic_renderer/Renderer.cpp create mode 100755 jni/feature_mos/src/mosaic_renderer/Renderer.h create mode 100755 jni/feature_mos/src/mosaic_renderer/YVURenderer.cpp create mode 100755 jni/feature_mos/src/mosaic_renderer/YVURenderer.h diff --git a/jni/Android.mk b/jni/Android.mk index 0802754..2043aa2 100755 --- a/jni/Android.mk +++ b/jni/Android.mk @@ -21,8 +21,10 @@ LOCAL_SRC_FILES := \ feature_mos/src/mosaic/ImageUtils.cpp \ feature_mos/src/mosaic/Mosaic.cpp \ feature_mos/src/mosaic/Pyramid.cpp \ + feature_mos/src/mosaic_renderer/Renderer.cpp \ feature_mos/src/mosaic_renderer/WarpRenderer.cpp \ feature_mos/src/mosaic_renderer/SurfaceTextureRenderer.cpp \ + feature_mos/src/mosaic_renderer/YVURenderer.cpp \ feature_mos/src/mosaic_renderer/FrameBuffer.cpp \ feature_stab/db_vlvm/db_feature_detection.cpp \ feature_stab/db_vlvm/db_feature_matching.cpp \ diff --git a/jni/feature_mos/src/mosaic/Blend.cpp b/jni/feature_mos/src/mosaic/Blend.cpp index eb0275e..46a4410 100644 --- a/jni/feature_mos/src/mosaic/Blend.cpp +++ b/jni/feature_mos/src/mosaic/Blend.cpp @@ -70,6 +70,25 @@ int Blend::initialize(int blendingType, int frame_width, int frame_height) return BLEND_RET_OK; } +void Blend::AlignToMiddleFrame(MosaicFrame **frames, int frames_size) +{ + // Unwarp this frame and Warp the others to match + MosaicFrame *mb = NULL; + MosaicFrame *ref = frames[int(frames_size/2)]; // Middle frame + + double invtrs[3][3]; + inv33d(ref->trs, invtrs); + + for(int mfit = 0; mfit < frames_size; mfit++) + { + mb = frames[mfit]; + double temp[3][3]; + mult33d(temp, invtrs, mb->trs); + memcpy(mb->trs, temp, sizeof(temp)); + normProjMat33d(mb->trs); + } +} + int Blend::runBlend(MosaicFrame **frames, int frames_size, ImageType &imageMosaicYVU, int &mosaicWidth, int &mosaicHeight) { int ret; @@ -872,8 +891,8 @@ void Blend::ComputeBlendParameters(MosaicFrame **frames, int frames_size, int is MosaicFrame *last = frames[frames_size-1]; MosaicFrame *mb; - double lxpos = last->trs[0][2], lypos = last->trs[1][5]; - double fxpos = first->trs[0][2], fypos = first->trs[1][5]; + double lxpos = last->trs[0][2], lypos = last->trs[1][2]; + double fxpos = first->trs[0][2], fypos = first->trs[1][2]; // Calculate warp to produce proper stitching. // get x, y displacement diff --git a/jni/feature_mos/src/mosaic/Blend.h b/jni/feature_mos/src/mosaic/Blend.h index 9998aa6..3e5b17c 100644 --- a/jni/feature_mos/src/mosaic/Blend.h +++ b/jni/feature_mos/src/mosaic/Blend.h @@ -98,6 +98,7 @@ protected: void MosaicToFrame(double trs[3][3], double x, double y, double &wx, double &wy); void FrameToMosaicRect(int width, int height, double trs[3][3], BlendRect &brect); void ClipBlendRect(CSite *csite, BlendRect &brect); + void AlignToMiddleFrame(MosaicFrame **frames, int frames_size); int DoMergeAndBlend(MosaicFrame **frames, int nsite, int width, int height, YUVinfo &imgMos, MosaicRect &rect, MosaicRect &cropping_rect); void ComputeMask(CSite *csite, BlendRect &vcrect, BlendRect &brect, MosaicRect &rect, YUVinfo &imgMos, int site_idx); diff --git a/jni/feature_mos/src/mosaic_renderer/Renderer.cpp b/jni/feature_mos/src/mosaic_renderer/Renderer.cpp new file mode 100755 index 0000000..93049cb --- /dev/null +++ b/jni/feature_mos/src/mosaic_renderer/Renderer.cpp @@ -0,0 +1,216 @@ +#include "Renderer.h" + +#include + +#include +#define LOG_TAG "Renderer" +#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__) +#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__) + +Renderer::Renderer() + : mGlProgram(0), + mInputTextureName(-1), + mInputTextureWidth(0), + mInputTextureHeight(0), + mSurfaceWidth(0), + mSurfaceHeight(0) +{ + InitializeGLContext(); +} + +Renderer::~Renderer() { +} + +GLuint Renderer::loadShader(GLenum shaderType, const char* pSource) { + GLuint shader = glCreateShader(shaderType); + if (shader) { + glShaderSource(shader, 1, &pSource, NULL); + glCompileShader(shader); + GLint compiled = 0; + glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled); + if (!compiled) { + GLint infoLen = 0; + glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &infoLen); + if (infoLen) { + char* buf = (char*) malloc(infoLen); + if (buf) { + glGetShaderInfoLog(shader, infoLen, NULL, buf); + LOGE("Could not compile shader %d:\n%s\n", + shaderType, buf); + free(buf); + } + glDeleteShader(shader); + shader = 0; + } + } + } + return shader; +} + +GLuint Renderer::createProgram(const char* pVertexSource, const char* pFragmentSource) +{ + GLuint vertexShader = loadShader(GL_VERTEX_SHADER, pVertexSource); + if (!vertexShader) + { + return 0; + } + LOGI("VertexShader Loaded!"); + + GLuint pixelShader = loadShader(GL_FRAGMENT_SHADER, pFragmentSource); + if (!pixelShader) + { + return 0; + } + LOGI("FragmentShader Loaded!"); + + GLuint program = glCreateProgram(); + if (program) + { + glAttachShader(program, vertexShader); + checkGlError("glAttachShader"); + glAttachShader(program, pixelShader); + checkGlError("glAttachShader"); + + LOGI("Shaders Attached!"); + + glLinkProgram(program); + GLint linkStatus = GL_FALSE; + glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); + + LOGI("Program Linked!"); + + if (linkStatus != GL_TRUE) + { + GLint bufLength = 0; + glGetProgramiv(program, GL_INFO_LOG_LENGTH, &bufLength); + if (bufLength) + { + char* buf = (char*) malloc(bufLength); + if (buf) + { + glGetProgramInfoLog(program, bufLength, NULL, buf); + LOGE("Could not link program:\n%s\n", buf); + free(buf); + } + } + glDeleteProgram(program); + program = 0; + } + } + return program; +} + +// Set this renderer to use the default frame-buffer (screen) and +// set the viewport size to be the given width and height (pixels). +bool Renderer::SetupGraphics(int width, int height) +{ + bool succeeded = false; + do { + if (mGlProgram == 0) + { + if (!InitializeGLProgram()) + { + break; + } + } + glUseProgram(mGlProgram); + if (!checkGlError("glUseProgram")) break; + + glBindFramebuffer(GL_FRAMEBUFFER, 0); + + mFrameBuffer = NULL; + mSurfaceWidth = width; + mSurfaceHeight = height; + + glViewport(0, 0, mSurfaceWidth, mSurfaceHeight); + if (!checkGlError("glViewport")) break; + succeeded = true; + } while (false); + + return succeeded; +} + + +// Set this renderer to use the specified FBO and +// set the viewport size to be the width and height of this FBO. +bool Renderer::SetupGraphics(FrameBuffer* buffer) +{ + bool succeeded = false; + do { + if (mGlProgram == 0) + { + if (!InitializeGLProgram()) + { + break; + } + } + glUseProgram(mGlProgram); + if (!checkGlError("glUseProgram")) break; + + glBindFramebuffer(GL_FRAMEBUFFER, buffer->GetFrameBufferName()); + + mFrameBuffer = buffer; + mSurfaceWidth = mFrameBuffer->GetWidth(); + mSurfaceHeight = mFrameBuffer->GetHeight(); + + glViewport(0, 0, mSurfaceWidth, mSurfaceHeight); + if (!checkGlError("glViewport")) break; + succeeded = true; + } while (false); + + return succeeded; +} + +bool Renderer::Clear(float r, float g, float b, float a) +{ + bool succeeded = false; + do { + bool rt = (mFrameBuffer == NULL)? + SetupGraphics(mSurfaceWidth, mSurfaceHeight) : + SetupGraphics(mFrameBuffer); + + if(!rt) + break; + + glClearColor(r, g, b, a); + glClear(GL_COLOR_BUFFER_BIT); + + succeeded = true; + } while (false); + return succeeded; + +} + +void Renderer::InitializeGLContext() +{ + if(mFrameBuffer != NULL) + { + delete mFrameBuffer; + mFrameBuffer = NULL; + } + + mInputTextureName = -1; + mInputTextureType = GL_TEXTURE_2D; + mGlProgram = 0; +} + +int Renderer::GetTextureName() +{ + return mInputTextureName; +} + +void Renderer::SetInputTextureName(GLuint textureName) +{ + mInputTextureName = textureName; +} + +void Renderer::SetInputTextureType(GLenum textureType) +{ + mInputTextureType = textureType; +} + +void Renderer::SetInputTextureDimensions(int width, int height) +{ + mInputTextureWidth = width; + mInputTextureHeight = height; +} diff --git a/jni/feature_mos/src/mosaic_renderer/Renderer.h b/jni/feature_mos/src/mosaic_renderer/Renderer.h new file mode 100755 index 0000000..a43e802 --- /dev/null +++ b/jni/feature_mos/src/mosaic_renderer/Renderer.h @@ -0,0 +1,65 @@ +#pragma once + +#include "FrameBuffer.h" + +#include + +#include +#include +#include + +class Renderer { + public: + Renderer(); + virtual ~Renderer(); + + // Initialize OpenGL resources + // @return true if successful + virtual bool InitializeGLProgram() = 0; + + bool SetupGraphics(FrameBuffer* buffer); + bool SetupGraphics(int width, int height); + + bool Clear(float r, float g, float b, float a); + + int GetTextureName(); + void SetInputTextureName(GLuint textureName); + void SetInputTextureDimensions(int width, int height); + void SetInputTextureType(GLenum textureType); + + void InitializeGLContext(); + + protected: + + GLuint loadShader(GLenum shaderType, const char* pSource); + GLuint createProgram(const char*, const char* ); + + int SurfaceWidth() const { return mSurfaceWidth; } + int SurfaceHeight() const { return mSurfaceHeight; } + + // Source code for shaders. + virtual const char* VertexShaderSource() const = 0; + virtual const char* FragmentShaderSource() const = 0; + + // Redefine this to use special texture types such as + // GL_TEXTURE_EXTERNAL_OES. + GLenum InputTextureType() const { return mInputTextureType; } + + GLuint mGlProgram; + GLuint mInputTextureName; + GLenum mInputTextureType; + int mInputTextureWidth; + int mInputTextureHeight; + + // Attribute locations + GLint mScalingtransLoc; + GLint maPositionHandle; + GLint maTextureHandle; + + + int mSurfaceWidth; // Width of target surface. + int mSurfaceHeight; // Height of target surface. + + FrameBuffer *mFrameBuffer; +}; + diff --git a/jni/feature_mos/src/mosaic_renderer/SurfaceTextureRenderer.cpp b/jni/feature_mos/src/mosaic_renderer/SurfaceTextureRenderer.cpp index fb124aa..e37406c 100755 --- a/jni/feature_mos/src/mosaic_renderer/SurfaceTextureRenderer.cpp +++ b/jni/feature_mos/src/mosaic_renderer/SurfaceTextureRenderer.cpp @@ -7,29 +7,6 @@ #define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__) #define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__) -static const char gVertexShader[] = -"uniform mat4 uSTMatrix;\n" -"uniform mat4 u_scalingtrans; \n" -"attribute vec4 aPosition;\n" -"attribute vec4 aTextureCoord;\n" -"varying vec2 vTextureCoord;\n" -"varying vec2 vTextureNormCoord;\n" -"void main() {\n" -" gl_Position = u_scalingtrans * aPosition;\n" -" vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" -" vTextureNormCoord = aTextureCoord.xy;\n" -"}\n"; - -static const char gFragmentShader[] = -"#extension GL_OES_EGL_image_external : require\n" -"precision mediump float;\n" -"varying vec2 vTextureCoord;\n" -"varying vec2 vTextureNormCoord;\n" -"uniform samplerExternalOES sTexture;\n" -"void main() {\n" -" gl_FragColor = texture2D(sTexture, vTextureNormCoord);\n" -"}\n"; - const GLfloat g_vVertices[] = { -1.f, -1.f, 0.0f, 1.0f, // Position 0 0.0f, 1.0f, // TexCoord 0 @@ -46,110 +23,51 @@ const int GL_TEXTURE_EXTERNAL_OES_ENUM = 0x8D65; const int VERTEX_STRIDE = 6 * sizeof(GLfloat); -SurfaceTextureRenderer::SurfaceTextureRenderer() - : mGlProgram(0), - mInputTextureName(-1), - mInputTextureWidth(0), - mInputTextureHeight(0), - mSurfaceWidth(0), - mSurfaceHeight(0) -{ +SurfaceTextureRenderer::SurfaceTextureRenderer() : Renderer() { memset(mSTMatrix, 0.0, 16*sizeof(float)); mSTMatrix[0] = 1.0f; mSTMatrix[5] = 1.0f; mSTMatrix[10] = 1.0f; mSTMatrix[15] = 1.0f; - - InitializeGLContext(); } SurfaceTextureRenderer::~SurfaceTextureRenderer() { } -void SurfaceTextureRenderer::SetSTMatrix(float *stmat) +void SurfaceTextureRenderer::SetViewportMatrix(int w, int h, int W, int H) { - memcpy(mSTMatrix, stmat, 16*sizeof(float)); -} - -GLuint SurfaceTextureRenderer::loadShader(GLenum shaderType, const char* pSource) { - GLuint shader = glCreateShader(shaderType); - if (shader) { - glShaderSource(shader, 1, &pSource, NULL); - glCompileShader(shader); - GLint compiled = 0; - glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled); - if (!compiled) { - GLint infoLen = 0; - glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &infoLen); - if (infoLen) { - char* buf = (char*) malloc(infoLen); - if (buf) { - glGetShaderInfoLog(shader, infoLen, NULL, buf); - LOGE("Could not compile shader %d:\n%s\n", - shaderType, buf); - free(buf); - } - glDeleteShader(shader); - shader = 0; - } - } + for(int i=0; i<16; i++) + { + mViewportMatrix[i] = 0.0f; } - return shader; + + mViewportMatrix[0] = float(w)/float(W); + mViewportMatrix[5] = float(h)/float(H); + mViewportMatrix[10] = 1.0f; + mViewportMatrix[12] = -1.0f + float(w)/float(W); + mViewportMatrix[13] = -1.0f + float(h)/float(H); + mViewportMatrix[15] = 1.0f; } -GLuint SurfaceTextureRenderer::createProgram(const char* pVertexSource, const char* pFragmentSource) +void SurfaceTextureRenderer::SetScalingMatrix(float xscale, float yscale) { - GLuint vertexShader = loadShader(GL_VERTEX_SHADER, pVertexSource); - if (!vertexShader) - { - return 0; - } - LOGI("VertexShader Loaded!"); - - GLuint pixelShader = loadShader(GL_FRAGMENT_SHADER, pFragmentSource); - if (!pixelShader) + for(int i=0; i<16; i++) { - return 0; + mScalingMatrix[i] = 0.0f; } - LOGI("FragmentShader Loaded!"); - GLuint program = glCreateProgram(); - if (program) - { - glAttachShader(program, vertexShader); - checkGlError("glAttachShader"); - glAttachShader(program, pixelShader); - checkGlError("glAttachShader"); - - LOGI("Shaders Attached!"); - - glLinkProgram(program); - GLint linkStatus = GL_FALSE; - glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); - - LOGI("Program Linked!"); + mScalingMatrix[0] = xscale; + mScalingMatrix[5] = yscale; + mScalingMatrix[10] = 1.0f; + mScalingMatrix[15] = 1.0f; +} - if (linkStatus != GL_TRUE) - { - GLint bufLength = 0; - glGetProgramiv(program, GL_INFO_LOG_LENGTH, &bufLength); - if (bufLength) - { - char* buf = (char*) malloc(bufLength); - if (buf) - { - glGetProgramInfoLog(program, bufLength, NULL, buf); - LOGE("Could not link program:\n%s\n", buf); - free(buf); - } - } - glDeleteProgram(program); - program = 0; - } - } - return program; +void SurfaceTextureRenderer::SetSTMatrix(float *stmat) +{ + memcpy(mSTMatrix, stmat, 16*sizeof(float)); } + bool SurfaceTextureRenderer::InitializeGLProgram() { bool succeeded = false; @@ -186,115 +104,6 @@ bool SurfaceTextureRenderer::InitializeGLProgram() return succeeded; } -void SurfaceTextureRenderer::SetViewportMatrix(int w, int h, int W, int H) -{ - for(int i=0; i<16; i++) - { - mViewportMatrix[i] = 0.0f; - } - - mViewportMatrix[0] = float(w)/float(W); - mViewportMatrix[5] = float(h)/float(H); - mViewportMatrix[10] = 1.0f; - mViewportMatrix[12] = -1.0f + float(w)/float(W); - mViewportMatrix[13] = -1.0f + float(h)/float(H); - mViewportMatrix[15] = 1.0f; -} - -void SurfaceTextureRenderer::SetScalingMatrix(float xscale, float yscale) -{ - for(int i=0; i<16; i++) - { - mScalingMatrix[i] = 0.0f; - } - - mScalingMatrix[0] = xscale; - mScalingMatrix[5] = yscale; - mScalingMatrix[10] = 1.0f; - mScalingMatrix[15] = 1.0f; -} - -// Set this renderer to use the default frame-buffer (screen) and -// set the viewport size to be the given width and height (pixels). -bool SurfaceTextureRenderer::SetupGraphics(int width, int height) -{ - bool succeeded = false; - do { - if (mGlProgram == 0) - { - if (!InitializeGLProgram()) - { - break; - } - } - glUseProgram(mGlProgram); - if (!checkGlError("glUseProgram")) break; - - glBindFramebuffer(GL_FRAMEBUFFER, 0); - - mFrameBuffer = NULL; - mSurfaceWidth = width; - mSurfaceHeight = height; - - glViewport(0, 0, mSurfaceWidth, mSurfaceHeight); - if (!checkGlError("glViewport")) break; - succeeded = true; - } while (false); - - return succeeded; -} - - -// Set this renderer to use the specified FBO and -// set the viewport size to be the width and height of this FBO. -bool SurfaceTextureRenderer::SetupGraphics(FrameBuffer* buffer) -{ - bool succeeded = false; - do { - if (mGlProgram == 0) - { - if (!InitializeGLProgram()) - { - break; - } - } - glUseProgram(mGlProgram); - if (!checkGlError("glUseProgram")) break; - - glBindFramebuffer(GL_FRAMEBUFFER, buffer->GetFrameBufferName()); - - mFrameBuffer = buffer; - mSurfaceWidth = mFrameBuffer->GetWidth(); - mSurfaceHeight = mFrameBuffer->GetHeight(); - - glViewport(0, 0, mSurfaceWidth, mSurfaceHeight); - if (!checkGlError("glViewport")) break; - succeeded = true; - } while (false); - - return succeeded; -} - -bool SurfaceTextureRenderer::Clear(float r, float g, float b, float a) -{ - bool succeeded = false; - do { - bool rt = (mFrameBuffer == NULL)? - SetupGraphics(mSurfaceWidth, mSurfaceHeight) : - SetupGraphics(mFrameBuffer); - - if(!rt) - break; - - glClearColor(r, g, b, a); - glClear(GL_COLOR_BUFFER_BIT); - - succeeded = true; - } while (false); - return succeeded; - -} - bool SurfaceTextureRenderer::DrawTexture(GLfloat *affine) { bool succeeded = false; @@ -336,47 +145,35 @@ bool SurfaceTextureRenderer::DrawTexture(GLfloat *affine) return succeeded; } -void SurfaceTextureRenderer::InitializeGLContext() -{ - if(mFrameBuffer != NULL) - { - delete mFrameBuffer; - mFrameBuffer = NULL; - } - - mInputTextureName = -1; - mInputTextureType = GL_TEXTURE_EXTERNAL_OES_ENUM; - mGlProgram = 0; -} - -int SurfaceTextureRenderer::GetTextureName() -{ - return mInputTextureName; -} - -void SurfaceTextureRenderer::SetInputTextureName(GLuint textureName) -{ - mInputTextureName = textureName; -} - -void SurfaceTextureRenderer::SetInputTextureType(GLenum textureType) -{ - mInputTextureType = textureType; -} - -void SurfaceTextureRenderer::SetInputTextureDimensions(int width, int height) -{ - mInputTextureWidth = width; - mInputTextureHeight = height; -} - - const char* SurfaceTextureRenderer::VertexShaderSource() const { + static const char gVertexShader[] = + "uniform mat4 uSTMatrix;\n" + "uniform mat4 u_scalingtrans; \n" + "attribute vec4 aPosition;\n" + "attribute vec4 aTextureCoord;\n" + "varying vec2 vTextureCoord;\n" + "varying vec2 vTextureNormCoord;\n" + "void main() {\n" + " gl_Position = u_scalingtrans * aPosition;\n" + " vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" + " vTextureNormCoord = aTextureCoord.xy;\n" + "}\n"; + return gVertexShader; } const char* SurfaceTextureRenderer::FragmentShaderSource() const { + static const char gFragmentShader[] = + "#extension GL_OES_EGL_image_external : require\n" + "precision mediump float;\n" + "varying vec2 vTextureCoord;\n" + "varying vec2 vTextureNormCoord;\n" + "uniform samplerExternalOES sTexture;\n" + "void main() {\n" + " gl_FragColor = texture2D(sTexture, vTextureNormCoord);\n" + "}\n"; + return gFragmentShader; } diff --git a/jni/feature_mos/src/mosaic_renderer/SurfaceTextureRenderer.h b/jni/feature_mos/src/mosaic_renderer/SurfaceTextureRenderer.h index e74bd64..ea2b81a 100755 --- a/jni/feature_mos/src/mosaic_renderer/SurfaceTextureRenderer.h +++ b/jni/feature_mos/src/mosaic_renderer/SurfaceTextureRenderer.h @@ -1,6 +1,7 @@ #pragma once #include "FrameBuffer.h" +#include "Renderer.h" #include @@ -8,8 +9,7 @@ #include #include -//TODO: Add a base class Renderer for WarpRenderer and SurfaceTextureRenderer. -class SurfaceTextureRenderer { +class SurfaceTextureRenderer: public Renderer { public: SurfaceTextureRenderer(); virtual ~SurfaceTextureRenderer(); @@ -18,46 +18,16 @@ class SurfaceTextureRenderer { // @return true if successful bool InitializeGLProgram(); - bool SetupGraphics(FrameBuffer* buffer); - bool SetupGraphics(int width, int height); - - bool Clear(float r, float g, float b, float a); + bool DrawTexture(GLfloat *affine); void SetViewportMatrix(int w, int h, int W, int H); void SetScalingMatrix(float xscale, float yscale); - bool DrawTexture(GLfloat *affine); - - int GetTextureName(); - void SetInputTextureName(GLuint textureName); - void SetInputTextureDimensions(int width, int height); - void SetInputTextureType(GLenum textureType); - - void InitializeGLContext(); - void SetSTMatrix(float *stmat); - protected: - - GLuint loadShader(GLenum shaderType, const char* pSource); - GLuint createProgram(const char*, const char* ); - - int SurfaceWidth() const { return mSurfaceWidth; } - int SurfaceHeight() const { return mSurfaceHeight; } - private: // Source code for shaders. - virtual const char* VertexShaderSource() const; - virtual const char* FragmentShaderSource() const; - - // Redefine this to use special texture types such as - // GL_TEXTURE_EXTERNAL_OES. - virtual GLenum InputTextureType() const { return mInputTextureType; } - - GLuint mGlProgram; - GLuint mInputTextureName; - GLenum mInputTextureType; - int mInputTextureWidth; - int mInputTextureHeight; + const char* VertexShaderSource() const; + const char* FragmentShaderSource() const; // Attribute locations GLint mScalingtransLoc; @@ -67,11 +37,8 @@ class SurfaceTextureRenderer { GLfloat mViewportMatrix[16]; GLfloat mScalingMatrix[16]; - GLfloat mSTMatrix[16]; - int mSurfaceWidth; // Width of target surface. - int mSurfaceHeight; // Height of target surface. + GLfloat mSTMatrix[16]; - FrameBuffer *mFrameBuffer; }; diff --git a/jni/feature_mos/src/mosaic_renderer/WarpRenderer.cpp b/jni/feature_mos/src/mosaic_renderer/WarpRenderer.cpp index b2ca8d4..f1d5883 100755 --- a/jni/feature_mos/src/mosaic_renderer/WarpRenderer.cpp +++ b/jni/feature_mos/src/mosaic_renderer/WarpRenderer.cpp @@ -7,40 +7,14 @@ #define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__) #define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__) - -static const char gVertexShader[] = -"uniform mat4 u_affinetrans; \n" -"uniform mat4 u_viewporttrans; \n" -"uniform mat4 u_scalingtrans; \n" -"attribute vec4 a_position; \n" -"attribute vec2 a_texCoord; \n" -"varying vec2 v_texCoord; \n" -"void main() \n" -"{ \n" -" gl_Position = u_scalingtrans * u_viewporttrans * u_affinetrans * a_position; \n" -" v_texCoord = a_texCoord; \n" -"} \n"; - -static const char gFragmentShader[] = -"precision mediump float; \n" -"varying vec2 v_texCoord; \n" -"uniform sampler2D s_texture; \n" -"void main() \n" -"{ \n" -" vec4 color; \n" -" color = texture2D(s_texture, v_texCoord); \n" -" gl_FragColor = color; \n" -"} \n"; - - const GLfloat g_vVertices[] = { - -1.f, -1.f, 0.0f, 1.0f, // Position 0 + -1.f, 1.f, 0.0f, 1.0f, // Position 0 0.0f, 1.0f, // TexCoord 0 - 1.f, -1.f, 0.0f, 1.0f, // Position 1 + 1.f, 1.f, 0.0f, 1.0f, // Position 1 1.0f, 1.0f, // TexCoord 1 - -1.f, 1.f, 0.0f, 1.0f, // Position 2 + -1.f, -1.f, 0.0f, 1.0f, // Position 2 0.0f, 0.0f, // TexCoord 2 - 1.f, 1.f, 0.0f, 1.0f, // Position 3 + 1.f, -1.f, 0.0f, 1.0f, // Position 3 1.0f, 0.0f // TexCoord 3 }; @@ -48,97 +22,39 @@ const int VERTEX_STRIDE = 6 * sizeof(GLfloat); GLushort g_iIndices[] = { 0, 1, 2, 3 }; -WarpRenderer::WarpRenderer() - : mGlProgram(0), - mInputTextureName(-1), - mInputTextureWidth(0), - mInputTextureHeight(0), - mSurfaceWidth(0), - mSurfaceHeight(0) - { - InitializeGLContext(); +WarpRenderer::WarpRenderer() : Renderer() + { } WarpRenderer::~WarpRenderer() { } -GLuint WarpRenderer::loadShader(GLenum shaderType, const char* pSource) { - GLuint shader = glCreateShader(shaderType); - if (shader) { - glShaderSource(shader, 1, &pSource, NULL); - glCompileShader(shader); - GLint compiled = 0; - glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled); - if (!compiled) { - GLint infoLen = 0; - glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &infoLen); - if (infoLen) { - char* buf = (char*) malloc(infoLen); - if (buf) { - glGetShaderInfoLog(shader, infoLen, NULL, buf); - LOGE("Could not compile shader %d:\n%s\n", - shaderType, buf); - free(buf); - } - glDeleteShader(shader); - shader = 0; - } - } - } - return shader; -} - -GLuint WarpRenderer::createProgram(const char* pVertexSource, const char* pFragmentSource) +void WarpRenderer::SetViewportMatrix(int w, int h, int W, int H) { - GLuint vertexShader = loadShader(GL_VERTEX_SHADER, pVertexSource); - if (!vertexShader) + for(int i=0; i<16; i++) { - return 0; + mViewportMatrix[i] = 0.0f; } - LOGI("VertexShader Loaded!"); - GLuint pixelShader = loadShader(GL_FRAGMENT_SHADER, pFragmentSource); - if (!pixelShader) - { - return 0; - } - LOGI("FragmentShader Loaded!"); + mViewportMatrix[0] = float(w)/float(W); + mViewportMatrix[5] = float(h)/float(H); + mViewportMatrix[10] = 1.0f; + mViewportMatrix[12] = -1.0f + float(w)/float(W); + mViewportMatrix[13] = -1.0f + float(h)/float(H); + mViewportMatrix[15] = 1.0f; +} - GLuint program = glCreateProgram(); - if (program) +void WarpRenderer::SetScalingMatrix(float xscale, float yscale) +{ + for(int i=0; i<16; i++) { - glAttachShader(program, vertexShader); - checkGlError("glAttachShader"); - glAttachShader(program, pixelShader); - checkGlError("glAttachShader"); - - LOGI("Shaders Attached!"); - - glLinkProgram(program); - GLint linkStatus = GL_FALSE; - glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); - - LOGI("Program Linked!"); - - if (linkStatus != GL_TRUE) - { - GLint bufLength = 0; - glGetProgramiv(program, GL_INFO_LOG_LENGTH, &bufLength); - if (bufLength) - { - char* buf = (char*) malloc(bufLength); - if (buf) - { - glGetProgramInfoLog(program, bufLength, NULL, buf); - LOGE("Could not link program:\n%s\n", buf); - free(buf); - } - } - glDeleteProgram(program); - program = 0; - } + mScalingMatrix[i] = 0.0f; } - return program; + + mScalingMatrix[0] = xscale; + mScalingMatrix[5] = yscale; + mScalingMatrix[10] = 1.0f; + mScalingMatrix[15] = 1.0f; } bool WarpRenderer::InitializeGLProgram() @@ -178,115 +94,6 @@ bool WarpRenderer::InitializeGLProgram() return succeeded; } -void WarpRenderer::SetViewportMatrix(int w, int h, int W, int H) -{ - for(int i=0; i<16; i++) - { - mViewportMatrix[i] = 0.0f; - } - - mViewportMatrix[0] = float(w)/float(W); - mViewportMatrix[5] = float(h)/float(H); - mViewportMatrix[10] = 1.0f; - mViewportMatrix[12] = -1.0f + float(w)/float(W); - mViewportMatrix[13] = -1.0f + float(h)/float(H); - mViewportMatrix[15] = 1.0f; -} - -void WarpRenderer::SetScalingMatrix(float xscale, float yscale) -{ - for(int i=0; i<16; i++) - { - mScalingMatrix[i] = 0.0f; - } - - mScalingMatrix[0] = xscale; - mScalingMatrix[5] = yscale; - mScalingMatrix[10] = 1.0f; - mScalingMatrix[15] = 1.0f; -} - -// Set this renderer to use the default frame-buffer (screen) and -// set the viewport size to be the given width and height (pixels). -bool WarpRenderer::SetupGraphics(int width, int height) -{ - bool succeeded = false; - do { - if (mGlProgram == 0) - { - if (!InitializeGLProgram()) - { - break; - } - } - glUseProgram(mGlProgram); - if (!checkGlError("glUseProgram")) break; - - glBindFramebuffer(GL_FRAMEBUFFER, 0); - - mFrameBuffer = NULL; - mSurfaceWidth = width; - mSurfaceHeight = height; - - glViewport(0, 0, mSurfaceWidth, mSurfaceHeight); - if (!checkGlError("glViewport")) break; - succeeded = true; - } while (false); - - return succeeded; -} - - -// Set this renderer to use the specified FBO and -// set the viewport size to be the width and height of this FBO. -bool WarpRenderer::SetupGraphics(FrameBuffer* buffer) -{ - bool succeeded = false; - do { - if (mGlProgram == 0) - { - if (!InitializeGLProgram()) - { - break; - } - } - glUseProgram(mGlProgram); - if (!checkGlError("glUseProgram")) break; - - glBindFramebuffer(GL_FRAMEBUFFER, buffer->GetFrameBufferName()); - - mFrameBuffer = buffer; - mSurfaceWidth = mFrameBuffer->GetWidth(); - mSurfaceHeight = mFrameBuffer->GetHeight(); - - glViewport(0, 0, mSurfaceWidth, mSurfaceHeight); - if (!checkGlError("glViewport")) break; - succeeded = true; - } while (false); - - return succeeded; -} - -bool WarpRenderer::Clear(float r, float g, float b, float a) -{ - bool succeeded = false; - do { - bool rt = (mFrameBuffer == NULL)? - SetupGraphics(mSurfaceWidth, mSurfaceHeight) : - SetupGraphics(mFrameBuffer); - - if(!rt) - break; - - glClearColor(r, g, b, a); - glClear(GL_COLOR_BUFFER_BIT); - - succeeded = true; - } while (false); - return succeeded; - -} - bool WarpRenderer::DrawTexture(GLfloat *affine) { bool succeeded = false; @@ -337,48 +144,36 @@ bool WarpRenderer::DrawTexture(GLfloat *affine) return succeeded; } -void WarpRenderer::InitializeGLContext() -{ - if(mFrameBuffer != NULL) - { - delete mFrameBuffer; - mFrameBuffer = NULL; - } - - mInputTextureName = -1; - mInputTextureType = GL_TEXTURE_2D; - mGlProgram = 0; - mTexHandle = 0; -} - -int WarpRenderer::GetTextureName() -{ - return mInputTextureName; -} - -void WarpRenderer::SetInputTextureName(GLuint textureName) -{ - mInputTextureName = textureName; -} - -void WarpRenderer::SetInputTextureType(GLenum textureType) -{ - mInputTextureType = textureType; -} - -void WarpRenderer::SetInputTextureDimensions(int width, int height) -{ - mInputTextureWidth = width; - mInputTextureHeight = height; -} - - const char* WarpRenderer::VertexShaderSource() const { + static const char gVertexShader[] = + "uniform mat4 u_affinetrans; \n" + "uniform mat4 u_viewporttrans; \n" + "uniform mat4 u_scalingtrans; \n" + "attribute vec4 a_position; \n" + "attribute vec2 a_texCoord; \n" + "varying vec2 v_texCoord; \n" + "void main() \n" + "{ \n" + " gl_Position = u_scalingtrans * u_viewporttrans * u_affinetrans * a_position; \n" + " v_texCoord = a_texCoord; \n" + "} \n"; + return gVertexShader; } const char* WarpRenderer::FragmentShaderSource() const { + static const char gFragmentShader[] = + "precision mediump float; \n" + "varying vec2 v_texCoord; \n" + "uniform sampler2D s_texture; \n" + "void main() \n" + "{ \n" + " vec4 color; \n" + " color = texture2D(s_texture, v_texCoord); \n" + " gl_FragColor = color; \n" + "} \n"; + return gFragmentShader; } diff --git a/jni/feature_mos/src/mosaic_renderer/WarpRenderer.h b/jni/feature_mos/src/mosaic_renderer/WarpRenderer.h index 774b33f..8e9a694 100755 --- a/jni/feature_mos/src/mosaic_renderer/WarpRenderer.h +++ b/jni/feature_mos/src/mosaic_renderer/WarpRenderer.h @@ -1,6 +1,7 @@ #pragma once #include "FrameBuffer.h" +#include "Renderer.h" #include @@ -8,7 +9,7 @@ #include #include -class WarpRenderer { +class WarpRenderer: public Renderer { public: WarpRenderer(); virtual ~WarpRenderer(); @@ -17,45 +18,15 @@ class WarpRenderer { // @return true if successful bool InitializeGLProgram(); - bool SetupGraphics(FrameBuffer* buffer); - bool SetupGraphics(int width, int height); - - bool Clear(float r, float g, float b, float a); - void SetViewportMatrix(int w, int h, int W, int H); void SetScalingMatrix(float xscale, float yscale); - bool DrawTexture(GLfloat *affine); - - int GetTextureName(); - void SetInputTextureName(GLuint textureName); - void SetInputTextureDimensions(int width, int height); - void SetInputTextureType(GLenum textureType); - - void InitializeGLContext(); - - protected: - - GLuint loadShader(GLenum shaderType, const char* pSource); - GLuint createProgram(const char*, const char* ); - int SurfaceWidth() const { return mSurfaceWidth; } - int SurfaceHeight() const { return mSurfaceHeight; } + bool DrawTexture(GLfloat *affine); private: // Source code for shaders. - virtual const char* VertexShaderSource() const; - virtual const char* FragmentShaderSource() const; - - // Redefine this to use special texture types such as - // GL_TEXTURE_EXTERNAL_OES. - virtual GLenum InputTextureType() const { return mInputTextureType; } - - - GLuint mGlProgram; - GLuint mInputTextureName; - GLenum mInputTextureType; - int mInputTextureWidth; - int mInputTextureHeight; + const char* VertexShaderSource() const; + const char* FragmentShaderSource() const; GLuint mTexHandle; // Handle to s_texture. GLuint mTexCoordHandle; // Handle to a_texCoord. @@ -73,10 +44,5 @@ class WarpRenderer { // Sampler location GLint mSamplerLoc; - - int mSurfaceWidth; // Width of target surface. - int mSurfaceHeight; // Height of target surface. - - FrameBuffer *mFrameBuffer; }; diff --git a/jni/feature_mos/src/mosaic_renderer/YVURenderer.cpp b/jni/feature_mos/src/mosaic_renderer/YVURenderer.cpp new file mode 100755 index 0000000..e244ccf --- /dev/null +++ b/jni/feature_mos/src/mosaic_renderer/YVURenderer.cpp @@ -0,0 +1,149 @@ +#include "YVURenderer.h" + +#include + +#include +#define LOG_TAG "YVURenderer" +#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__) +#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__) + +const GLfloat g_vVertices[] = { + -1.f, 1.f, 0.0f, 1.0f, // Position 0 + 0.0f, 1.0f, // TexCoord 0 + 1.f, 1.f, 0.0f, 1.0f, // Position 1 + 1.0f, 1.0f, // TexCoord 1 + -1.f, -1.f, 0.0f, 1.0f, // Position 2 + 0.0f, 0.0f, // TexCoord 2 + 1.f, -1.f, 0.0f, 1.0f, // Position 3 + 1.0f, 0.0f // TexCoord 3 +}; + +const int VERTEX_STRIDE = 6 * sizeof(GLfloat); + +GLushort g_iIndices3[] = { 0, 1, 2, 3 }; + +YVURenderer::YVURenderer() : Renderer() + { +} + +YVURenderer::~YVURenderer() { +} + +bool YVURenderer::InitializeGLProgram() +{ + bool succeeded = false; + do { + GLuint glProgram; + glProgram = createProgram(VertexShaderSource(), + FragmentShaderSource()); + if (!glProgram) { + break; + } + + glUseProgram(glProgram); + if (!checkGlError("glUseProgram")) break; + + // Get attribute locations + mPositionLoc = glGetAttribLocation(glProgram, "a_Position"); + mTexCoordLoc = glGetAttribLocation(glProgram, "a_texCoord"); + + // Get sampler location + mSamplerLoc = glGetUniformLocation(glProgram, "s_texture"); + + mGlProgram = glProgram; + succeeded = true; + } while (false); + + if (!succeeded && (mGlProgram != 0)) + { + glDeleteProgram(mGlProgram); + checkGlError("glDeleteProgram"); + mGlProgram = 0; + } + return succeeded; +} + +bool YVURenderer::DrawTexture() +{ + bool succeeded = false; + do { + bool rt = (mFrameBuffer == NULL)? + SetupGraphics(mSurfaceWidth, mSurfaceHeight) : + SetupGraphics(mFrameBuffer); + + if(!rt) + break; + + glDisable(GL_BLEND); + + glActiveTexture(GL_TEXTURE0); + if (!checkGlError("glActiveTexture")) break; + + const GLenum texture_type = InputTextureType(); + glBindTexture(texture_type, mInputTextureName); + if (!checkGlError("glBindTexture")) break; + + // Set the sampler texture unit to 0 + glUniform1i(mSamplerLoc, 0); + + // Load the vertex position + glVertexAttribPointer(mPositionLoc, 4, GL_FLOAT, + GL_FALSE, VERTEX_STRIDE, g_vVertices); + + // Load the texture coordinate + glVertexAttribPointer(mTexCoordLoc, 2, GL_FLOAT, + GL_FALSE, VERTEX_STRIDE, &g_vVertices[4]); + + glEnableVertexAttribArray(mPositionLoc); + glEnableVertexAttribArray(mTexCoordLoc); + + // And, finally, execute the GL draw command. + glDrawElements(GL_TRIANGLE_STRIP, 4, GL_UNSIGNED_SHORT, g_iIndices3); + + checkGlError("glDrawElements"); + + glBindFramebuffer(GL_FRAMEBUFFER, 0); + succeeded = true; + } while (false); + return succeeded; +} + +const char* YVURenderer::VertexShaderSource() const +{ + // All this really does is copy the coordinates into + // variables for the fragment shader to pick up. + static const char gVertexShader[] = + "attribute vec4 a_Position;\n" + "attribute vec2 a_texCoord;\n" + "varying vec2 v_texCoord;\n" + "void main() {\n" + " gl_Position = a_Position;\n" + " v_texCoord = a_texCoord;\n" + "}\n"; + + return gVertexShader; +} + +const char* YVURenderer::FragmentShaderSource() const +{ + static const char gFragmentShader[] = + "precision mediump float;\n" + "uniform sampler2D s_texture;\n" + "const vec4 coeff_y = vec4(0.257, 0.594, 0.098, 0.063);\n" + "const vec4 coeff_v = vec4(0.439, -0.368, -0.071, 0.500);\n" + "const vec4 coeff_u = vec4(-0.148, -0.291, 0.439, 0.500);\n" + "varying vec2 v_texCoord;\n" + "void main() {\n" + " vec4 p;\n" + " p = texture2D(s_texture, v_texCoord);\n" + " gl_FragColor[0] = dot(p, coeff_y);\n" + " p = texture2D(s_texture, v_texCoord);\n" + " gl_FragColor[1] = dot(p, coeff_v);\n" + " p = texture2D(s_texture, v_texCoord);\n" + " gl_FragColor[2] = dot(p, coeff_u);\n" + " p = texture2D(s_texture, v_texCoord);\n" + " gl_FragColor[3] = dot(p, coeff_y);\n" + "}\n"; + + return gFragmentShader; +} diff --git a/jni/feature_mos/src/mosaic_renderer/YVURenderer.h b/jni/feature_mos/src/mosaic_renderer/YVURenderer.h new file mode 100755 index 0000000..d14a4b9 --- /dev/null +++ b/jni/feature_mos/src/mosaic_renderer/YVURenderer.h @@ -0,0 +1,35 @@ +#pragma once + +#include "FrameBuffer.h" +#include "Renderer.h" + +#include + +#include +#include +#include + +class YVURenderer: public Renderer { + public: + YVURenderer(); + virtual ~YVURenderer(); + + // Initialize OpenGL resources + // @return true if successful + bool InitializeGLProgram(); + + bool DrawTexture(); + + private: + // Source code for shaders. + const char* VertexShaderSource() const; + const char* FragmentShaderSource() const; + + // Attribute locations + GLint mPositionLoc; + GLint mTexCoordLoc; + + // Sampler location + GLint mSamplerLoc; +}; + diff --git a/jni/feature_mos_jni.cpp b/jni/feature_mos_jni.cpp index b06c85f..de87241 100644 --- a/jni/feature_mos_jni.cpp +++ b/jni/feature_mos_jni.cpp @@ -53,6 +53,7 @@ char buffer[1024]; double g_dAffinetrans[16]; double g_dAffinetransInv[16]; +double g_dTranslation[16]; const int MAX_FRAMES_HR = 100; const int MAX_FRAMES_LR = 200; @@ -367,6 +368,23 @@ void decodeYUV444SP(unsigned char* rgb, unsigned char* yuv420sp, int width, static int count = 0; +void ConvertYVUAiToPlanarYVU(unsigned char *planar, unsigned char *in, int width, + int height) +{ + int planeSize = width * height; + unsigned char* Yptr = planar; + unsigned char* Vptr = planar + planeSize; + unsigned char* Uptr = Vptr + planeSize; + + for (int i = 0; i < planeSize; i++) + { + *Yptr++ = *in++; + *Vptr++ = *in++; + *Uptr++ = *in++; + in++; // Alpha + } +} + JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceImageFromGPU( JNIEnv* env, jobject thiz) { @@ -379,25 +397,32 @@ JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceI t0 = now_ms(); - sem_wait(&gPreviewImageRGB_semaphore); - ImageUtils::rgba2yvu(tImage[LR][frame_number_LR], gPreviewImageRGB[LR], + sem_wait(&gPreviewImage_semaphore); + ConvertYVUAiToPlanarYVU(tImage[LR][frame_number_LR], gPreviewImage[LR], tWidth[LR], tHeight[LR]); - sem_post(&gPreviewImageRGB_semaphore); + + sem_post(&gPreviewImage_semaphore); t1 = now_ms(); time_c = t1 - t0; - LOGV("[%d] RGB => YVU [%d]: %g ms", frame_number_HR, frame_number_LR, + LOGV("[%d] RGB [LR] => YVU [LR] [%d]: %g ms", frame_number_HR, frame_number_LR, time_c); int ret_code = AddFrame(LR, frame_number_LR, gTRS); if(ret_code == Mosaic::MOSAIC_RET_OK) { + t0 = now_ms(); // Copy into HR buffer only if this is a valid frame - sem_wait(&gPreviewImageRGB_semaphore); - ImageUtils::rgba2yvu(tImage[HR][frame_number_HR], gPreviewImageRGB[HR], + sem_wait(&gPreviewImage_semaphore); + ConvertYVUAiToPlanarYVU(tImage[HR][frame_number_HR], gPreviewImage[HR], tWidth[HR], tHeight[HR]); - sem_post(&gPreviewImageRGB_semaphore); + sem_post(&gPreviewImage_semaphore); + + t1 = now_ms(); + time_c = t1 - t0; + LOGV("[%d] RGB [HR] => YVU [HR] [%d]: %g ms", frame_number_HR, frame_number_LR, + time_c); frame_number_LR++; frame_number_HR++; @@ -457,10 +482,10 @@ JNIEXPORT jfloatArray JNICALL Java_com_android_camera_panorama_Mosaic_setSourceI tHeight[HR], tImage[LR][frame_number_LR]); - sem_wait(&gPreviewImageRGB_semaphore); - decodeYUV444SP(gPreviewImageRGB[LR], tImage[LR][frame_number_LR], - gPreviewImageRGBWidth[LR], gPreviewImageRGBHeight[LR]); - sem_post(&gPreviewImageRGB_semaphore); + sem_wait(&gPreviewImage_semaphore); + decodeYUV444SP(gPreviewImage[LR], tImage[LR][frame_number_LR], + gPreviewImageWidth[LR], gPreviewImageHeight[LR]); + sem_post(&gPreviewImage_semaphore); t1 = now_ms(); time_c = t1 - t0; diff --git a/jni/mosaic_renderer_jni.cpp b/jni/mosaic_renderer_jni.cpp index c7e11b0..b58e529 100644 --- a/jni/mosaic_renderer_jni.cpp +++ b/jni/mosaic_renderer_jni.cpp @@ -7,6 +7,7 @@ #include "mosaic_renderer/FrameBuffer.h" #include "mosaic_renderer/WarpRenderer.h" #include "mosaic_renderer/SurfaceTextureRenderer.h" +#include "mosaic_renderer/YVURenderer.h" #include #include #include @@ -23,17 +24,16 @@ GLuint gSurfaceTextureID[1]; bool gWarpImage = true; -// Low-Res input image frame in RGB format for preview rendering and processing -// and high-res RGB input image for processing. -unsigned char* gPreviewImageRGB[NR]; +// Low-Res input image frame in YUVA format for preview rendering and processing +// and high-res YUVA input image for processing. +unsigned char* gPreviewImage[NR]; // Low-Res & high-res preview image width -int gPreviewImageRGBWidth[NR]; +int gPreviewImageWidth[NR]; // Low-Res & high-res preview image height -int gPreviewImageRGBHeight[NR]; +int gPreviewImageHeight[NR]; -// Semaphore to protect simultaneous read/writes from gPreviewImageRGB -sem_t gPreviewImageRGB_semaphore; -sem_t gPreviewImageReady_semaphore; +// Semaphore to protect simultaneous read/writes from gPreviewImage +sem_t gPreviewImage_semaphore; // Off-screen preview FBO width (large enough to store the entire // preview mosaic). @@ -49,12 +49,36 @@ SurfaceTextureRenderer gSurfTexRenderer[NR]; // Off-screen FBOs to store the low-res and high-res RGBA copied out from // the SurfaceTexture by the gSurfTexRenderers. FrameBuffer gBufferInput[NR]; + +// Shader to convert RGBA textures into YVU textures for processing +YVURenderer gYVURenderer[NR]; +// Off-screen FBOs to store the low-res and high-res YVU textures for processing +FrameBuffer gBufferInputYVU[NR]; + // Shader to add warped current frame to the preview FBO -WarpRenderer gWarper; +WarpRenderer gWarper1; +// Shader to translate the preview FBO +WarpRenderer gWarper2; +// Off-screen FBOs (flip-flop) to store the result of gWarper1 & gWarper2 +FrameBuffer gBuffer[2]; + // Shader to warp and render the preview FBO to the screen WarpRenderer gPreview; -// Off-screen FBO to store the result of gWarper -FrameBuffer gBuffer; + +// Index of the gBuffer FBO gWarper1 is going to write into +int gCurrentFBOIndex = 0; + +// Variables to represent the present top-left corner of the first frame +// in the previewFBO +double gOriginX = 0.0f; +double gOriginY = 0.0f; + +// Variables tracking the translation value for the current frame and the +// last frame (both w.r.t the first frame). The difference between these +// values is used to control the panning speed of the viewfinder display +// on the UI screen. +double gThisTx = 0.0f; +double gLastTx = 0.0f; // Affine transformation in GL 4x4 format (column-major) to warp the // current frame into the first frame coordinate system. @@ -64,6 +88,10 @@ GLfloat g_dAffinetransGL[16]; // preview FBO into the current frame coordinate system. GLfloat g_dAffinetransInvGL[16]; +// XY translation in GL 4x4 format (column-major) to slide the preview +// viewfinder across the preview FBO +GLfloat g_dTranslationGL[16]; + // GL 4x4 Identity transformation GLfloat g_dAffinetransIdent[] = { 1., 0., 0., 0., @@ -71,6 +99,11 @@ GLfloat g_dAffinetransIdent[] = { 0., 0., 1., 0., 0., 0., 0., 1.}; +float g_dIdent3x3[] = { + 1.0, 0.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 0.0, 1.0}; + const int GL_TEXTURE_EXTERNAL_OES_ENUM = 0x8D65; static void printGLString(const char *name, GLenum s) { @@ -104,11 +137,11 @@ void bindSurfaceTexture(GLuint texId) GL_CLAMP_TO_EDGE); } -void ClearPreviewImageRGB(int mID) +void ClearPreviewImage(int mID) { - unsigned char* ptr = gPreviewImageRGB[mID]; + unsigned char* ptr = gPreviewImage[mID]; for(int j = 0, i = 0; - j < gPreviewImageRGBWidth[mID] * gPreviewImageRGBHeight[mID] * 4; + j < gPreviewImageWidth[mID] * gPreviewImageHeight[mID] * 4; j += 4) { ptr[i++] = 0; @@ -142,10 +175,10 @@ void ConvertAffine3x3toGL4x4(double *matGL44, double *mat33) matGL44[15] = mat33[8]; } -// This function computes fills the 4x4 matrices g_dAffinetrans and -// g_dAffinetransInv using the specified 3x3 affine transformation -// between the first captured frame and the current frame. The computed -// g_dAffinetrans is such that it warps the current frame into the +// This function computes fills the 4x4 matrices g_dAffinetrans, +// g_dAffinetransInv and g_dTranslation using the specified 3x3 affine +// transformation between the first captured frame and the current frame. +// The computed g_dAffinetrans is such that it warps the current frame into the // coordinate system of the first frame. Thus, applying this transformation // to each successive frame builds up the preview mosaic in the first frame // coordinate system. Then the computed g_dAffinetransInv is such that it @@ -155,21 +188,21 @@ void ConvertAffine3x3toGL4x4(double *matGL44, double *mat33) // mosaic data to still be in alignment with this frame. void UpdateWarpTransformation(float *trs) { - double H[9], Hinv[9], Hp[9], Htemp[9]; + double H[9], Hinv[9], Hp[9], Htemp[9], T[9], Tp[9], Ttemp[9]; double K[9], Kinv[9]; - int w = gPreviewImageRGBWidth[LR]; - int h = gPreviewImageRGBHeight[LR]; + int w = gPreviewImageWidth[LR]; + int h = gPreviewImageHeight[LR]; // K is the transformation to map the canonical [-1,1] vertex coordinate // system to the [0,w] image coordinate system before applying the given // affine transformation trs. - K[0] = w / 2.0; + K[0] = w / 2.0 - 0.5; K[1] = 0.0; - K[2] = w / 2.0; + K[2] = w / 2.0 - 0.5; K[3] = 0.0; - K[4] = -h / 2.0; - K[5] = h / 2.0; + K[4] = h / 2.0 - 0.5; + K[5] = h / 2.0 - 0.5; K[6] = 0.0; K[7] = 0.0; K[8] = 1.0; @@ -182,30 +215,36 @@ void UpdateWarpTransformation(float *trs) H[i] = trs[i]; } + gThisTx = trs[2]; + // Move the origin such that the frame is centered in the previewFBO - H[2] += (gPreviewFBOWidth / 2 - gPreviewImageRGBWidth[LR] / 2); - H[5] -= (gPreviewFBOHeight / 2 - gPreviewImageRGBHeight[LR] / 2); + H[2] += gOriginX; + H[5] += gOriginY; // Hp = inv(K) * H * K + // K moves the coordinate system from openGL to image pixels so + // that the alignment transform H can be applied to them. + // inv(K) moves the coordinate system back to openGL normalized + // coordinates so that the shader can correctly render it. db_Identity3x3(Htemp); db_Multiply3x3_3x3(Htemp, H, K); db_Multiply3x3_3x3(Hp, Kinv, Htemp); ConvertAffine3x3toGL4x4(g_dAffinetrans, Hp); - //////////////////////////////////////////////// - ////// Compute g_dAffinetransInv now... ////// - //////////////////////////////////////////////// + //////////////////////////////////////////////////////////////// + ////// Compute g_Translation & g_dAffinetransInv now... ////// + //////////////////////////////////////////////////////////////// w = gPreviewFBOWidth; h = gPreviewFBOHeight; - K[0] = w / 2.0; + K[0] = w / 2.0 - 0.5; K[1] = 0.0; - K[2] = w / 2.0; + K[2] = w / 2.0 - 0.5; K[3] = 0.0; - K[4] = h / 2.0; - K[5] = h / 2.0; + K[4] = h / 2.0 - 0.5; + K[5] = h / 2.0 - 0.5; K[6] = 0.0; K[7] = 0.0; K[8] = 1.0; @@ -213,11 +252,28 @@ void UpdateWarpTransformation(float *trs) db_Identity3x3(Kinv); db_InvertCalibrationMatrix(Kinv, K); + // T only has a fraction of the x-translation of this frame relative + // to the last frame. + db_Identity3x3(T); + T[2] = (gThisTx - gLastTx) * VIEWFINDER_PAN_FACTOR_HORZ; + T[5] = 0; + + gLastTx = gThisTx; + db_Identity3x3(Hinv); db_InvertAffineTransform(Hinv, H); - Hinv[2] += (gPreviewFBOWidth / 2 - gPreviewImageRGBWidth[LR] / 2); - Hinv[5] -= (gPreviewFBOHeight / 2 - gPreviewImageRGBHeight[LR] / 2); + Hinv[2] += gOriginX; + Hinv[5] += gOriginY; + + // We update the origin of where the first frame is laid out in the + // previewFBO to reflect that we have panned the entire preview mosaic + // inside the previewFBO by translation T. This is needed to ensure + // that the next frame can be correctly rendered aligned with the existing + // mosaic. + gOriginX += T[2]; + gOriginY += T[5]; + // Hp = inv(K) * Hinv * K db_Identity3x3(Htemp); @@ -225,43 +281,52 @@ void UpdateWarpTransformation(float *trs) db_Multiply3x3_3x3(Hp, Kinv, Htemp); ConvertAffine3x3toGL4x4(g_dAffinetransInv, Hp); + + // Tp = inv(K) * T * K + db_Identity3x3(Ttemp); + db_Multiply3x3_3x3(Ttemp, T, K); + db_Multiply3x3_3x3(Tp, Kinv, Ttemp); + + ConvertAffine3x3toGL4x4(g_dTranslation, Tp); + } void AllocateTextureMemory(int widthHR, int heightHR, int widthLR, int heightLR) { - gPreviewImageRGBWidth[HR] = widthHR; - gPreviewImageRGBHeight[HR] = heightHR; + gPreviewImageWidth[HR] = widthHR; + gPreviewImageHeight[HR] = heightHR; + + gPreviewImageWidth[LR] = widthLR; + gPreviewImageHeight[LR] = heightLR; - gPreviewImageRGBWidth[LR] = widthLR; - gPreviewImageRGBHeight[LR] = heightLR; + sem_init(&gPreviewImage_semaphore, 0, 1); - sem_init(&gPreviewImageRGB_semaphore, 0, 1); - sem_init(&gPreviewImageReady_semaphore, 0, 1); + sem_wait(&gPreviewImage_semaphore); + gPreviewImage[LR] = ImageUtils::allocateImage(gPreviewImageWidth[LR], + gPreviewImageHeight[LR], 4); + ClearPreviewImage(LR); + gPreviewImage[HR] = ImageUtils::allocateImage(gPreviewImageWidth[HR], + gPreviewImageHeight[HR], 4); + ClearPreviewImage(HR); + sem_post(&gPreviewImage_semaphore); - sem_wait(&gPreviewImageRGB_semaphore); - gPreviewImageRGB[LR] = ImageUtils::allocateImage(gPreviewImageRGBWidth[LR], - gPreviewImageRGBHeight[LR], 4); - ClearPreviewImageRGB(LR); - gPreviewImageRGB[HR] = ImageUtils::allocateImage(gPreviewImageRGBWidth[HR], - gPreviewImageRGBHeight[HR], 4); - ClearPreviewImageRGB(HR); - sem_post(&gPreviewImageRGB_semaphore); + gPreviewFBOWidth = PREVIEW_FBO_WIDTH_SCALE * gPreviewImageWidth[LR]; + gPreviewFBOHeight = PREVIEW_FBO_HEIGHT_SCALE * gPreviewImageHeight[LR]; - gPreviewFBOWidth = PREVIEW_FBO_WIDTH_SCALE * gPreviewImageRGBWidth[LR]; - gPreviewFBOHeight = PREVIEW_FBO_HEIGHT_SCALE * gPreviewImageRGBHeight[LR]; + gOriginX = (gPreviewFBOWidth / 2 - gPreviewImageWidth[LR] / 2); + gOriginY = (gPreviewFBOHeight / 2 - gPreviewImageHeight[LR] / 2); - UpdateWarpTransformation(g_dAffinetransIdent); + UpdateWarpTransformation(g_dIdent3x3); } void FreeTextureMemory() { - sem_wait(&gPreviewImageRGB_semaphore); - ImageUtils::freeImage(gPreviewImageRGB[LR]); - ImageUtils::freeImage(gPreviewImageRGB[HR]); - sem_post(&gPreviewImageRGB_semaphore); + sem_wait(&gPreviewImage_semaphore); + ImageUtils::freeImage(gPreviewImage[LR]); + ImageUtils::freeImage(gPreviewImage[HR]); + sem_post(&gPreviewImage_semaphore); - sem_destroy(&gPreviewImageRGB_semaphore); - sem_destroy(&gPreviewImageReady_semaphore); + sem_destroy(&gPreviewImage_semaphore); } extern "C" @@ -287,11 +352,17 @@ JNIEXPORT jint JNICALL Java_com_android_camera_panorama_MosaicRenderer_init( { gSurfTexRenderer[LR].InitializeGLProgram(); gSurfTexRenderer[HR].InitializeGLProgram(); - gWarper.InitializeGLProgram(); + gYVURenderer[LR].InitializeGLProgram(); + gYVURenderer[HR].InitializeGLProgram(); + gWarper1.InitializeGLProgram(); + gWarper2.InitializeGLProgram(); gPreview.InitializeGLProgram(); - gBuffer.InitializeGLContext(); + gBuffer[0].InitializeGLContext(); + gBuffer[1].InitializeGLContext(); gBufferInput[LR].InitializeGLContext(); gBufferInput[HR].InitializeGLContext(); + gBufferInputYVU[LR].InitializeGLContext(); + gBufferInputYVU[HR].InitializeGLContext(); glBindFramebuffer(GL_FRAMEBUFFER, 0); @@ -306,18 +377,25 @@ JNIEXPORT jint JNICALL Java_com_android_camera_panorama_MosaicRenderer_init( JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_reset( JNIEnv * env, jobject obj, jint width, jint height) { - gBuffer.Init(gPreviewFBOWidth, gPreviewFBOHeight, GL_RGBA); + gBuffer[0].Init(gPreviewFBOWidth, gPreviewFBOHeight, GL_RGBA); + gBuffer[1].Init(gPreviewFBOWidth, gPreviewFBOHeight, GL_RGBA); + + gBufferInput[LR].Init(gPreviewImageWidth[LR], + gPreviewImageHeight[LR], GL_RGBA); + + gBufferInput[HR].Init(gPreviewImageWidth[HR], + gPreviewImageHeight[HR], GL_RGBA); - gBufferInput[LR].Init(gPreviewImageRGBWidth[LR], - gPreviewImageRGBHeight[LR], GL_RGBA); + gBufferInputYVU[LR].Init(gPreviewImageWidth[LR], + gPreviewImageHeight[LR], GL_RGBA); - gBufferInput[HR].Init(gPreviewImageRGBWidth[HR], - gPreviewImageRGBHeight[HR], GL_RGBA); + gBufferInputYVU[HR].Init(gPreviewImageWidth[HR], + gPreviewImageHeight[HR], GL_RGBA); - sem_wait(&gPreviewImageRGB_semaphore); - ClearPreviewImageRGB(LR); - ClearPreviewImageRGB(HR); - sem_post(&gPreviewImageRGB_semaphore); + sem_wait(&gPreviewImage_semaphore); + ClearPreviewImage(LR); + ClearPreviewImage(HR); + sem_post(&gPreviewImage_semaphore); // bind the surface texture bindSurfaceTexture(gSurfaceTextureID[0]); @@ -336,20 +414,43 @@ JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_reset( gSurfTexRenderer[HR].SetInputTextureName(gSurfaceTextureID[0]); gSurfTexRenderer[HR].SetInputTextureType(GL_TEXTURE_EXTERNAL_OES_ENUM); - gWarper.SetupGraphics(&gBuffer); - gWarper.Clear(0.0, 0.0, 0.0, 1.0); - gWarper.SetViewportMatrix(gPreviewImageRGBWidth[LR], - gPreviewImageRGBHeight[LR], gBuffer.GetWidth(), - gBuffer.GetHeight()); - gWarper.SetScalingMatrix(1.0f, 1.0f); - gWarper.SetInputTextureName(gBufferInput[LR].GetTextureName()); - gWarper.SetInputTextureType(GL_TEXTURE_2D); + gYVURenderer[LR].SetupGraphics(&gBufferInputYVU[LR]); + gYVURenderer[LR].Clear(0.0, 0.0, 0.0, 1.0); + gYVURenderer[LR].SetInputTextureName(gBufferInput[LR].GetTextureName()); + gYVURenderer[LR].SetInputTextureType(GL_TEXTURE_2D); + + gYVURenderer[HR].SetupGraphics(&gBufferInputYVU[HR]); + gYVURenderer[HR].Clear(0.0, 0.0, 0.0, 1.0); + gYVURenderer[HR].SetInputTextureName(gBufferInput[HR].GetTextureName()); + gYVURenderer[HR].SetInputTextureType(GL_TEXTURE_2D); + + // gBufferInput[LR] --> gWarper1 --> gBuffer[gCurrentFBOIndex] + gWarper1.SetupGraphics(&gBuffer[gCurrentFBOIndex]); + gWarper1.Clear(0.0, 0.0, 0.0, 1.0); + gWarper1.SetViewportMatrix(gPreviewImageWidth[LR], + gPreviewImageHeight[LR], gBuffer[gCurrentFBOIndex].GetWidth(), + gBuffer[gCurrentFBOIndex].GetHeight()); + gWarper1.SetScalingMatrix(1.0f, 1.0f); + gWarper1.SetInputTextureName(gBufferInput[LR].GetTextureName()); + gWarper1.SetInputTextureType(GL_TEXTURE_2D); + + // gBuffer[gCurrentFBOIndex] --> gWarper2 --> gBuffer[1-gCurrentFBOIndex] + gWarper2.SetupGraphics(&gBuffer[1-gCurrentFBOIndex]); + gWarper2.Clear(0.0, 0.0, 0.0, 1.0); + gWarper2.SetViewportMatrix(1, 1, 1, 1); + gWarper2.SetScalingMatrix(1.0f, 1.0f); + gWarper2.SetInputTextureName(gBuffer[gCurrentFBOIndex].GetTextureName()); + gWarper2.SetInputTextureType(GL_TEXTURE_2D); gPreview.SetupGraphics(width, height); gPreview.Clear(0.0, 0.0, 0.0, 1.0); gPreview.SetViewportMatrix(1, 1, 1, 1); - gPreview.SetScalingMatrix(1.0f, -1.0f); - gPreview.SetInputTextureName(gBuffer.GetTextureName()); + // Scale the previewFBO so that the viewfinder window fills the layout height + // while maintaining the image aspect ratio + gPreview.SetScalingMatrix((PREVIEW_FBO_WIDTH_SCALE / PREVIEW_FBO_HEIGHT_SCALE) * + (gPreviewImageWidth[LR] / gPreviewImageHeight[LR]) / (width / height) * + PREVIEW_FBO_HEIGHT_SCALE, -1.0f*PREVIEW_FBO_HEIGHT_SCALE); + gPreview.SetInputTextureName(gBuffer[1-gCurrentFBOIndex].GetTextureName()); gPreview.SetInputTextureType(GL_TEXTURE_2D); } @@ -367,47 +468,108 @@ JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_preproces gSurfTexRenderer[HR].DrawTexture(g_dAffinetransIdent); } +#ifndef now_ms +#include +static double +now_ms(void) +{ + //struct timespec res; + struct timeval res; + //clock_gettime(CLOCK_REALTIME, &res); + gettimeofday(&res, NULL); + return 1000.0*res.tv_sec + (double)res.tv_usec/1e3; +} +#endif + + + JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_transferGPUtoCPU( JNIEnv * env, jobject obj) { - sem_wait(&gPreviewImageRGB_semaphore); + double t0, t1, time_c; + + t0 = now_ms(); + + gYVURenderer[LR].DrawTexture(); + gYVURenderer[HR].DrawTexture(); + + t1 = now_ms(); + time_c = t1 - t0; + LOGV("YVU Rendering: %g ms", time_c); + + sem_wait(&gPreviewImage_semaphore); // Bind to the input LR FBO and read the Low-Res data from there... - glBindFramebuffer(GL_FRAMEBUFFER, gBufferInput[LR].GetFrameBufferName()); + glBindFramebuffer(GL_FRAMEBUFFER, gBufferInputYVU[LR].GetFrameBufferName()); + t0 = now_ms(); glReadPixels(0, 0, gBufferInput[LR].GetWidth(), gBufferInput[LR].GetHeight(), GL_RGBA, GL_UNSIGNED_BYTE, - gPreviewImageRGB[LR]); + gPreviewImage[LR]); checkGlError("glReadPixels LR"); + t1 = now_ms(); + time_c = t1 - t0; + LOGV("glReadPixels LR: %g ms", time_c); // Bind to the input HR FBO and read the high-res data from there... - glBindFramebuffer(GL_FRAMEBUFFER, gBufferInput[HR].GetFrameBufferName()); + glBindFramebuffer(GL_FRAMEBUFFER, gBufferInputYVU[HR].GetFrameBufferName()); + t0 = now_ms(); glReadPixels(0, 0, gBufferInput[HR].GetWidth(), gBufferInput[HR].GetHeight(), GL_RGBA, GL_UNSIGNED_BYTE, - gPreviewImageRGB[HR]); + gPreviewImage[HR]); checkGlError("glReadPixels HR"); + t1 = now_ms(); + time_c = t1 - t0; + LOGV("glReadPixels HR: %g ms", time_c); - sem_post(&gPreviewImageRGB_semaphore); + sem_post(&gPreviewImage_semaphore); } JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_step( JNIEnv * env, jobject obj) { - // Use the gWarper shader to apply the current frame transformation to the - // current frame and then add it to the gBuffer FBO. - gWarper.DrawTexture(g_dAffinetransGL); + if(!gWarpImage) + { + gWarper1.SetupGraphics(&gBuffer[gCurrentFBOIndex]); + gPreview.SetInputTextureName(gBuffer[gCurrentFBOIndex].GetTextureName()); + + // Use gWarper1 shader to apply the current frame transformation to the + // current frame and then add it to the gBuffer FBO. + gWarper1.DrawTexture(g_dAffinetransGL); + + // Use the gPreview shader to apply the inverse of the current frame + // transformation to the gBuffer FBO and render it to the screen. + gPreview.DrawTexture(g_dAffinetransInvGL); + } + else + { + gWarper1.SetupGraphics(&gBuffer[gCurrentFBOIndex]); + gWarper2.SetupGraphics(&gBuffer[1-gCurrentFBOIndex]); + gWarper2.SetInputTextureName(gBuffer[gCurrentFBOIndex].GetTextureName()); + gPreview.SetInputTextureName(gBuffer[1-gCurrentFBOIndex].GetTextureName()); - // Use the gPreview shader to apply the inverse of the current frame - // transformation to the gBuffer FBO and render it to the screen. - gPreview.DrawTexture(g_dAffinetransInvGL); + // Use gWarper1 shader to apply the current frame transformation to the + // current frame and then add it to the gBuffer FBO. + gWarper1.DrawTexture(g_dAffinetransGL); + + // Use gWarper2 to translate the contents of the gBuffer FBO and copy + // it into the second gBuffer FBO + gWarper2.DrawTexture(g_dTranslationGL); + + // Use the gPreview shader to apply the inverse of the current frame + // transformation to the gBuffer FBO and render it to the screen. + gPreview.DrawTexture(g_dAffinetransInvGL); + + gCurrentFBOIndex = 1 - gCurrentFBOIndex; + } } JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_setWarping( @@ -416,7 +578,8 @@ JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_setWarpin // TODO: Review this logic if(gWarpImage != (bool) flag) //switching from viewfinder to capture or vice-versa { - gWarper.Clear(0.0, 0.0, 0.0, 1.0); + gWarper1.Clear(0.0, 0.0, 0.0, 1.0); + gWarper2.Clear(0.0, 0.0, 0.0, 1.0); gPreview.Clear(0.0, 0.0, 0.0, 1.0); // Clear the screen to black. glBindFramebuffer(GL_FRAMEBUFFER, 0); @@ -432,21 +595,18 @@ JNIEXPORT void JNICALL Java_com_android_camera_panorama_MosaicRenderer_ready( { if(!gWarpImage) { - // TODO: Review this logic... - UpdateWarpTransformation(g_dAffinetransIdent); + UpdateWarpTransformation(g_dIdent3x3); for(int i=0; i<16; i++) { - g_dAffinetrans[i] = g_dAffinetransIdent[i]; g_dAffinetransInv[i] = g_dAffinetransIdent[i]; } - g_dAffinetrans[12] = 1.0f; - g_dAffinetrans[13] = 1.0f; } for(int i=0; i<16; i++) { g_dAffinetransGL[i] = g_dAffinetrans[i]; g_dAffinetransInvGL[i] = g_dAffinetransInv[i]; + g_dTranslationGL[i] = g_dTranslation[i]; } } diff --git a/jni/mosaic_renderer_jni.h b/jni/mosaic_renderer_jni.h index c4ba500..31dd3f0 100644 --- a/jni/mosaic_renderer_jni.h +++ b/jni/mosaic_renderer_jni.h @@ -4,11 +4,17 @@ #include // The Preview FBO dimensions are determined from the low-res -// frame dimensions (gPreviewImageRGBWidth, gPreviewImageRGBHeight) +// frame dimensions (gPreviewImageWidth, gPreviewImageHeight) // using the scale factors below. -const int PREVIEW_FBO_WIDTH_SCALE = 4; +const int PREVIEW_FBO_WIDTH_SCALE = 8; const int PREVIEW_FBO_HEIGHT_SCALE = 2; +// The factor below determines the (horizontal) speed at which the viewfinder +// will pan across the UI during capture. A value of 0.0 will keep the viewfinder +// static in the center of the screen and 1.0f will make it pan at the +// same speed as the device. +const float VIEWFINDER_PAN_FACTOR_HORZ = 0.2f; + const int LR = 0; // Low-resolution mode const int HR = 1; // High-resolution mode const int NR = 2; // Number of resolution modes @@ -18,12 +24,12 @@ extern "C" void AllocateTextureMemory(int widthHR, int heightHR, extern "C" void FreeTextureMemory(); extern "C" void UpdateWarpTransformation(float *trs); -extern unsigned char* gPreviewImageRGB[NR]; -extern int gPreviewImageRGBWidth[NR]; -extern int gPreviewImageRGBHeight[NR]; +extern unsigned char* gPreviewImage[NR]; +extern int gPreviewImageWidth[NR]; +extern int gPreviewImageHeight[NR]; -extern sem_t gPreviewImageRGB_semaphore; -extern sem_t gPreviewImageReady_semaphore; +extern sem_t gPreviewImage_semaphore; extern double g_dAffinetrans[16]; extern double g_dAffinetransInv[16]; +extern double g_dTranslation[16]; -- cgit v1.1