Enable multi-touch emulation with -gpu on

This CL implements a callback that gets invoked by OpenGLES emulator on it framebuffer
updates. This allows transferring framebuffer changes to the supporting device.

Proper implementation of this new callback also required changes to JPEG compression,
addressing:

1. OpenGLES framebuffer format is RGBA8889, which required implementing line conversion
   for this format.
2. OpenGLES framebuffer is (or at least could be) bottom-up arranged. This requires changes
   to the compressor, so it compresses the FB starting from the bottom, so the resulting
   image is up-bottom.

Change-Id: Icd4efbe4a251c838adfa3518decbfc43a7ef06c8

6 files changed, 131 insertions, 34 deletions

diff --git a/android/multitouch-port.c b/android/multitouch-port.c
index c8e3ca5..9a9313c 100644
--- a/android/multitouch-port.c
+++ b/android/multitouch-port.c
@@ -409,10 +409,12 @@ static void
 _fb_compress(const AndroidMTSPort* mtsp,
              const MTFrameHeader* fmt,
              const uint8_t* fb,
-             int jpeg_quality)
+             int jpeg_quality,
+             int ydir)
 {
     jpeg_compressor_compress_fb(mtsp->jpeg_compressor, fmt->x, fmt->y, fmt->w,
-                                fmt->h, fmt->bpp, fmt->bpl, fb, jpeg_quality);
+                                fmt->h, fmt->disp_height, fmt->bpp, fmt->bpl,
+                                fb, jpeg_quality, ydir);
 }
 
 int
@@ -420,7 +422,8 @@ mts_port_send_frame(AndroidMTSPort* mtsp,
                     MTFrameHeader* fmt,
                     const uint8_t* fb,
                     async_send_cb cb,
-                    void* cb_opaque)
+                    void* cb_opaque,
+                    int ydir)
 {
     char* query;
     int blob_size, off;
@@ -432,7 +435,7 @@ mts_port_send_frame(AndroidMTSPort* mtsp,
 
     /* Compress framebuffer region. 10% quality seems to be sufficient. */
     fmt->format = MTFB_JPEG;
-    _fb_compress(mtsp, fmt, fb, 10);
+    _fb_compress(mtsp, fmt, fb, 10, ydir);
 
     /* Total size of the blob: header + JPEG image. */
     blob_size = sizeof(MTFrameHeader) +
diff --git a/android/multitouch-port.h b/android/multitouch-port.h
index 553617b..d99b3fd 100644
--- a/android/multitouch-port.h
+++ b/android/multitouch-port.h
@@ -120,6 +120,9 @@ extern int mts_port_stop(AndroidMTSPort* amtp);
  *  cb - Callback to invoke when update has been transferred to the MT-emulating
  *      application on the device.
  *  cb_opaque - An opaque parameter to pass back to the 'cb' callback.
+ *  ydir - Indicates direction in which lines are arranged in the framebuffer. If
+ *      this value is negative, lines are arranged in bottom-up format (i.e. the
+ *      bottom line is at the beginning of the buffer).
  * Return:
  *  0 on success, or != 0 on failure.
  */
@@ -127,6 +130,7 @@ extern int mts_port_send_frame(AndroidMTSPort* mtsp,
                                MTFrameHeader* fmt,
                                const uint8_t* fb,
                                async_send_cb cb,
-                               void* cb_opaque);
+                               void* cb_opaque,
+                               int ydir);
 
 #endif  /* ANDROID_ANDROID_MULTITOUCH_PORT_H_ */
diff --git a/android/multitouch-screen.c b/android/multitouch-screen.c
index e1a9a8a..bd96f79 100644
--- a/android/multitouch-screen.c
+++ b/android/multitouch-screen.c
@@ -73,6 +73,12 @@ typedef struct MTSState {
     /* Boolean value indicating if framebuffer updates are currently being
      * transferred to the application running on the device. */
     int             fb_transfer_in_progress;
+    /* Indicates direction in which lines are arranged in the framebuffer. If
+     * this value is negative, lines are arranged in bottom-up format (i.e. the
+     * bottom line is at the beginning of the buffer). */
+    int             ydir;
+    /* Current framebuffer pointer. */
+    uint8_t*        current_fb;
 } MTSState;
 
 /* Default multi-touch screen descriptor */
@@ -228,6 +234,9 @@ _mts_pointer_move(MTSState* mts_state, int slot_index, int x, int y, int pressur
  *                       Multi-touch API
  *******************************************************************************/
 
+/* Multi-touch service initialization flag. */
+static int _is_mt_initialized = 0;
+
 /* Callback that is invoked when framebuffer update has been transmitted to the
  * device. */
 static void
@@ -240,8 +249,8 @@ _on_fb_sent(void* opaque, ATResult res, void* data, int size, int sent)
     if (mts_state->fb_header.w && mts_state->fb_header.h) {
         /* Send accumulated updates. */
         if (mts_port_send_frame(mts_state->mtsp, &mts_state->fb_header,
-                                mts_state->ds->surface->data, _on_fb_sent,
-                                mts_state)) {
+                                mts_state->current_fb, _on_fb_sent, mts_state,
+                                mts_state->ydir)) {
             mts_state->fb_transfer_in_progress = 0;
         }
     } else {
@@ -250,17 +259,12 @@ _on_fb_sent(void* opaque, ATResult res, void* data, int size, int sent)
     }
 }
 
-/* A callback invoked on framebuffer updates.
- * Param:
- *  opaque - MTSState instance.
- *  x, y, w, h - Defines an updated rectangle inside the framebuffer.
+/* Common handler for framebuffer updates invoked by both, software, and OpenGLES
+ * renderers.
  */
 static void
-_mt_fb_update(void* opaque, int x, int y, int w, int h)
+_mt_fb_common_update(MTSState* mts_state, int x, int y, int w, int h)
 {
-    MTSState* const mts_state = (MTSState*)opaque;
-    const DisplaySurface* const surface = mts_state->ds->surface;
-
     if (mts_state->fb_header.w == 0 && mts_state->fb_header.h == 0) {
         /* First update after previous one has been transmitted to the device. */
         mts_state->fb_header.x = x;
@@ -297,31 +301,69 @@ _mt_fb_update(void* opaque, int x, int y, int w, int h)
         mts_state->fb_header.h = bottom - mts_state->fb_header.y;
     }
 
-    /* TODO: Looks like general framebuffer properties can change on the fly.
-     * Find a callback that can catch that. For now, just copy FB properties
-     * over in every FB update. */
-    mts_state->fb_header.bpp = surface->pf.bytes_per_pixel;
-    mts_state->fb_header.bpl = surface->linesize;
-    mts_state->fb_header.disp_width = surface->width;
-    mts_state->fb_header.disp_height = surface->height;
-
     /* We will send updates to the device only after previous transmission is
      * completed. */
     if (!mts_state->fb_transfer_in_progress) {
         mts_state->fb_transfer_in_progress = 1;
         if (mts_port_send_frame(mts_state->mtsp, &mts_state->fb_header,
-                                surface->data, _on_fb_sent, mts_state)) {
+                                mts_state->current_fb, _on_fb_sent, mts_state,
+                                mts_state->ydir)) {
             mts_state->fb_transfer_in_progress = 0;
         }
     }
 }
 
+/* A callback invoked on framebuffer updates by software renderer.
+ * Param:
+ *  opaque - MTSState instance.
+ *  x, y, w, h - Defines an updated rectangle inside the framebuffer.
+ */
+static void
+_mt_fb_update(void* opaque, int x, int y, int w, int h)
+{
+    MTSState* const mts_state = (MTSState*)opaque;
+    const DisplaySurface* const surface = mts_state->ds->surface;
+
+    /* TODO: For sofware renderer general framebuffer properties can change on
+     * the fly. Find a callback that can catch that. For now, just copy FB
+     * properties over in every FB update. */
+    mts_state->fb_header.bpp = surface->pf.bytes_per_pixel;
+    mts_state->fb_header.bpl = surface->linesize;
+    mts_state->fb_header.disp_width = surface->width;
+    mts_state->fb_header.disp_height = surface->height;
+    mts_state->current_fb = surface->data;
+    mts_state->ydir = 1;
+
+    _mt_fb_common_update(mts_state, x, y, w, h);
+}
 void
-multitouch_init(AndroidMTSPort* mtsp)
+multitouch_opengles_fb_update(void* context,
+                              int w, int h, int ydir,
+                              int format, int type,
+                              unsigned char* pixels)
 {
-    /* Multi-touch service initialization flag. */
-    static int _is_mt_initialized = 0;
+    MTSState* const mts_state = &_MTSState;
 
+    /* Make sure MT port is initialized. */
+    if (!_is_mt_initialized) {
+        return;
+    }
+
+    /* GLES format is always RGBA8888 */
+    mts_state->fb_header.bpp = 4;
+    mts_state->fb_header.bpl = 4 * w;
+    mts_state->fb_header.disp_width = w;
+    mts_state->fb_header.disp_height = h;
+    mts_state->current_fb = pixels;
+    mts_state->ydir = ydir;
+
+    /* GLES emulator alwas update the entire framebuffer. */
+    _mt_fb_common_update(mts_state, 0, 0, w, h);
+}
+
+void
+multitouch_init(AndroidMTSPort* mtsp)
+{
     if (!_is_mt_initialized) {
         MTSState* const mts_state = &_MTSState;
         DisplayState* const ds = get_displaystate();
diff --git a/android/multitouch-screen.h b/android/multitouch-screen.h
index 9288073..433944d 100644
--- a/android/multitouch-screen.h
+++ b/android/multitouch-screen.h
@@ -64,5 +64,37 @@ extern int multitouch_get_max_slot();
 /* Saves screen size reported by the device that emulates multi-touch. */
 extern void multitouch_set_device_screen_size(int width, int height);
 
-#endif  /* ANDROID_MULTITOUCH_SCREEN_H_ */
+/* A callback set to monitor OpenGLES framebuffer updates.
+ * This callback is called by the renderer just before each new frame is
+ * displayed, providing a copy of the framebuffer contents.
+ * The callback will be called from one of the renderer's threads, so it may
+ * require synchronization on any data structures it modifies. The pixels buffer
+ * may be overwritten as soon as the callback returns.
+ * The pixels buffer is intentionally not const: the callback may modify the data
+ * without copying to another buffer if it wants, e.g. in-place RGBA to RGB
+ * conversion, or in-place y-inversion.
+ * Param:
+ *   context        The pointer optionally provided when the callback was
+ *                  registered. The client can use this to pass whatever
+ *                  information it wants to the callback.
+ *   width, height  Dimensions of the image, in pixels. Rows are tightly packed;
+ *                  there is no inter-row padding.
+ *   ydir           Indicates row order: 1 means top-to-bottom order, -1 means
+ *                  bottom-to-top order.
+ *   format, type   Format and type GL enums, as used in glTexImage2D() or
+ *                  glReadPixels(), describing the pixel format.
+ *   pixels         The framebuffer image.
+ *
+ * In the first implementation, ydir is always -1 (bottom to top), format and
+ * type are always GL_RGBA and GL_UNSIGNED_BYTE, and the width and height will
+ * always be the same as the ones passed to initOpenGLRenderer().
+ */
+extern void multitouch_opengles_fb_update(void* context,
+                                          int width,
+                                          int height,
+                                          int ydir,
+                                          int format,
+                                          int type,
+                                          unsigned char* pixels);
 
+#endif  /* ANDROID_MULTITOUCH_SCREEN_H_ */
diff --git a/android/utils/jpeg-compress.c b/android/utils/jpeg-compress.c
index 2b42a00..cd45bf5 100644
--- a/android/utils/jpeg-compress.c
+++ b/android/utils/jpeg-compress.c
@@ -151,12 +151,14 @@ jpeg_compressor_get_header_size(const AJPEGDesc* dsc)
 
 void
 jpeg_compressor_compress_fb(AJPEGDesc* dsc,
-                            int x, int y, int w, int h,
+                            int x, int y, int w, int h, int num_lines,
                             int bpp, int bpl,
                             const uint8_t* fb,
-                            int jpeg_quality){
+                            int jpeg_quality,
+                            int ydir){
     struct jpeg_compress_struct cinfo = {0};
     struct jpeg_error_mgr err_mgr;
+    const int x_shift = x * bpp;
 
     /*
      * Initialize compressin information structure, and start compression
@@ -186,9 +188,17 @@ jpeg_compressor_compress_fb(AJPEGDesc* dsc,
     jpeg_start_compress(&cinfo, TRUE);
 
     /* Line by line compress the region. */
-    while (cinfo.next_scanline < cinfo.image_height) {
-        JSAMPROW rgb = (JSAMPROW)(fb + (cinfo.next_scanline + y) * bpl + x * bpp);
-        jpeg_write_scanlines(&cinfo, (JSAMPARRAY)&rgb, 1);
+    if (ydir >= 0) {
+        while (cinfo.next_scanline < cinfo.image_height) {
+            JSAMPROW rgb = (JSAMPROW)(fb + (cinfo.next_scanline + y) * bpl + x_shift);
+            jpeg_write_scanlines(&cinfo, (JSAMPARRAY)&rgb, 1);
+        }
+    } else {
+        const int y_shift = num_lines - y - 1;
+        while (cinfo.next_scanline < cinfo.image_height) {
+            JSAMPROW rgb = (JSAMPROW)(fb + (y_shift - cinfo.next_scanline) * bpl + x_shift);
+            jpeg_write_scanlines(&cinfo, (JSAMPARRAY)&rgb, 1);
+        }
     }
 
     /* Complete the compression. */
diff --git a/android/utils/jpeg-compress.h b/android/utils/jpeg-compress.h
index 4e0e61a..1f84a5d 100644
--- a/android/utils/jpeg-compress.h
+++ b/android/utils/jpeg-compress.h
@@ -77,17 +77,23 @@ extern int jpeg_compressor_get_header_size(const AJPEGDesc* dsc);
  * Param:
  *  dsc - Compression descriptor, obtained with jpeg_compressor_create.
  *  x, y, w, h - Coordinates and sizes of framebuffer region to compress.
+ *  num_lines - Number of lines in the framebuffer (true height).
  *  bpp - Number of bytes per pixel in the framebuffer.
  *  bpl - Number of bytes per line in the framebuffer.
  *  fb - Beginning of the framebuffer.
  *  jpeg_quality JPEG compression quality. A number from 1 to 100. Note that
  *      value 10 provides pretty decent image for the purpose of multi-touch
  *      emulation.
+ *  ydir - Indicates direction in which lines are arranged in the framebuffer. If
+ *      this value is negative, lines are arranged in bottom-up format (i.e. the
+ *      bottom line is at the beginning of the buffer).
  */
 extern void jpeg_compressor_compress_fb(AJPEGDesc* dsc,
                                         int x, int y, int w, int h,
+                                        int num_lines,
                                         int bpp, int bpl,
                                         const uint8_t* fb,
-                                        int jpeg_quality);
+                                        int jpeg_quality,
+                                        int ydir);
 
 #endif  /* _ANDROID_UTILS_JPEG_COMPRESS_H */