Add DEPTH image formats, support in ImageReader

- Add an explicit mapping between public ImageFormat/
  PixelFormat enums and internal HAL format/dataspace.
- Add DEPTH16 and DEPTH_POINT_CLOUD formats
- Wire up mapping layer to ImageReader to support depth
  formats

Change-Id: I8197eccef900cc91baddcfcb934ccd4d8c972eff

7 files changed, 209 insertions, 34 deletions

diff --git a/api/current.txt b/api/current.txt
index b22dd1c..612d16d 100644
--- a/api/current.txt
+++ b/api/current.txt
@@ -10973,6 +10973,8 @@ package android.graphics {
   public class ImageFormat {
     ctor public ImageFormat();
     method public static int getBitsPerPixel(int);
+    field public static final int DEPTH16 = 1144402265; // 0x44363159
+    field public static final int DEPTH_POINT_CLOUD = 257; // 0x101
     field public static final int JPEG = 256; // 0x100
     field public static final int NV16 = 16; // 0x10
     field public static final int NV21 = 17; // 0x11
diff --git a/api/system-current.txt b/api/system-current.txt
index 2b8ce98..4315d1d 100644
--- a/api/system-current.txt
+++ b/api/system-current.txt
@@ -11248,6 +11248,8 @@ package android.graphics {
   public class ImageFormat {
     ctor public ImageFormat();
     method public static int getBitsPerPixel(int);
+    field public static final int DEPTH16 = 1144402265; // 0x44363159
+    field public static final int DEPTH_POINT_CLOUD = 257; // 0x101
     field public static final int JPEG = 256; // 0x100
     field public static final int NV16 = 16; // 0x10
     field public static final int NV21 = 17; // 0x11
diff --git a/core/jni/android_view_Surface.cpp b/core/jni/android_view_Surface.cpp
index bfa0534..39064ed 100644
--- a/core/jni/android_view_Surface.cpp
+++ b/core/jni/android_view_Surface.cpp
@@ -130,6 +130,100 @@ jobject android_view_Surface_createFromIGraphicBufferProducer(JNIEnv* env,
     return surfaceObj;
 }
 
+int android_view_Surface_mapPublicFormatToHalFormat(PublicFormat f) {
+
+    switch(f) {
+        case PublicFormat::JPEG:
+        case PublicFormat::DEPTH_POINT_CLOUD:
+            return HAL_PIXEL_FORMAT_BLOB;
+        case PublicFormat::DEPTH16:
+            return HAL_PIXEL_FORMAT_Y16;
+        case PublicFormat::RAW_SENSOR:
+            return HAL_PIXEL_FORMAT_RAW16;
+        default:
+            // Most formats map 1:1
+            return static_cast<int>(f);
+    }
+}
+
+android_dataspace android_view_Surface_mapPublicFormatToHalDataspace(
+        PublicFormat f) {
+    switch(f) {
+        case PublicFormat::JPEG:
+            return HAL_DATASPACE_JFIF;
+        case PublicFormat::DEPTH_POINT_CLOUD:
+        case PublicFormat::DEPTH16:
+            return HAL_DATASPACE_DEPTH;
+        case PublicFormat::RAW_SENSOR:
+        case PublicFormat::RAW10:
+            return HAL_DATASPACE_ARBITRARY;
+        case PublicFormat::YUV_420_888:
+        case PublicFormat::NV21:
+        case PublicFormat::YV12:
+            return HAL_DATASPACE_JFIF;
+        default:
+            // Most formats map to UNKNOWN
+            return HAL_DATASPACE_UNKNOWN;
+    }
+}
+
+PublicFormat android_view_Surface_mapHalFormatDataspaceToPublicFormat(
+        int format, android_dataspace dataSpace) {
+    switch(format) {
+        case HAL_PIXEL_FORMAT_RGBA_8888:
+        case HAL_PIXEL_FORMAT_RGBX_8888:
+        case HAL_PIXEL_FORMAT_RGB_888:
+        case HAL_PIXEL_FORMAT_RGB_565:
+        case HAL_PIXEL_FORMAT_Y8:
+        case HAL_PIXEL_FORMAT_RAW10:
+        case HAL_PIXEL_FORMAT_YCbCr_420_888:
+        case HAL_PIXEL_FORMAT_YV12:
+            // Enums overlap in both name and value
+            return static_cast<PublicFormat>(format);
+        case HAL_PIXEL_FORMAT_RAW16:
+            // Name differs, though value is the same
+            return PublicFormat::RAW_SENSOR;
+        case HAL_PIXEL_FORMAT_YCbCr_422_SP:
+            // Name differs, though the value is the same
+            return PublicFormat::NV16;
+        case HAL_PIXEL_FORMAT_YCrCb_420_SP:
+            // Name differs, though the value is the same
+            return PublicFormat::NV21;
+        case HAL_PIXEL_FORMAT_YCbCr_422_I:
+            // Name differs, though the value is the same
+            return PublicFormat::YUY2;
+        case HAL_PIXEL_FORMAT_Y16:
+            // Dataspace-dependent
+            switch (dataSpace) {
+                case HAL_DATASPACE_DEPTH:
+                    return PublicFormat::DEPTH16;
+                default:
+                    // Assume non-depth Y16 is just Y16.
+                    return PublicFormat::Y16;
+            }
+            break;
+        case HAL_PIXEL_FORMAT_BLOB:
+            // Dataspace-dependent
+            switch (dataSpace) {
+                case HAL_DATASPACE_DEPTH:
+                    return PublicFormat::DEPTH_POINT_CLOUD;
+                case HAL_DATASPACE_JFIF:
+                    return PublicFormat::JPEG;
+                default:
+                    // Assume otherwise-marked blobs are also JPEG
+                    return PublicFormat::JPEG;
+            }
+            break;
+        case HAL_PIXEL_FORMAT_BGRA_8888:
+        case HAL_PIXEL_FORMAT_RAW_OPAQUE:
+        case HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED:
+            // Not defined in public API
+            return PublicFormat::UNKNOWN;
+
+        default:
+            return PublicFormat::UNKNOWN;
+    }
+}
 // ----------------------------------------------------------------------------
 
 static inline bool isSurfaceValid(const sp<Surface>& sur) {
diff --git a/graphics/java/android/graphics/ImageFormat.java b/graphics/java/android/graphics/ImageFormat.java
index 3efb9c0..49c4247 100644
--- a/graphics/java/android/graphics/ImageFormat.java
+++ b/graphics/java/android/graphics/ImageFormat.java
@@ -356,6 +356,38 @@ public class ImageFormat {
     public static final int RAW10 = 0x25;
 
     /**
+     * Android dense depth image format.
+     *
+     * Each pixel is 16 bits, representing a depth ranging measurement from
+     * a depth camera or similar sensor.
+     *
+     * <p>This format assumes
+     * <ul>
+     * <li>an even width</li>
+     * <li>an even height</li>
+     * <li>a horizontal stride multiple of 16 pixels</li>
+     * </ul>
+     * </p>
+     *
+     * <pre> y_size = stride * height </pre>
+     *
+     * When produced by a camera, the units are millimeters.
+     */
+    public static final int DEPTH16 = 0x44363159;
+
+    /**
+     * Android sparse depth point cloud format.
+     *
+     * <p>A variable-length list of 3D points, with each point represented
+     * by a triple of floats.</p>
+     *
+     * <p>The number of points is {@code (size of the buffer in bytes) / 12}.
+     *
+     * The coordinate system and units depend on the source of the point cloud data.
+     */
+    public static final int DEPTH_POINT_CLOUD = 0x101;
+
+    /**
      * Use this function to retrieve the number of bits per pixel of an
      * ImageFormat.
      *
@@ -376,6 +408,7 @@ public class ImageFormat {
             case Y8:
                 return 8;
             case Y16:
+            case DEPTH16:
                 return 16;
             case NV21:
                 return 12;
@@ -412,6 +445,8 @@ public class ImageFormat {
             case YUV_420_888:
             case RAW_SENSOR:
             case RAW10:
+            case DEPTH16:
+            case DEPTH_POINT_CLOUD:
                 return true;
         }
 
diff --git a/include/android_runtime/android_view_Surface.h b/include/android_runtime/android_view_Surface.h
index 53e8b49..a6836a8 100644
--- a/include/android_runtime/android_view_Surface.h
+++ b/include/android_runtime/android_view_Surface.h
@@ -26,6 +26,33 @@ namespace android {
 class Surface;
 class IGraphicBufferProducer;
 
+/**
+ * Enum mirroring the public API definitions for image and pixel formats.
+ * Some of these are hidden in the public API
+ *
+ * Keep up to date with android.graphics.ImageFormat and
+ * android.graphics.PixelFormat
+ */
+enum class PublicFormat {
+    UNKNOWN           = 0x0,
+    RGBA_8888         = 0x1,
+    RGBX_8888         = 0x2,
+    RGB_888           = 0x3,
+    RGB_565           = 0x4,
+    NV16              = 0x10,
+    NV21              = 0x11,
+    YUY2              = 0x14,
+    RAW_SENSOR        = 0x20,
+    YUV_420_888       = 0x23,
+    RAW10             = 0x25,
+    JPEG              = 0x100,
+    DEPTH_POINT_CLOUD = 0x101,
+    YV12              = 0x32315659,
+    Y8                = 0x20203859, // @hide
+    Y16               = 0x20363159, // @hide
+    DEPTH16           = 0x44363159
+};
+
 /* Gets the underlying ANativeWindow for a Surface. */
 extern sp<ANativeWindow> android_view_Surface_getNativeWindow(
         JNIEnv* env, jobject surfaceObj);
@@ -40,6 +67,21 @@ extern sp<Surface> android_view_Surface_getSurface(JNIEnv* env, jobject surfaceO
 extern jobject android_view_Surface_createFromIGraphicBufferProducer(JNIEnv* env,
         const sp<IGraphicBufferProducer>& bufferProducer);
 
+/* Convert from android.graphics.ImageFormat/PixelFormat enums to graphics.h HAL
+ * format */
+extern int android_view_Surface_mapPublicFormatToHalFormat(PublicFormat f);
+
+/* Convert from android.graphics.ImageFormat/PixelFormat enums to graphics.h HAL
+ * dataspace */
+extern android_dataspace android_view_Surface_mapPublicFormatToHalDataspace(
+        PublicFormat f);
+
+/* Convert from HAL format, dataspace pair to
+ * android.graphics.ImageFormat/PixelFormat.
+ * For unknown/unspecified pairs, returns PublicFormat::UNKNOWN */
+extern PublicFormat android_view_Surface_mapHalFormatDataspaceToPublicFormat(
+        int format, android_dataspace dataSpace);
+
 } // namespace android
 
 #endif // _ANDROID_VIEW_SURFACE_H
diff --git a/media/java/android/media/ImageReader.java b/media/java/android/media/ImageReader.java
index 8d6a588..824a7ad 100644
--- a/media/java/android/media/ImageReader.java
+++ b/media/java/android/media/ImageReader.java
@@ -483,6 +483,8 @@ public class ImageReader implements AutoCloseable {
             case ImageFormat.Y16:
             case ImageFormat.RAW_SENSOR:
             case ImageFormat.RAW10:
+            case ImageFormat.DEPTH16:
+            case ImageFormat.DEPTH_POINT_CLOUD:
                 return 1;
             default:
                 throw new UnsupportedOperationException(
diff --git a/media/jni/android_media_ImageReader.cpp b/media/jni/android_media_ImageReader.cpp
index cf69b8f..b247493 100644
--- a/media/jni/android_media_ImageReader.cpp
+++ b/media/jni/android_media_ImageReader.cpp
@@ -95,6 +95,9 @@ public:
     void setBufferFormat(int format) { mFormat = format; }
     int getBufferFormat() { return mFormat; }
 
+    void setBufferDataspace(android_dataspace dataSpace) { mDataSpace = dataSpace; }
+    android_dataspace getBufferDataspace() { return mDataSpace; }
+
     void setBufferWidth(int width) { mWidth = width; }
     int getBufferWidth() { return mWidth; }
 
@@ -111,6 +114,7 @@ private:
     jobject mWeakThiz;
     jclass mClazz;
     int mFormat;
+    android_dataspace mDataSpace;
     int mWidth;
     int mHeight;
 };
@@ -263,29 +267,6 @@ static void Image_setBuffer(JNIEnv* env, jobject thiz,
     env->SetLongField(thiz, gSurfaceImageClassInfo.mLockedBuffer, reinterpret_cast<jlong>(buffer));
 }
 
-// Some formats like JPEG defined with different values between android.graphics.ImageFormat and
-// graphics.h, need convert to the one defined in graphics.h here.
-static int Image_getPixelFormat(JNIEnv* env, int format)
-{
-    int jpegFormat;
-    jfieldID fid;
-
-    ALOGV("%s: format = 0x%x", __FUNCTION__, format);
-
-    jclass imageFormatClazz = env->FindClass("android/graphics/ImageFormat");
-    ALOG_ASSERT(imageFormatClazz != NULL);
-
-    fid = env->GetStaticFieldID(imageFormatClazz, "JPEG", "I");
-    jpegFormat = env->GetStaticIntField(imageFormatClazz, fid);
-
-    // Translate the JPEG to BLOB for camera purpose.
-    if (format == jpegFormat) {
-        format = HAL_PIXEL_FORMAT_BLOB;
-    }
-
-    return format;
-}
-
 static uint32_t Image_getJpegSize(CpuConsumer::LockedBuffer* buffer, bool usingRGBAOverride)
 {
     ALOG_ASSERT(buffer != NULL, "Input buffer is NULL!!!");
@@ -483,7 +464,7 @@ static void Image_getLockedBufferInfo(JNIEnv* env, CpuConsumer::LockedBuffer* bu
 }
 
 static jint Image_imageGetPixelStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx,
-        int32_t readerFormat)
+        int32_t halReaderFormat)
 {
     ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
     ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0), "Index is out of range:%d", idx);
@@ -493,7 +474,7 @@ static jint Image_imageGetPixelStride(JNIEnv* env, CpuConsumer::LockedBuffer* bu
 
     int32_t fmt = buffer->flexFormat;
 
-    fmt = applyFormatOverrides(fmt, readerFormat);
+    fmt = applyFormatOverrides(fmt, halReaderFormat);
 
     switch (fmt) {
         case HAL_PIXEL_FORMAT_YCbCr_420_888:
@@ -543,7 +524,7 @@ static jint Image_imageGetPixelStride(JNIEnv* env, CpuConsumer::LockedBuffer* bu
 }
 
 static jint Image_imageGetRowStride(JNIEnv* env, CpuConsumer::LockedBuffer* buffer, int idx,
-        int32_t readerFormat)
+        int32_t halReaderFormat)
 {
     ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
     ALOG_ASSERT((idx < IMAGE_READER_MAX_NUM_PLANES) && (idx >= 0));
@@ -553,7 +534,7 @@ static jint Image_imageGetRowStride(JNIEnv* env, CpuConsumer::LockedBuffer* buff
 
     int32_t fmt = buffer->flexFormat;
 
-    fmt = applyFormatOverrides(fmt, readerFormat);
+    fmt = applyFormatOverrides(fmt, halReaderFormat);
 
     switch (fmt) {
         case HAL_PIXEL_FORMAT_YCbCr_420_888:
@@ -682,11 +663,16 @@ static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz,
 {
     status_t res;
     int nativeFormat;
+    android_dataspace nativeDataspace;
 
     ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d",
           __FUNCTION__, width, height, format, maxImages);
 
-    nativeFormat = Image_getPixelFormat(env, format);
+    PublicFormat publicFormat = static_cast<PublicFormat>(format);
+    nativeFormat = android_view_Surface_mapPublicFormatToHalFormat(
+        publicFormat);
+    nativeDataspace = android_view_Surface_mapPublicFormatToHalDataspace(
+        publicFormat);
 
     sp<IGraphicBufferProducer> gbProducer;
     sp<IGraphicBufferConsumer> gbConsumer;
@@ -710,10 +696,11 @@ static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz,
     consumer->setFrameAvailableListener(ctx);
     ImageReader_setNativeContext(env, thiz, ctx);
     ctx->setBufferFormat(nativeFormat);
+    ctx->setBufferDataspace(nativeDataspace);
     ctx->setBufferWidth(width);
     ctx->setBufferHeight(height);
 
-    // Set the width/height/format to the CpuConsumer
+    // Set the width/height/format/dataspace to the CpuConsumer
     res = consumer->setDefaultBufferSize(width, height);
     if (res != OK) {
         jniThrowException(env, "java/lang/IllegalStateException",
@@ -725,6 +712,12 @@ static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz,
         jniThrowException(env, "java/lang/IllegalStateException",
                           "Failed to set CpuConsumer buffer format");
     }
+    res = consumer->setDefaultBufferDataSpace(nativeDataspace);
+    if (res != OK) {
+        jniThrowException(env, "java/lang/IllegalStateException",
+                          "Failed to set CpuConsumer buffer dataSpace");
+    }
+
 }
 
 static void ImageReader_close(JNIEnv* env, jobject thiz)
@@ -884,6 +877,8 @@ static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz)
 static jobject Image_createSurfacePlane(JNIEnv* env, jobject thiz, int idx, int readerFormat)
 {
     int rowStride, pixelStride;
+    PublicFormat publicReaderFormat = static_cast<PublicFormat>(readerFormat);
+
     ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
 
     CpuConsumer::LockedBuffer* buffer = Image_getLockedBuffer(env, thiz);
@@ -893,10 +888,11 @@ static jobject Image_createSurfacePlane(JNIEnv* env, jobject thiz, int idx, int
         jniThrowException(env, "java/lang/IllegalStateException", "Image was released");
     }
 
-    readerFormat = Image_getPixelFormat(env, readerFormat);
+    int halReaderFormat = android_view_Surface_mapPublicFormatToHalFormat(
+        publicReaderFormat);
 
-    rowStride = Image_imageGetRowStride(env, buffer, idx, readerFormat);
-    pixelStride = Image_imageGetPixelStride(env, buffer, idx, readerFormat);
+    rowStride = Image_imageGetRowStride(env, buffer, idx, halReaderFormat);
+    pixelStride = Image_imageGetPixelStride(env, buffer, idx, halReaderFormat);
 
     jobject surfPlaneObj = env->NewObject(gSurfacePlaneClassInfo.clazz,
             gSurfacePlaneClassInfo.ctor, thiz, idx, rowStride, pixelStride);
@@ -909,6 +905,7 @@ static jobject Image_getByteBuffer(JNIEnv* env, jobject thiz, int idx, int reade
     uint8_t *base = NULL;
     uint32_t size = 0;
     jobject byteBuffer;
+    PublicFormat readerPublicFormat = static_cast<PublicFormat>(readerFormat);
 
     ALOGV("%s: buffer index: %d", __FUNCTION__, idx);
 
@@ -918,10 +915,11 @@ static jobject Image_getByteBuffer(JNIEnv* env, jobject thiz, int idx, int reade
         jniThrowException(env, "java/lang/IllegalStateException", "Image was released");
     }
 
-    readerFormat = Image_getPixelFormat(env, readerFormat);
+    int readerHalFormat = android_view_Surface_mapPublicFormatToHalFormat(
+            readerPublicFormat);
 
     // Create byteBuffer from native buffer
-    Image_getLockedBufferInfo(env, buffer, idx, &base, &size, readerFormat);
+    Image_getLockedBufferInfo(env, buffer, idx, &base, &size, readerHalFormat);
 
     if (size > static_cast<uint32_t>(INT32_MAX)) {
         // Byte buffer have 'int capacity', so check the range