Merge "Camera2: Allow rendering to arbitrary surface sizes in LEGACY mode." into lmp-mr1-dev

4 files changed, 137 insertions, 81 deletions

diff --git a/core/java/android/hardware/camera2/legacy/GLThreadManager.java b/core/java/android/hardware/camera2/legacy/GLThreadManager.java
index 64c532b..b160d2a 100644
--- a/core/java/android/hardware/camera2/legacy/GLThreadManager.java
+++ b/core/java/android/hardware/camera2/legacy/GLThreadManager.java
@@ -22,6 +22,8 @@ import android.os.ConditionVariable;
 import android.os.Handler;
 import android.os.Message;
 import android.util.Log;
+import android.util.Pair;
+import android.util.Size;
 import android.view.Surface;
 
 import java.util.Collection;
@@ -57,11 +59,11 @@ public class GLThreadManager {
      */
     private static class ConfigureHolder {
         public final ConditionVariable condition;
-        public final Collection<Surface> surfaces;
+        public final Collection<Pair<Surface, Size>> surfaces;
         public final CaptureCollector collector;
 
-        public ConfigureHolder(ConditionVariable condition, Collection<Surface> surfaces,
-                               CaptureCollector collector) {
+        public ConfigureHolder(ConditionVariable condition, Collection<Pair<Surface,
+                Size>> surfaces, CaptureCollector collector) {
             this.condition = condition;
             this.surfaces = surfaces;
             this.collector = collector;
@@ -202,10 +204,12 @@ public class GLThreadManager {
      * Configure the GL renderer for the given set of output surfaces, and block until
      * this configuration has been applied.
      *
-     * @param surfaces a collection of {@link android.view.Surface}s to configure.
+     * @param surfaces a collection of pairs of {@link android.view.Surface}s and their
+     *                 corresponding sizes to configure.
      * @param collector a {@link CaptureCollector} to retrieve requests from.
      */
-    public void setConfigurationAndWait(Collection<Surface> surfaces, CaptureCollector collector) {
+    public void setConfigurationAndWait(Collection<Pair<Surface, Size>> surfaces,
+                                        CaptureCollector collector) {
         checkNotNull(collector, "collector must not be null");
         Handler handler = mGLHandlerThread.getHandler();
 
diff --git a/core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java b/core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java
index 3a976ba..3043d13 100644
--- a/core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java
+++ b/core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java
@@ -24,7 +24,6 @@ import android.hardware.camera2.CaptureRequest;
 import android.hardware.camera2.impl.CameraDeviceImpl;
 import android.hardware.camera2.impl.CaptureResultExtras;
 import android.hardware.camera2.ICameraDeviceCallbacks;
-import android.hardware.camera2.params.StreamConfiguration;
 import android.hardware.camera2.params.StreamConfigurationMap;
 import android.hardware.camera2.utils.ArrayUtils;
 import android.hardware.camera2.utils.CameraBinderDecorator;
@@ -36,6 +35,7 @@ import android.os.Handler;
 import android.os.HandlerThread;
 import android.os.RemoteException;
 import android.util.Log;
+import android.util.Pair;
 import android.util.Size;
 import android.view.Surface;
 
@@ -78,6 +78,15 @@ public class LegacyCameraDevice implements AutoCloseable {
     private final Handler mResultHandler;
     private static final int ILLEGAL_VALUE = -1;
 
+    // Keep up to date with values in hardware/libhardware/include/hardware/gralloc.h
+    private static final int GRALLOC_USAGE_RENDERSCRIPT = 0x00100000;
+    private static final int GRALLOC_USAGE_SW_READ_OFTEN = 0x00000003;
+    private static final int GRALLOC_USAGE_HW_TEXTURE = 0x00000100;
+    private static final int GRALLOC_USAGE_HW_COMPOSER = 0x00000800;
+    private static final int GRALLOC_USAGE_HW_VIDEO_ENCODER = 0x00010000;
+
+    private static final int MAX_DIMEN_FOR_ROUNDING = 1080; // maximum allowed width for rounding
+
     private CaptureResultExtras getExtrasFromRequest(RequestHolder holder) {
         if (holder == null) {
             return new CaptureResultExtras(ILLEGAL_VALUE, ILLEGAL_VALUE, ILLEGAL_VALUE,
@@ -276,6 +285,7 @@ public class LegacyCameraDevice implements AutoCloseable {
      *          on success.
      */
     public int configureOutputs(List<Surface> outputs) {
+        List<Pair<Surface, Size>> sizedSurfaces = new ArrayList<>();
         if (outputs != null) {
             for (Surface output : outputs) {
                 if (output == null) {
@@ -289,16 +299,25 @@ public class LegacyCameraDevice implements AutoCloseable {
                 try {
                     Size s = getSurfaceSize(output);
                     int surfaceType = detectSurfaceType(output);
-                    Size[] sizes = streamConfigurations.getOutputSizes(surfaceType);
+                    int usageFlags = detectSurfaceUsageFlags(output);
 
+                    // Keep up to date with allowed consumer types in
+                    // frameworks/av/services/camera/libcameraservice/api2/CameraDeviceClient.cpp
+                    int disallowedFlags = GRALLOC_USAGE_HW_VIDEO_ENCODER | GRALLOC_USAGE_RENDERSCRIPT;
+                    int allowedFlags = GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_SW_READ_OFTEN |
+                            GRALLOC_USAGE_HW_COMPOSER;
+                    boolean flexibleConsumer = ((usageFlags & disallowedFlags) == 0 &&
+                            (usageFlags & allowedFlags) != 0);
+
+                    Size[] sizes = streamConfigurations.getOutputSizes(surfaceType);
                     if (sizes == null) {
                         // WAR: Override default format to IMPLEMENTATION_DEFINED for b/9487482
                         if ((surfaceType >= LegacyMetadataMapper.HAL_PIXEL_FORMAT_RGBA_8888 &&
                             surfaceType <= LegacyMetadataMapper.HAL_PIXEL_FORMAT_BGRA_8888)) {
 
-                            // YUV_420_888 is always present in LEGACY for all IMPLEMENTATION_DEFINED
-                            // output sizes, and is publicly visible in the API (i.e.
-                            // {@code #getOutputSizes} works here).
+                            // YUV_420_888 is always present in LEGACY for all
+                            // IMPLEMENTATION_DEFINED output sizes, and is publicly visible in the
+                            // API (i.e. {@code #getOutputSizes} works here).
                             sizes = streamConfigurations.getOutputSizes(ImageFormat.YUV_420_888);
                         } else if (surfaceType == LegacyMetadataMapper.HAL_PIXEL_FORMAT_BLOB) {
                             sizes = streamConfigurations.getOutputSizes(ImageFormat.JPEG);
@@ -306,12 +325,18 @@ public class LegacyCameraDevice implements AutoCloseable {
                     }
 
                     if (!ArrayUtils.contains(sizes, s)) {
-                        String reason = (sizes == null) ? "format is invalid." :
-                                ("size not in valid set: " + Arrays.toString(sizes));
-                        Log.e(TAG, String.format("Surface with size (w=%d, h=%d) and format 0x%x is"
-                                + " not valid, %s", s.getWidth(), s.getHeight(), surfaceType,
-                                reason));
-                        return BAD_VALUE;
+                        if (flexibleConsumer && (s = findClosestSize(s, sizes)) != null) {
+                            sizedSurfaces.add(new Pair<>(output, s));
+                        } else {
+                            String reason = (sizes == null) ? "format is invalid." :
+                                    ("size not in valid set: " + Arrays.toString(sizes));
+                            Log.e(TAG, String.format("Surface with size (w=%d, h=%d) and format " +
+                                    "0x%x is not valid, %s", s.getWidth(), s.getHeight(),
+                                    surfaceType, reason));
+                            return BAD_VALUE;
+                        }
+                    } else {
+                        sizedSurfaces.add(new Pair<>(output, s));
                     }
                 } catch (BufferQueueAbandonedException e) {
                     Log.e(TAG, "Surface bufferqueue is abandoned, cannot configure as output: ", e);
@@ -323,7 +348,7 @@ public class LegacyCameraDevice implements AutoCloseable {
 
         boolean success = false;
         if (mDeviceState.setConfiguring()) {
-            mRequestThreadManager.configure(outputs);
+            mRequestThreadManager.configure(sizedSurfaces);
             success = mDeviceState.setIdle();
         }
 
@@ -473,6 +498,31 @@ public class LegacyCameraDevice implements AutoCloseable {
         }
     }
 
+    static long findEuclidDistSquare(Size a, Size b) {
+        long d0 = a.getWidth() - b.getWidth();
+        long d1 = a.getHeight() - b.getHeight();
+        return d0 * d0 + d1 * d1;
+    }
+
+    // Keep up to date with rounding behavior in
+    // frameworks/av/services/camera/libcameraservice/api2/CameraDeviceClient.cpp
+    static Size findClosestSize(Size size, Size[] supportedSizes) {
+        if (size == null || supportedSizes == null) {
+            return null;
+        }
+        Size bestSize = null;
+        for (Size s : supportedSizes) {
+            if (s.equals(size)) {
+                return size;
+            } else if (s.getWidth() <= MAX_DIMEN_FOR_ROUNDING && (bestSize == null ||
+                    LegacyCameraDevice.findEuclidDistSquare(size, s) <
+                    LegacyCameraDevice.findEuclidDistSquare(bestSize, s))) {
+                bestSize = s;
+            }
+        }
+        return bestSize;
+    }
+
     /**
      * Query the surface for its currently configured default buffer size.
      * @param surface a non-{@code null} {@code Surface}
@@ -490,6 +540,11 @@ public class LegacyCameraDevice implements AutoCloseable {
         return new Size(dimens[0], dimens[1]);
     }
 
+    static int detectSurfaceUsageFlags(Surface surface) {
+        checkNotNull(surface);
+        return nativeDetectSurfaceUsageFlags(surface);
+    }
+
     static int detectSurfaceType(Surface surface) throws BufferQueueAbandonedException {
         checkNotNull(surface);
         return LegacyExceptionUtils.throwOnError(nativeDetectSurfaceType(surface));
@@ -608,5 +663,7 @@ public class LegacyCameraDevice implements AutoCloseable {
 
     private static native int nativeSetNextTimestamp(Surface surface, long timestamp);
 
+    private static native int nativeDetectSurfaceUsageFlags(Surface surface);
+
     static native int nativeGetJpegFooterSize();
 }
diff --git a/core/java/android/hardware/camera2/legacy/RequestThreadManager.java b/core/java/android/hardware/camera2/legacy/RequestThreadManager.java
index 35deb71..6535a4e 100644
--- a/core/java/android/hardware/camera2/legacy/RequestThreadManager.java
+++ b/core/java/android/hardware/camera2/legacy/RequestThreadManager.java
@@ -41,6 +41,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
@@ -116,9 +117,10 @@ public class RequestThreadManager {
      */
     private static class ConfigureHolder {
         public final ConditionVariable condition;
-        public final Collection<Surface> surfaces;
+        public final Collection<Pair<Surface, Size>> surfaces;
 
-        public ConfigureHolder(ConditionVariable condition, Collection<Surface> surfaces) {
+        public ConfigureHolder(ConditionVariable condition, Collection<Pair<Surface,
+                Size>> surfaces) {
             this.condition = condition;
             this.surfaces = surfaces;
         }
@@ -317,7 +319,7 @@ public class RequestThreadManager {
         startPreview();
     }
 
-    private void configureOutputs(Collection<Surface> outputs) {
+    private void configureOutputs(Collection<Pair<Surface, Size>> outputs) {
         if (DEBUG) {
             String outputsStr = outputs == null ? "null" : (outputs.size() + " surfaces");
             Log.d(TAG, "configureOutputs with " + outputsStr);
@@ -346,10 +348,15 @@ public class RequestThreadManager {
         mJpegSurfaceIds.clear();
         mPreviewTexture = null;
 
+        List<Size> previewOutputSizes = new ArrayList<>();
+        List<Size> callbackOutputSizes = new ArrayList<>();
+
         int facing = mCharacteristics.get(CameraCharacteristics.LENS_FACING);
         int orientation = mCharacteristics.get(CameraCharacteristics.SENSOR_ORIENTATION);
         if (outputs != null) {
-            for (Surface s : outputs) {
+            for (Pair<Surface, Size> outPair : outputs) {
+                Surface s = outPair.first;
+                Size outSize = outPair.second;
                 try {
                     int format = LegacyCameraDevice.detectSurfaceType(s);
                     LegacyCameraDevice.setSurfaceOrientation(s, facing, orientation);
@@ -362,9 +369,11 @@ public class RequestThreadManager {
                             }
                             mJpegSurfaceIds.add(LegacyCameraDevice.getSurfaceId(s));
                             mCallbackOutputs.add(s);
+                            callbackOutputSizes.add(outSize);
                             break;
                         default:
                             mPreviewOutputs.add(s);
+                            previewOutputSizes.add(outSize);
                             break;
                     }
                 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
@@ -391,18 +400,9 @@ public class RequestThreadManager {
         mParams.setPreviewFpsRange(bestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX],
                 bestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]);
 
-        if (mPreviewOutputs.size() > 0) {
-            List<Size> outputSizes = new ArrayList<>(outputs.size());
-            for (Surface s : mPreviewOutputs) {
-                try {
-                    Size size = LegacyCameraDevice.getSurfaceSize(s);
-                    outputSizes.add(size);
-                } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
-                    Log.w(TAG, "Surface abandoned, skipping...", e);
-                }
-            }
+        if (previewOutputSizes.size() > 0) {
 
-            Size largestOutput = SizeAreaComparator.findLargestByArea(outputSizes);
+            Size largestOutput = SizeAreaComparator.findLargestByArea(previewOutputSizes);
 
             // Find largest jpeg dimension - assume to have the same aspect ratio as sensor.
             Size largestJpegDimen = ParameterUtils.getLargestSupportedJpegSizeByArea(mParams);
@@ -439,7 +439,8 @@ public class RequestThreadManager {
             }
         }
 
-        Size smallestSupportedJpegSize = calculatePictureSize(mCallbackOutputs, mParams);
+        Size smallestSupportedJpegSize = calculatePictureSize(mCallbackOutputs,
+                callbackOutputSizes, mParams);
         if (smallestSupportedJpegSize != null) {
             /*
              * Set takePicture size to the smallest supported JPEG size large enough
@@ -457,7 +458,12 @@ public class RequestThreadManager {
             mGLThreadManager.start();
         }
         mGLThreadManager.waitUntilStarted();
-        mGLThreadManager.setConfigurationAndWait(mPreviewOutputs, mCaptureCollector);
+        List<Pair<Surface, Size>> previews = new ArrayList<>();
+        Iterator<Size> previewSizeIter = previewOutputSizes.iterator();
+        for (Surface p : mPreviewOutputs) {
+            previews.add(new Pair<>(p, previewSizeIter.next()));
+        }
+        mGLThreadManager.setConfigurationAndWait(previews, mCaptureCollector);
         mGLThreadManager.allowNewFrames();
         mPreviewTexture = mGLThreadManager.getCurrentSurfaceTexture();
         if (mPreviewTexture != null) {
@@ -499,26 +505,25 @@ public class RequestThreadManager {
      *          {@code null} if the {@code callbackOutputs} did not have any {@code JPEG}
      *          surfaces.
      */
-    private Size calculatePictureSize(
-            Collection<Surface> callbackOutputs, Camera.Parameters params) {
+    private Size calculatePictureSize( List<Surface> callbackOutputs,
+                                       List<Size> callbackSizes, Camera.Parameters params) {
         /*
          * Find the largest JPEG size (if any), from the configured outputs:
          * - the api1 picture size should be set to the smallest legal size that's at least as large
          *   as the largest configured JPEG size
          */
-        List<Size> configuredJpegSizes = new ArrayList<Size>();
+        if (callbackOutputs.size() != callbackSizes.size()) {
+            throw new IllegalStateException("Input collections must be same length");
+        }
+        List<Size> configuredJpegSizes = new ArrayList<>();
+        Iterator<Size> sizeIterator = callbackSizes.iterator();
         for (Surface callbackSurface : callbackOutputs) {
-            try {
-
+            Size jpegSize = sizeIterator.next();
                 if (!LegacyCameraDevice.containsSurfaceId(callbackSurface, mJpegSurfaceIds)) {
                     continue; // Ignore non-JPEG callback formats
                 }
 
-                Size jpegSize = LegacyCameraDevice.getSurfaceSize(callbackSurface);
                 configuredJpegSizes.add(jpegSize);
-            } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
-                Log.w(TAG, "Surface abandoned, skipping...", e);
-            }
         }
         if (!configuredJpegSizes.isEmpty()) {
             /*
@@ -994,7 +999,7 @@ public class RequestThreadManager {
      *
      * @param outputs a {@link java.util.Collection} of outputs to configure.
      */
-    public void configure(Collection<Surface> outputs) {
+    public void configure(Collection<Pair<Surface, Size>> outputs) {
         Handler handler = mRequestThread.waitAndGetHandler();
         final ConditionVariable condition = new ConditionVariable(/*closed*/false);
         ConfigureHolder holder = new ConfigureHolder(condition, outputs);
diff --git a/core/java/android/hardware/camera2/legacy/SurfaceTextureRenderer.java b/core/java/android/hardware/camera2/legacy/SurfaceTextureRenderer.java
index c0d1d5e..4853b81 100644
--- a/core/java/android/hardware/camera2/legacy/SurfaceTextureRenderer.java
+++ b/core/java/android/hardware/camera2/legacy/SurfaceTextureRenderer.java
@@ -397,16 +397,9 @@ public class SurfaceTextureRenderer {
                 EGL14.EGL_NONE
         };
         for (EGLSurfaceHolder holder : surfaces) {
-            try {
-                Size size = LegacyCameraDevice.getSurfaceSize(holder.surface);
-                holder.width = size.getWidth();
-                holder.height = size.getHeight();
-                holder.eglSurface = EGL14.eglCreateWindowSurface(mEGLDisplay, mConfigs,
-                        holder.surface, surfaceAttribs, /*offset*/ 0);
-                checkEglError("eglCreateWindowSurface");
-            } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
-                Log.w(TAG, "Surface abandoned, skipping...", e);
-            }
+            holder.eglSurface = EGL14.eglCreateWindowSurface(mEGLDisplay, mConfigs,
+                    holder.surface, surfaceAttribs, /*offset*/ 0);
+            checkEglError("eglCreateWindowSurface");
         }
     }
 
@@ -417,24 +410,17 @@ public class SurfaceTextureRenderer {
 
         int maxLength = 0;
         for (EGLSurfaceHolder holder : surfaces) {
-            try {
-                Size size = LegacyCameraDevice.getSurfaceSize(holder.surface);
-                int length = size.getWidth() * size.getHeight();
-                // Find max surface size, ensure PBuffer can hold this many pixels
-                maxLength = (length > maxLength) ? length : maxLength;
-                int[] surfaceAttribs = {
-                        EGL14.EGL_WIDTH, size.getWidth(),
-                        EGL14.EGL_HEIGHT, size.getHeight(),
-                        EGL14.EGL_NONE
-                };
-                holder.width = size.getWidth();
-                holder.height = size.getHeight();
-                holder.eglSurface =
-                        EGL14.eglCreatePbufferSurface(mEGLDisplay, mConfigs, surfaceAttribs, 0);
-                checkEglError("eglCreatePbufferSurface");
-            } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
-                Log.w(TAG, "Surface abandoned, skipping...", e);
-            }
+            int length = holder.width * holder.height;
+            // Find max surface size, ensure PBuffer can hold this many pixels
+            maxLength = (length > maxLength) ? length : maxLength;
+            int[] surfaceAttribs = {
+                    EGL14.EGL_WIDTH, holder.width,
+                    EGL14.EGL_HEIGHT, holder.height,
+                    EGL14.EGL_NONE
+            };
+            holder.eglSurface =
+                    EGL14.eglCreatePbufferSurface(mEGLDisplay, mConfigs, surfaceAttribs, 0);
+            checkEglError("eglCreatePbufferSurface");
         }
         mPBufferPixels = ByteBuffer.allocateDirect(maxLength * PBUFFER_PIXEL_BYTES)
                 .order(ByteOrder.nativeOrder());
@@ -569,7 +555,7 @@ public class SurfaceTextureRenderer {
      *
      * @param surfaces a {@link Collection} of surfaces.
      */
-    public void configureSurfaces(Collection<Surface> surfaces) {
+    public void configureSurfaces(Collection<Pair<Surface, Size>> surfaces) {
         releaseEGLContext();
 
         if (surfaces == null || surfaces.size() == 0) {
@@ -577,18 +563,20 @@ public class SurfaceTextureRenderer {
             return;
         }
 
-        for (Surface s : surfaces) {
+        for (Pair<Surface, Size> p : surfaces) {
+            Surface s = p.first;
+            Size surfaceSize = p.second;
             // If pixel conversions aren't handled by egl, use a pbuffer
             try {
+                EGLSurfaceHolder holder = new EGLSurfaceHolder();
+                holder.surface = s;
+                holder.width = surfaceSize.getWidth();
+                holder.height = surfaceSize.getHeight();
                 if (LegacyCameraDevice.needsConversion(s)) {
                     // Always override to YV12 output for YUV surface formats.
                     LegacyCameraDevice.setSurfaceFormat(s, ImageFormat.YV12);
-                    EGLSurfaceHolder holder = new EGLSurfaceHolder();
-                    holder.surface = s;
                     mConversionSurfaces.add(holder);
                 } else {
-                    EGLSurfaceHolder holder = new EGLSurfaceHolder();
-                    holder.surface = s;
                     mSurfaces.add(holder);
                 }
             } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
@@ -672,10 +660,11 @@ public class SurfaceTextureRenderer {
         List<Long> targetSurfaceIds = LegacyCameraDevice.getSurfaceIds(targetSurfaces);
         for (EGLSurfaceHolder holder : mSurfaces) {
             if (LegacyCameraDevice.containsSurfaceId(holder.surface, targetSurfaceIds)) {
-                makeCurrent(holder.eglSurface);
-                try {
+                try{
                     LegacyCameraDevice.setSurfaceDimens(holder.surface, holder.width,
                             holder.height);
+                    makeCurrent(holder.eglSurface);
+
                     LegacyCameraDevice.setNextTimestamp(holder.surface, captureHolder.second);
                     drawFrame(mSurfaceTexture, holder.width, holder.height);
                     swapBuffers(holder.eglSurface);
@@ -695,10 +684,11 @@ public class SurfaceTextureRenderer {
 
                 try {
                     int format = LegacyCameraDevice.detectSurfaceType(holder.surface);
+                    LegacyCameraDevice.setSurfaceDimens(holder.surface, holder.width,
+                            holder.height);
                     LegacyCameraDevice.setNextTimestamp(holder.surface, captureHolder.second);
                     LegacyCameraDevice.produceFrame(holder.surface, mPBufferPixels.array(),
                             holder.width, holder.height, format);
-                    swapBuffers(holder.eglSurface);
                 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
                     Log.w(TAG, "Surface abandoned, dropping frame. ", e);
                 }