camera2: Add shim frame duration and stall characteristics.

Bug: 15116722
Change-Id: Ief8b05d46fa12f63cf6a5d41e312c94d5a033553

8 files changed, 104 insertions, 25 deletions

diff --git a/core/java/android/hardware/camera2/CameraMetadata.java b/core/java/android/hardware/camera2/CameraMetadata.java
index 5a02435..dad1854 100644
--- a/core/java/android/hardware/camera2/CameraMetadata.java
+++ b/core/java/android/hardware/camera2/CameraMetadata.java
@@ -248,7 +248,6 @@ public abstract class CameraMetadata<TKey> {
      * <li>Manual frame duration control<ul>
      * <li>{@link CaptureRequest#SENSOR_FRAME_DURATION android.sensor.frameDuration}</li>
      * <li>{@link CameraCharacteristics#SENSOR_INFO_MAX_FRAME_DURATION android.sensor.info.maxFrameDuration}</li>
-     * <li>{@link CameraCharacteristics#SCALER_STREAM_CONFIGURATION_MAP android.scaler.streamConfigurationMap}</li>
      * </ul>
      * </li>
      * <li>Manual exposure control<ul>
@@ -279,6 +278,9 @@ public abstract class CameraMetadata<TKey> {
      * result.</p>
      * <p>A given camera device may also support additional manual sensor controls,
      * but this capability only covers the above list of controls.</p>
+     * <p>If this is supported, {@link CameraCharacteristics#SCALER_STREAM_CONFIGURATION_MAP android.scaler.streamConfigurationMap} will
+     * additionally return a min frame duration that is greater than
+     * zero for each supported size-format combination.</p>
      *
      * @see CaptureRequest#BLACK_LEVEL_LOCK
      * @see CameraCharacteristics#SCALER_STREAM_CONFIGURATION_MAP
diff --git a/core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java b/core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java
index 54d9c3c..aa2f026 100644
--- a/core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java
+++ b/core/java/android/hardware/camera2/legacy/CameraDeviceUserShim.java
@@ -169,7 +169,6 @@ public class CameraDeviceUserShim implements ICameraDeviceUser {
             }
             int numSurfaces = mSurfaces.size();
             if (numSurfaces > 0) {
-                surfaces = new ArrayList<Surface>();
                 for (int i = 0; i < numSurfaces; ++i) {
                     surfaces.add(mSurfaces.valueAt(i));
                 }
diff --git a/core/java/android/hardware/camera2/legacy/GLThreadManager.java b/core/java/android/hardware/camera2/legacy/GLThreadManager.java
index 0382557..5d44fd2 100644
--- a/core/java/android/hardware/camera2/legacy/GLThreadManager.java
+++ b/core/java/android/hardware/camera2/legacy/GLThreadManager.java
@@ -100,6 +100,7 @@ public class GLThreadManager {
                     break;
                 case MSG_ALLOW_FRAMES:
                     mDroppingFrames = false;
+                    break;
                 default:
                     Log.e(TAG, "Unhandled message " + msg.what + " on GLThread.");
                     break;
diff --git a/core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java b/core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java
index c34a34d..3f9c6ed 100644
--- a/core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java
+++ b/core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java
@@ -50,6 +50,7 @@ public class LegacyCameraDevice implements AutoCloseable {
     public static final String DEBUG_PROP = "HAL1ShimLogging";
     private final String TAG;
 
+    private static final boolean DEBUG = false;
     private final int mCameraId;
     private final ICameraDeviceCallbacks mDeviceCallbacks;
     private final CameraDeviceState mDeviceState = new CameraDeviceState();
@@ -84,6 +85,9 @@ public class LegacyCameraDevice implements AutoCloseable {
             mResultHandler.post(new Runnable() {
                 @Override
                 public void run() {
+                    if (DEBUG) {
+                        Log.d(TAG, "doing onError callback.");
+                    }
                     try {
                         mDeviceCallbacks.onCameraError(errorCode, extras);
                     } catch (RemoteException e) {
@@ -92,13 +96,14 @@ public class LegacyCameraDevice implements AutoCloseable {
                     }
                 }
             });
-
-
         }
 
         @Override
         public void onConfiguring() {
             // Do nothing
+            if (DEBUG) {
+                Log.d(TAG, "doing onConfiguring callback.");
+            }
         }
 
         @Override
@@ -108,6 +113,9 @@ public class LegacyCameraDevice implements AutoCloseable {
             mResultHandler.post(new Runnable() {
                 @Override
                 public void run() {
+                    if (DEBUG) {
+                        Log.d(TAG, "doing onIdle callback.");
+                    }
                     try {
                         mDeviceCallbacks.onCameraIdle();
                     } catch (RemoteException e) {
@@ -126,6 +134,9 @@ public class LegacyCameraDevice implements AutoCloseable {
             mResultHandler.post(new Runnable() {
                 @Override
                 public void run() {
+                    if (DEBUG) {
+                        Log.d(TAG, "doing onCaptureStarted callback.");
+                    }
                     try {
                         // TODO: Don't fake timestamp
                         mDeviceCallbacks.onCaptureStarted(extras, timestamp);
@@ -135,7 +146,6 @@ public class LegacyCameraDevice implements AutoCloseable {
                     }
                 }
             });
-
         }
 
         @Override
@@ -145,6 +155,9 @@ public class LegacyCameraDevice implements AutoCloseable {
             mResultHandler.post(new Runnable() {
                 @Override
                 public void run() {
+                    if (DEBUG) {
+                        Log.d(TAG, "doing onCaptureResult callback.");
+                    }
                     try {
                         // TODO: Don't fake metadata
                         mDeviceCallbacks.onResultReceived(result, extras);
diff --git a/core/java/android/hardware/camera2/legacy/LegacyMetadataMapper.java b/core/java/android/hardware/camera2/legacy/LegacyMetadataMapper.java
index 8bb066f..e675f87 100644
--- a/core/java/android/hardware/camera2/legacy/LegacyMetadataMapper.java
+++ b/core/java/android/hardware/camera2/legacy/LegacyMetadataMapper.java
@@ -44,6 +44,11 @@ public class LegacyMetadataMapper {
     private static final int HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED = 0x22;
     private static final int HAL_PIXEL_FORMAT_BLOB = 0x21;
 
+    private static final long APPROXIMATE_CAPTURE_DELAY_MS = 200; // ms
+    private static final long APPROXIMATE_SENSOR_AREA = (1 << 20); // 8mp
+    private static final long APPROXIMATE_JPEG_ENCODE_TIME = 600; // ms
+    private static final long NS_PER_MS = 1000000;
+
     /**
      * Create characteristics for a legacy device by mapping the {@code parameters}
      * and {@code info}
@@ -91,9 +96,6 @@ public class LegacyMetadataMapper {
     }
 
     private static void mapStreamConfigs(CameraMetadataNative m, Camera.Parameters p) {
-        // TODO: set non-empty durations
-        m.set(SCALER_AVAILABLE_MIN_FRAME_DURATIONS, new StreamConfigurationDuration[] {} );
-        m.set(SCALER_AVAILABLE_STALL_DURATIONS, new StreamConfigurationDuration[] {} );
 
         ArrayList<StreamConfiguration> availableStreamConfigs = new ArrayList<>();
         /*
@@ -122,10 +124,25 @@ public class LegacyMetadataMapper {
                         String.format("mapStreamConfigs - Skipping non-public format %x", format));
             }
         }
+
+        List<Camera.Size> jpegSizes = p.getSupportedPictureSizes();
         appendStreamConfig(availableStreamConfigs,
                 HAL_PIXEL_FORMAT_BLOB, p.getSupportedPictureSizes());
         m.set(SCALER_AVAILABLE_STREAM_CONFIGURATIONS,
                 availableStreamConfigs.toArray(new StreamConfiguration[0]));
+
+        // No frame durations available
+        m.set(SCALER_AVAILABLE_MIN_FRAME_DURATIONS, new StreamConfigurationDuration[0]);
+
+        StreamConfigurationDuration[] jpegStalls =
+                new StreamConfigurationDuration[jpegSizes.size()];
+        int i = 0;
+        for (Camera.Size s : jpegSizes) {
+            jpegStalls[i++] = new StreamConfigurationDuration(HAL_PIXEL_FORMAT_BLOB, s.width,
+                    s.height, calculateJpegStallDuration(s));
+        }
+        // Set stall durations for jpeg, other formats use default stall duration
+        m.set(SCALER_AVAILABLE_STALL_DURATIONS, jpegStalls);
     }
 
     private static void appendStreamConfig(
@@ -136,4 +153,17 @@ public class LegacyMetadataMapper {
             configs.add(config);
         }
     }
+
+    /**
+     * Return the stall duration for a given output jpeg size in nanoseconds.
+     *
+     * <p>An 8mp image is chosen to have a stall duration of 0.8 seconds.</p>
+     */
+    private static long calculateJpegStallDuration(Camera.Size size) {
+        long baseDuration = APPROXIMATE_CAPTURE_DELAY_MS * NS_PER_MS; // 200ms for capture
+        long area = size.width * (long) size.height;
+        long stallPerArea = APPROXIMATE_JPEG_ENCODE_TIME * NS_PER_MS /
+                APPROXIMATE_SENSOR_AREA; // 600ms stall for 8mp
+        return baseDuration + area * stallPerArea;
+    }
 }
diff --git a/core/java/android/hardware/camera2/legacy/RequestThreadManager.java b/core/java/android/hardware/camera2/legacy/RequestThreadManager.java
index 7b522ff..bf250a1 100644
--- a/core/java/android/hardware/camera2/legacy/RequestThreadManager.java
+++ b/core/java/android/hardware/camera2/legacy/RequestThreadManager.java
@@ -71,6 +71,8 @@ public class RequestThreadManager {
     private static final float ASPECT_RATIO_TOLERANCE = 0.01f;
     private boolean mPreviewRunning = false;
 
+    private volatile long mLastJpegTimestamp;
+    private volatile long mLastPreviewTimestamp;
     private volatile RequestHolder mInFlightPreview;
     private volatile RequestHolder mInFlightJpeg;
 
@@ -78,6 +80,7 @@ public class RequestThreadManager {
     private List<Surface> mCallbackOutputs = new ArrayList<Surface>();
     private GLThreadManager mGLThreadManager;
     private SurfaceTexture mPreviewTexture;
+    private Camera.Parameters mParams;
 
     private Size mIntermediateBufferSize;
 
@@ -86,6 +89,7 @@ public class RequestThreadManager {
     private Surface mDummySurface;
 
     private final FpsCounter mPrevCounter = new FpsCounter("Incoming Preview");
+    private final FpsCounter mRequestCounter = new FpsCounter("Incoming Requests");
 
     /**
      * Container object for Configure messages.
@@ -209,23 +213,34 @@ public class RequestThreadManager {
         }
     };
 
+    private final Camera.ShutterCallback mJpegShutterCallback = new Camera.ShutterCallback() {
+        @Override
+        public void onShutter() {
+            mLastJpegTimestamp = SystemClock.elapsedRealtimeNanos();
+        }
+    };
+
     private final SurfaceTexture.OnFrameAvailableListener mPreviewCallback =
             new SurfaceTexture.OnFrameAvailableListener() {
                 @Override
                 public void onFrameAvailable(SurfaceTexture surfaceTexture) {
-                    if (DEBUG) {
-                        mPrevCounter.countAndLog();
-                    }
                     RequestHolder holder = mInFlightPreview;
                     if (holder == null) {
+                        mGLThreadManager.queueNewFrame(null);
                         Log.w(TAG, "Dropping preview frame.");
-                        mInFlightPreview = null;
                         return;
                     }
+
+                    if (DEBUG) {
+                        mPrevCounter.countAndLog();
+                    }
+                    mInFlightPreview = null;
+
                     if (holder.hasPreviewTargets()) {
                         mGLThreadManager.queueNewFrame(holder.getHolderTargets());
                     }
 
+                    mLastPreviewTimestamp = surfaceTexture.getTimestamp();
                     mReceivedPreview.open();
                 }
             };
@@ -252,7 +267,7 @@ public class RequestThreadManager {
         }
         mInFlightJpeg = request;
         // TODO: Hook up shutter callback to CameraDeviceStateListener#onCaptureStarted
-        mCamera.takePicture(/*shutter*/null, /*raw*/null, mJpegCallback);
+        mCamera.takePicture(mJpegShutterCallback, /*raw*/null, mJpegCallback);
         mPreviewRunning = false;
     }
 
@@ -312,7 +327,7 @@ public class RequestThreadManager {
                     break;
             }
         }
-
+        mParams = mCamera.getParameters();
         if (mPreviewOutputs.size() > 0) {
             List<Size> outputSizes = new ArrayList<>(outputs.size());
             for (Surface s : mPreviewOutputs) {
@@ -323,13 +338,11 @@ public class RequestThreadManager {
 
             Size largestOutput = findLargestByArea(outputSizes);
 
-            Camera.Parameters params = mCamera.getParameters();
-
             // Find largest jpeg dimension - assume to have the same aspect ratio as sensor.
-            List<Size> supportedJpegSizes = convertSizeList(params.getSupportedPictureSizes());
+            List<Size> supportedJpegSizes = convertSizeList(mParams.getSupportedPictureSizes());
             Size largestJpegDimen = findLargestByArea(supportedJpegSizes);
 
-            List<Size> supportedPreviewSizes = convertSizeList(params.getSupportedPreviewSizes());
+            List<Size> supportedPreviewSizes = convertSizeList(mParams.getSupportedPreviewSizes());
 
             // Use smallest preview dimension with same aspect ratio as sensor that is >= than all
             // of the configured output dimensions.  If none exists, fall back to using the largest
@@ -428,6 +441,9 @@ public class RequestThreadManager {
                 return true;
             }
 
+            if (DEBUG) {
+                Log.d(TAG, "Request thread handling message:" + msg.what);
+            }
             switch (msg.what) {
                 case MSG_CONFIGURE_OUTPUTS:
                     ConfigureHolder config = (ConfigureHolder) msg.obj;
@@ -460,6 +476,7 @@ public class RequestThreadManager {
                             nextBurst.first.produceRequestHolders(nextBurst.second);
                     for (RequestHolder holder : requests) {
                         mDeviceState.setCaptureStart(holder);
+                        long timestamp = 0;
                         try {
                             if (holder.hasPreviewTargets()) {
                                 mReceivedPreview.close();
@@ -468,6 +485,7 @@ public class RequestThreadManager {
                                     // TODO: report error to CameraDevice
                                     Log.e(TAG, "Hit timeout for preview callback!");
                                 }
+                                timestamp = mLastPreviewTimestamp;
                             }
                             if (holder.hasJpegTargets()) {
                                 mReceivedJpeg.close();
@@ -478,16 +496,19 @@ public class RequestThreadManager {
                                     Log.e(TAG, "Hit timeout for jpeg callback!");
                                 }
                                 mInFlightJpeg = null;
+                                timestamp = mLastJpegTimestamp;
                             }
                         } catch (IOException e) {
                             // TODO: err handling
                             throw new IOError(e);
                         }
-                        Camera.Parameters params = mCamera.getParameters();
-                        CameraMetadataNative result = convertResultMetadata(params,
-                                holder.getRequest());
+                        CameraMetadataNative result = convertResultMetadata(mParams,
+                                holder.getRequest(), timestamp);
                         mDeviceState.setCaptureResult(holder, result);
                     }
+                    if (DEBUG) {
+                        mRequestCounter.countAndLog();
+                    }
                     break;
                 case MSG_CLEANUP:
                     mCleanup = true;
@@ -507,9 +528,11 @@ public class RequestThreadManager {
     };
 
     private CameraMetadataNative convertResultMetadata(Camera.Parameters params,
-                                                       CaptureRequest request) {
+                                                       CaptureRequest request,
+                                                       long timestamp) {
         CameraMetadataNative result = new CameraMetadataNative();
         result.set(CaptureResult.LENS_FOCAL_LENGTH, params.getFocalLength());
+        result.set(CaptureResult.SENSOR_TIMESTAMP, timestamp);
 
         // TODO: Remaining result metadata tags conversions.
         return result;
diff --git a/core/java/android/hardware/camera2/legacy/SurfaceTextureRenderer.java b/core/java/android/hardware/camera2/legacy/SurfaceTextureRenderer.java
index 49f419f..e9d32f0 100644
--- a/core/java/android/hardware/camera2/legacy/SurfaceTextureRenderer.java
+++ b/core/java/android/hardware/camera2/legacy/SurfaceTextureRenderer.java
@@ -486,6 +486,7 @@ public class SurfaceTextureRenderer {
         }
         checkGlError("before updateTexImage");
         mSurfaceTexture.updateTexImage();
+        if (targetSurfaces == null) return;
         for (EGLSurfaceHolder holder : mSurfaces) {
             if (targetSurfaces.contains(holder.surface)) {
                 makeCurrent(holder.eglSurface);
diff --git a/core/java/android/hardware/camera2/params/StreamConfigurationMap.java b/core/java/android/hardware/camera2/params/StreamConfigurationMap.java
index 3036425..fff171b 100644
--- a/core/java/android/hardware/camera2/params/StreamConfigurationMap.java
+++ b/core/java/android/hardware/camera2/params/StreamConfigurationMap.java
@@ -63,6 +63,12 @@ import static com.android.internal.util.Preconditions.*;
 public final class StreamConfigurationMap {
 
     private static final String TAG = "StreamConfigurationMap";
+
+    /**
+     * Indicates that a minimum frame duration is not available for a particular configuration.
+     */
+    public static final long NO_MIN_FRAME_DURATION = 0;
+
     /**
      * Create a new {@link StreamConfigurationMap}.
      *
@@ -359,7 +365,9 @@ public final class StreamConfigurationMap {
      *
      * @param format an image format from {@link ImageFormat} or {@link PixelFormat}
      * @param size an output-compatible size
-     * @return a minimum frame duration {@code >=} 0 in nanoseconds
+     * @return a minimum frame duration {@code >} 0 in nanoseconds, or
+     *          {@link #NO_MIN_FRAME_DURATION} if the minimum frame duration is not available (this
+     *          can only occur on limited mode devices).
      *
      * @throws IllegalArgumentException if {@code format} or {@code size} was not supported
      * @throws NullPointerException if {@code size} was {@code null}
@@ -406,7 +414,9 @@ public final class StreamConfigurationMap {
      *          a class which is supported by {@link #isOutputSupportedFor(Class)} and has a
      *          non-empty array returned by {@link #getOutputSizes(Class)}
      * @param size an output-compatible size
-     * @return a minimum frame duration {@code >=} 0 in nanoseconds
+     * @return a minimum frame duration {@code >} 0 in nanoseconds, or
+     *          {@link #NO_MIN_FRAME_DURATION} if the minimum frame duration is not available (this
+     *          can only occur on limited mode devices).
      *
      * @throws IllegalArgumentException if {@code klass} or {@code size} was not supported
      * @throws NullPointerException if {@code size} or {@code klass} was {@code null}
@@ -892,7 +902,7 @@ public final class StreamConfigurationMap {
     private long getDurationDefault(int duration) {
         switch (duration) {
             case DURATION_MIN_FRAME:
-                throw new AssertionError("Minimum frame durations are required to be listed");
+                return NO_MIN_FRAME_DURATION;
             case DURATION_STALL:
                 return 0L; // OK. A lack of a stall duration implies a 0 stall duration
             default: