Camera2Client: handle slower jpeg stream sensor

If largest jpeg stream cannot sustain 30 FPS, don't
create jpeg stream until takePicture is called and remove
it after still capture is done.
Also, disable video snapshot for such sensors so video snapshot
won't slow down video recording.

Bug: 22231605
Change-Id: I2b34d2537c224694ae10f2006b5a46be45a1b1a6

5 files changed, 154 insertions, 18 deletions

diff --git a/services/camera/libcameraservice/api1/Camera2Client.cpp b/services/camera/libcameraservice/api1/Camera2Client.cpp
index e109595..36e99dd 100644
--- a/services/camera/libcameraservice/api1/Camera2Client.cpp
+++ b/services/camera/libcameraservice/api1/Camera2Client.cpp
@@ -764,16 +764,22 @@ status_t Camera2Client::startPreviewL(Parameters &params, bool restart) {
     // first capture latency on HAL3 devices, and potentially on some HAL2
     // devices. So create it unconditionally at preview start. As a drawback,
     // this increases gralloc memory consumption for applications that don't
-    // ever take a picture.
+    // ever take a picture. Do not enter this mode when jpeg stream will slow
+    // down preview.
     // TODO: Find a better compromise, though this likely would involve HAL
     // changes.
     int lastJpegStreamId = mJpegProcessor->getStreamId();
-    res = updateProcessorStream(mJpegProcessor, params);
-    if (res != OK) {
-        ALOGE("%s: Camera %d: Can't pre-configure still image "
-                "stream: %s (%d)",
-                __FUNCTION__, mCameraId, strerror(-res), res);
-        return res;
+    // If jpeg stream will slow down preview, make sure we remove it before starting preview
+    if (params.slowJpegMode) {
+        mJpegProcessor->deleteStream();
+    } else {
+        res = updateProcessorStream(mJpegProcessor, params);
+        if (res != OK) {
+            ALOGE("%s: Camera %d: Can't pre-configure still image "
+                    "stream: %s (%d)",
+                    __FUNCTION__, mCameraId, strerror(-res), res);
+            return res;
+        }
     }
     bool jpegStreamChanged = mJpegProcessor->getStreamId() != lastJpegStreamId;
 
@@ -1453,9 +1459,12 @@ status_t Camera2Client::takePicture(int msgType) {
         }
 
         ALOGV("%s: Camera %d: Starting picture capture", __FUNCTION__, mCameraId);
-
         int lastJpegStreamId = mJpegProcessor->getStreamId();
-        res = updateProcessorStream(mJpegProcessor, l.mParameters);
+        // slowJpegMode will create jpeg stream in CaptureSequencer before capturing
+        if (!l.mParameters.slowJpegMode) {
+            res = updateProcessorStream(mJpegProcessor, l.mParameters);
+        }
+
         // If video snapshot fail to configureStream, try override video snapshot size to
         // video size
         if (res == BAD_VALUE && l.mParameters.state == Parameters::VIDEO_SNAPSHOT) {
@@ -1943,6 +1952,39 @@ status_t Camera2Client::stopStream() {
     return mStreamingProcessor->stopStream();
 }
 
+status_t Camera2Client::createJpegStreamL(Parameters &params) {
+    status_t res = OK;
+    int lastJpegStreamId = mJpegProcessor->getStreamId();
+    if (lastJpegStreamId != NO_STREAM) {
+        return INVALID_OPERATION;
+    }
+
+    res = mStreamingProcessor->togglePauseStream(/*pause*/true);
+    if (res != OK) {
+        ALOGE("%s: Camera %d: Can't pause streaming: %s (%d)",
+                __FUNCTION__, mCameraId, strerror(-res), res);
+        return res;
+    }
+
+    res = mDevice->flush();
+    if (res != OK) {
+        ALOGE("%s: Camera %d: Unable flush device: %s (%d)",
+                __FUNCTION__, mCameraId, strerror(-res), res);
+        return res;
+    }
+
+    // Ideally we don't need this, but current camera device
+    // status tracking mechanism demands it.
+    res = mDevice->waitUntilDrained();
+    if (res != OK) {
+        ALOGE("%s: Camera %d: Waiting device drain failed: %s (%d)",
+                __FUNCTION__, mCameraId, strerror(-res), res);
+    }
+
+    res = updateProcessorStream(mJpegProcessor, params);
+    return res;
+}
+
 const int32_t Camera2Client::kPreviewRequestIdStart;
 const int32_t Camera2Client::kPreviewRequestIdEnd;
 const int32_t Camera2Client::kRecordingRequestIdStart;
diff --git a/services/camera/libcameraservice/api1/Camera2Client.h b/services/camera/libcameraservice/api1/Camera2Client.h
index c288313..d50bf63 100644
--- a/services/camera/libcameraservice/api1/Camera2Client.h
+++ b/services/camera/libcameraservice/api1/Camera2Client.h
@@ -129,6 +129,9 @@ public:
 
     status_t stopStream();
 
+    // For the slowJpegMode to create jpeg stream when precapture sequence is done
+    status_t createJpegStreamL(camera2::Parameters &params);
+
     static size_t calculateBufferSize(int width, int height,
             int format, int stride);
 
@@ -145,6 +148,9 @@ public:
     static const char* kAutofocusLabel;
     static const char* kTakepictureLabel;
 
+    // Used with stream IDs
+    static const int NO_STREAM = -1;
+
 private:
     /** ICamera interface-related private members */
     typedef camera2::Parameters Parameters;
@@ -177,9 +183,6 @@ private:
     void     setPreviewCallbackFlagL(Parameters &params, int flag);
     status_t updateRequests(Parameters &params);
 
-    // Used with stream IDs
-    static const int NO_STREAM = -1;
-
     template <typename ProcessorT>
     status_t updateProcessorStream(sp<ProcessorT> processor, Parameters params);
     template <typename ProcessorT,
diff --git a/services/camera/libcameraservice/api1/client2/CaptureSequencer.cpp b/services/camera/libcameraservice/api1/client2/CaptureSequencer.cpp
index d847e0f..5f7fd74 100644
--- a/services/camera/libcameraservice/api1/client2/CaptureSequencer.cpp
+++ b/services/camera/libcameraservice/api1/client2/CaptureSequencer.cpp
@@ -504,6 +504,17 @@ CaptureSequencer::CaptureState CaptureSequencer::manageStandardCapture(
      *  - recording (if recording enabled)
      */
     outputStreams.push(client->getPreviewStreamId());
+
+    int captureStreamId = client->getCaptureStreamId();
+    if (captureStreamId == Camera2Client::NO_STREAM) {
+        res = client->createJpegStreamL(l.mParameters);
+        if (res != OK || client->getCaptureStreamId() == Camera2Client::NO_STREAM) {
+            ALOGE("%s: Camera %d: cannot create jpeg stream for slowJpeg mode: %s (%d)",
+                  __FUNCTION__, client->getCameraId(), strerror(-res), res);
+            return DONE;
+        }
+    }
+
     outputStreams.push(client->getCaptureStreamId());
 
     if (l.mParameters.previewCallbackFlags &
diff --git a/services/camera/libcameraservice/api1/client2/Parameters.cpp b/services/camera/libcameraservice/api1/client2/Parameters.cpp
index c3a6842..442eb75 100644
--- a/services/camera/libcameraservice/api1/client2/Parameters.cpp
+++ b/services/camera/libcameraservice/api1/client2/Parameters.cpp
@@ -889,15 +889,30 @@ status_t Parameters::initialize(const CameraMetadata *info, int deviceVersion) {
     previewCallbackOneShot = false;
     previewCallbackSurface = false;
 
+    Size maxJpegSize = getMaxSize(getAvailableJpegSizes());
+    int64_t minFrameDurationNs = getJpegStreamMinFrameDurationNs(maxJpegSize);
+
+    slowJpegMode = false;
+    if (minFrameDurationNs > kSlowJpegModeThreshold) {
+        slowJpegMode = true;
+        // Slow jpeg devices does not support video snapshot without
+        // slowing down preview.
+        // TODO: support video size video snapshot only?
+        params.set(CameraParameters::KEY_VIDEO_SNAPSHOT_SUPPORTED,
+            CameraParameters::FALSE);
+    }
+
     char value[PROPERTY_VALUE_MAX];
     property_get("camera.disable_zsl_mode", value, "0");
-    if (!strcmp(value,"1")) {
+    if (!strcmp(value,"1") || slowJpegMode) {
         ALOGI("Camera %d: Disabling ZSL mode", cameraId);
         zslMode = false;
     } else {
         zslMode = true;
     }
 
+    ALOGI("%s: zslMode: %d slowJpegMode %d", __FUNCTION__, zslMode, slowJpegMode);
+
     lightFx = LIGHTFX_NONE;
 
     state = STOPPED;
@@ -2778,6 +2793,17 @@ Parameters::Size Parameters::getMaxSizeForRatio(
     return maxSize;
 }
 
+Parameters::Size Parameters::getMaxSize(const Vector<Parameters::Size> &sizes) {
+    Size maxSize = {-1, -1};
+    for (size_t i = 0; i < sizes.size(); i++) {
+        if (sizes[i].width > maxSize.width ||
+                (sizes[i].width == maxSize.width && sizes[i].height > maxSize.height )) {
+            maxSize = sizes[i];
+        }
+    }
+    return maxSize;
+}
+
 Vector<Parameters::StreamConfiguration> Parameters::getStreamConfigurations() {
     const int STREAM_CONFIGURATION_SIZE = 4;
     const int STREAM_FORMAT_OFFSET = 0;
@@ -2792,7 +2818,7 @@ Vector<Parameters::StreamConfiguration> Parameters::getStreamConfigurations() {
 
     camera_metadata_ro_entry_t availableStreamConfigs =
                 staticInfo(ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS);
-    for (size_t i=0; i < availableStreamConfigs.count; i+= STREAM_CONFIGURATION_SIZE) {
+    for (size_t i = 0; i < availableStreamConfigs.count; i+= STREAM_CONFIGURATION_SIZE) {
         int32_t format = availableStreamConfigs.data.i32[i + STREAM_FORMAT_OFFSET];
         int32_t width = availableStreamConfigs.data.i32[i + STREAM_WIDTH_OFFSET];
         int32_t height = availableStreamConfigs.data.i32[i + STREAM_HEIGHT_OFFSET];
@@ -2803,11 +2829,52 @@ Vector<Parameters::StreamConfiguration> Parameters::getStreamConfigurations() {
     return scs;
 }
 
+int64_t Parameters::getJpegStreamMinFrameDurationNs(Parameters::Size size) {
+    if (mDeviceVersion >= CAMERA_DEVICE_API_VERSION_3_2) {
+        const int STREAM_DURATION_SIZE = 4;
+        const int STREAM_FORMAT_OFFSET = 0;
+        const int STREAM_WIDTH_OFFSET = 1;
+        const int STREAM_HEIGHT_OFFSET = 2;
+        const int STREAM_DURATION_OFFSET = 3;
+        camera_metadata_ro_entry_t availableStreamMinDurations =
+                    staticInfo(ANDROID_SCALER_AVAILABLE_MIN_FRAME_DURATIONS);
+        for (size_t i = 0; i < availableStreamMinDurations.count; i+= STREAM_DURATION_SIZE) {
+            int64_t format = availableStreamMinDurations.data.i64[i + STREAM_FORMAT_OFFSET];
+            int64_t width = availableStreamMinDurations.data.i64[i + STREAM_WIDTH_OFFSET];
+            int64_t height = availableStreamMinDurations.data.i64[i + STREAM_HEIGHT_OFFSET];
+            int64_t duration = availableStreamMinDurations.data.i64[i + STREAM_DURATION_OFFSET];
+            if (format == HAL_PIXEL_FORMAT_BLOB && width == size.width && height == size.height) {
+                return duration;
+            }
+        }
+    } else {
+        Vector<Size> availableJpegSizes = getAvailableJpegSizes();
+        size_t streamIdx = availableJpegSizes.size();
+        for (size_t i = 0; i < availableJpegSizes.size(); i++) {
+            if (availableJpegSizes[i].width == size.width &&
+                    availableJpegSizes[i].height == size.height) {
+                streamIdx = i;
+                break;
+            }
+        }
+        if (streamIdx != availableJpegSizes.size()) {
+            camera_metadata_ro_entry_t jpegMinDurations =
+                    staticInfo(ANDROID_SCALER_AVAILABLE_JPEG_MIN_DURATIONS);
+            if (streamIdx < jpegMinDurations.count) {
+                return jpegMinDurations.data.i64[streamIdx];
+            }
+        }
+    }
+    ALOGE("%s: cannot find min frame duration for jpeg size %dx%d",
+            __FUNCTION__, size.width, size.height);
+    return -1;
+}
+
 SortedVector<int32_t> Parameters::getAvailableOutputFormats() {
     SortedVector<int32_t> outputFormats; // Non-duplicated output formats
     if (mDeviceVersion >= CAMERA_DEVICE_API_VERSION_3_2) {
         Vector<StreamConfiguration> scs = getStreamConfigurations();
-        for (size_t i=0; i < scs.size(); i++) {
+        for (size_t i = 0; i < scs.size(); i++) {
             const StreamConfiguration &sc = scs[i];
             if (sc.isInput == ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT) {
                 outputFormats.add(sc.format);
@@ -2815,7 +2882,7 @@ SortedVector<int32_t> Parameters::getAvailableOutputFormats() {
         }
     } else {
         camera_metadata_ro_entry_t availableFormats = staticInfo(ANDROID_SCALER_AVAILABLE_FORMATS);
-        for (size_t i=0; i < availableFormats.count; i++) {
+        for (size_t i = 0; i < availableFormats.count; i++) {
             outputFormats.add(availableFormats.data.i32[i]);
         }
     }
@@ -2826,7 +2893,7 @@ Vector<Parameters::Size> Parameters::getAvailableJpegSizes() {
     Vector<Parameters::Size> jpegSizes;
     if (mDeviceVersion >= CAMERA_DEVICE_API_VERSION_3_2) {
         Vector<StreamConfiguration> scs = getStreamConfigurations();
-        for (size_t i=0; i < scs.size(); i++) {
+        for (size_t i = 0; i < scs.size(); i++) {
             const StreamConfiguration &sc = scs[i];
             if (sc.isInput == ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_OUTPUT &&
                     sc.format == HAL_PIXEL_FORMAT_BLOB) {
@@ -2840,7 +2907,7 @@ Vector<Parameters::Size> Parameters::getAvailableJpegSizes() {
         const int HEIGHT_OFFSET = 1;
         camera_metadata_ro_entry_t availableJpegSizes =
             staticInfo(ANDROID_SCALER_AVAILABLE_JPEG_SIZES);
-        for (size_t i=0; i < availableJpegSizes.count; i+= JPEG_SIZE_ENTRY_COUNT) {
+        for (size_t i = 0; i < availableJpegSizes.count; i+= JPEG_SIZE_ENTRY_COUNT) {
             int width = availableJpegSizes.data.i32[i + WIDTH_OFFSET];
             int height = availableJpegSizes.data.i32[i + HEIGHT_OFFSET];
             Size sz = {width, height};
diff --git a/services/camera/libcameraservice/api1/client2/Parameters.h b/services/camera/libcameraservice/api1/client2/Parameters.h
index 46d48bc..972d007 100644
--- a/services/camera/libcameraservice/api1/client2/Parameters.h
+++ b/services/camera/libcameraservice/api1/client2/Parameters.h
@@ -166,6 +166,9 @@ struct Parameters {
     bool previewCallbackSurface;
 
     bool zslMode;
+    // Whether the jpeg stream is slower than 30FPS and can slow down preview.
+    // When slowJpegMode is true, zslMode must be false to avoid slowing down preview.
+    bool slowJpegMode;
 
     // Overall camera state
     enum State {
@@ -190,6 +193,8 @@ struct Parameters {
     static const int MAX_INITIAL_PREVIEW_HEIGHT = 1080;
     // Aspect ratio tolerance
     static const CONSTEXPR float ASPECT_RATIO_TOLERANCE = 0.001;
+    // Threshold for slow jpeg mode
+    static const int64_t kSlowJpegModeThreshold = 33400000LL; // 33.4 ms
 
     // Full static camera info, object owned by someone else, such as
     // Camera2Device.
@@ -377,15 +382,23 @@ private:
         int32_t height;
         int32_t isInput;
     };
+
     // Helper function extract available stream configuration
     // Only valid since device HAL version 3.2
     // returns an empty Vector if device HAL version does support it
     Vector<StreamConfiguration> getStreamConfigurations();
 
+    // Helper function to get minimum frame duration for a jpeg size
+    // return -1 if input jpeg size cannot be found in supported size list
+    int64_t getJpegStreamMinFrameDurationNs(Parameters::Size size);
+
     // Helper function to get non-duplicated available output formats
     SortedVector<int32_t> getAvailableOutputFormats();
     // Helper function to get available output jpeg sizes
     Vector<Size> getAvailableJpegSizes();
+    // Helper function to get maximum size in input Size vector.
+    // The maximum size is defined by comparing width first, when width ties comparing height.
+    Size getMaxSize(const Vector<Size>& sizes);
 
     int mDeviceVersion;
 };