From c8474b68e435cd840ba66791fb09e3da59f1d93c Mon Sep 17 00:00:00 2001 From: Eino-Ville Talvala Date: Fri, 24 Aug 2012 16:30:44 -0700 Subject: Camera2: Move frame processing to its own thread. To avoid blocking the HAL on possibly long-running binder calls, move frame receiption and processing to its own thread. Bug: 6243944 Change-Id: I6385be4410b2f68e2331755027afc10c342187cf --- services/camera/libcameraservice/Camera2Client.cpp | 116 +++++++++++++++------ services/camera/libcameraservice/Camera2Client.h | 29 ++++-- services/camera/libcameraservice/Camera2Device.cpp | 34 ++---- services/camera/libcameraservice/Camera2Device.h | 17 +-- 4 files changed, 119 insertions(+), 77 deletions(-) (limited to 'services') diff --git a/services/camera/libcameraservice/Camera2Client.cpp b/services/camera/libcameraservice/Camera2Client.cpp index c06f6ba..21a0547 100644 --- a/services/camera/libcameraservice/Camera2Client.cpp +++ b/services/camera/libcameraservice/Camera2Client.cpp @@ -61,6 +61,7 @@ Camera2Client::Camera2Client(const sp& cameraService, mRecordingHeapCount(kDefaultRecordingHeapCount) { ATRACE_CALL(); + ALOGV("%s: Created client for camera %d", __FUNCTION__, cameraId); mDevice = new Camera2Device(cameraId); @@ -80,9 +81,14 @@ status_t Camera2Client::checkPid(const char* checkLocation) const { status_t Camera2Client::initialize(camera_module_t *module) { ATRACE_CALL(); - ALOGV("%s: E", __FUNCTION__); + ALOGV("%s: Initializing client for camera %d", __FUNCTION__, mCameraId); status_t res; + mFrameProcessor = new FrameProcessor(this); + String8 frameThreadName = String8::format("Camera2Client[%d]::FrameProcessor", + mCameraId); + mFrameProcessor->run(frameThreadName.string()); + res = mDevice->initialize(module); if (res != OK) { ALOGE("%s: Camera %d: unable to initialize device: %s (%d)", @@ -91,7 +97,6 @@ status_t Camera2Client::initialize(camera_module_t *module) } res = mDevice->setNotifyCallback(this); - res = mDevice->setFrameListener(this); res = buildDeviceInfo(); res = buildDefaultParameters(); @@ -113,13 +118,16 @@ status_t Camera2Client::initialize(camera_module_t *module) Camera2Client::~Camera2Client() { ATRACE_CALL(); - ALOGV("%s: Camera %d: Shutting down", __FUNCTION__, mCameraId); + ALOGV("%s: Camera %d: Shutting down client.", __FUNCTION__, mCameraId); mDestructionStarted = true; // Rewrite mClientPid to allow shutdown by CameraService mClientPid = getCallingPid(); disconnect(); + + mFrameProcessor->requestExit(); + ALOGV("%s: Camera %d: Shutdown complete", __FUNCTION__, mCameraId); } status_t Camera2Client::dump(int fd, const Vector& args) { @@ -315,6 +323,8 @@ status_t Camera2Client::dump(int fd, const Vector& args) { write(fd, result.string(), result.size()); } + mFrameProcessor->dump(fd, args); + result = " Device dump:\n"; write(fd, result.string(), result.size()); @@ -1986,36 +1996,76 @@ void Camera2Client::notifyAutoWhitebalance(uint8_t newState, int triggerId) { __FUNCTION__, newState, triggerId); } -void Camera2Client::onNewFrameAvailable() { +Camera2Client::FrameProcessor::FrameProcessor(wp client): + Thread(false), mClient(client) { +} + +Camera2Client::FrameProcessor::~FrameProcessor() { + ALOGV("%s: Exit", __FUNCTION__); +} + +void Camera2Client::FrameProcessor::dump(int fd, const Vector& args) { + String8 result(" Latest received frame:\n"); + write(fd, result.string(), result.size()); + mLastFrame.dump(fd, 2, 6); +} + +bool Camera2Client::FrameProcessor::threadLoop() { + status_t res; + + sp device; + { + sp client = mClient.promote(); + if (client == 0) return false; + device = client->mDevice; + } + + res = device->waitForNextFrame(kWaitDuration); + if (res == OK) { + sp client = mClient.promote(); + if (client == 0) return false; + processNewFrames(client); + } else if (res != TIMED_OUT) { + ALOGE("Camera2Client::FrameProcessor: Error waiting for new " + "frames: %s (%d)", strerror(-res), res); + } + + return true; +} + +void Camera2Client::FrameProcessor::processNewFrames(sp &client) { status_t res; CameraMetadata frame; - while ( (res = mDevice->getNextFrame(&frame)) == OK) { + while ( (res = client->mDevice->getNextFrame(&frame)) == OK) { camera_metadata_entry_t entry; entry = frame.find(ANDROID_REQUEST_FRAME_COUNT); if (entry.count == 0) { ALOGE("%s: Camera %d: Error reading frame number: %s (%d)", - __FUNCTION__, mCameraId, strerror(-res), res); + __FUNCTION__, client->mCameraId, strerror(-res), res); break; } - res = processFrameFaceDetect(frame); + res = processFaceDetect(frame, client); if (res != OK) break; + + mLastFrame.acquire(frame); } if (res != NOT_ENOUGH_DATA) { ALOGE("%s: Camera %d: Error getting next frame: %s (%d)", - __FUNCTION__, mCameraId, strerror(-res), res); + __FUNCTION__, client->mCameraId, strerror(-res), res); return; } return; } -status_t Camera2Client::processFrameFaceDetect(const CameraMetadata &frame) { +status_t Camera2Client::FrameProcessor::processFaceDetect( + const CameraMetadata &frame, sp &client) { status_t res; camera_metadata_ro_entry_t entry; bool enableFaceDetect; { - LockedParameters::Key k(mParameters); + LockedParameters::Key k(client->mParameters); enableFaceDetect = k.mParameters.enableFaceDetect; } entry = frame.find(ANDROID_STATS_FACE_DETECT_MODE); @@ -2031,16 +2081,16 @@ status_t Camera2Client::processFrameFaceDetect(const CameraMetadata &frame) { entry = frame.find(ANDROID_STATS_FACE_RECTANGLES); if (entry.count == 0) { ALOGE("%s: Camera %d: Unable to read face rectangles", - __FUNCTION__, mCameraId); + __FUNCTION__, client->mCameraId); return res; } camera_frame_metadata metadata; metadata.number_of_faces = entry.count / 4; if (metadata.number_of_faces > - mDeviceInfo->maxFaces) { + client->mDeviceInfo->maxFaces) { ALOGE("%s: Camera %d: More faces than expected! (Got %d, max %d)", - __FUNCTION__, mCameraId, - metadata.number_of_faces, mDeviceInfo->maxFaces); + __FUNCTION__, client->mCameraId, + metadata.number_of_faces, client->mDeviceInfo->maxFaces); return res; } const int32_t *faceRects = entry.data.i32; @@ -2048,7 +2098,7 @@ status_t Camera2Client::processFrameFaceDetect(const CameraMetadata &frame) { entry = frame.find(ANDROID_STATS_FACE_SCORES); if (entry.count == 0) { ALOGE("%s: Camera %d: Unable to read face scores", - __FUNCTION__, mCameraId); + __FUNCTION__, client->mCameraId); return res; } const uint8_t *faceScores = entry.data.u8; @@ -2060,7 +2110,7 @@ status_t Camera2Client::processFrameFaceDetect(const CameraMetadata &frame) { entry = frame.find(ANDROID_STATS_FACE_LANDMARKS); if (entry.count == 0) { ALOGE("%s: Camera %d: Unable to read face landmarks", - __FUNCTION__, mCameraId); + __FUNCTION__, client->mCameraId); return res; } faceLandmarks = entry.data.i32; @@ -2069,7 +2119,7 @@ status_t Camera2Client::processFrameFaceDetect(const CameraMetadata &frame) { if (entry.count == 0) { ALOGE("%s: Camera %d: Unable to read face IDs", - __FUNCTION__, mCameraId); + __FUNCTION__, client->mCameraId); return res; } faceIds = entry.data.i32; @@ -2081,20 +2131,26 @@ status_t Camera2Client::processFrameFaceDetect(const CameraMetadata &frame) { for (int i = 0; i < metadata.number_of_faces; i++) { camera_face_t face; - face.rect[0] = arrayXToNormalized(faceRects[i*4 + 0]); - face.rect[1] = arrayYToNormalized(faceRects[i*4 + 1]); - face.rect[2] = arrayXToNormalized(faceRects[i*4 + 2]); - face.rect[3] = arrayYToNormalized(faceRects[i*4 + 3]); + face.rect[0] = client->arrayXToNormalized(faceRects[i*4 + 0]); + face.rect[1] = client->arrayYToNormalized(faceRects[i*4 + 1]); + face.rect[2] = client->arrayXToNormalized(faceRects[i*4 + 2]); + face.rect[3] = client->arrayYToNormalized(faceRects[i*4 + 3]); face.score = faceScores[i]; if (faceDetectMode == ANDROID_STATS_FACE_DETECTION_FULL) { face.id = faceIds[i]; - face.left_eye[0] = arrayXToNormalized(faceLandmarks[i*6 + 0]); - face.left_eye[1] = arrayYToNormalized(faceLandmarks[i*6 + 1]); - face.right_eye[0] = arrayXToNormalized(faceLandmarks[i*6 + 2]); - face.right_eye[1] = arrayYToNormalized(faceLandmarks[i*6 + 3]); - face.mouth[0] = arrayXToNormalized(faceLandmarks[i*6 + 4]); - face.mouth[1] = arrayYToNormalized(faceLandmarks[i*6 + 5]); + face.left_eye[0] = + client->arrayXToNormalized(faceLandmarks[i*6 + 0]); + face.left_eye[1] = + client->arrayYToNormalized(faceLandmarks[i*6 + 1]); + face.right_eye[0] = + client->arrayXToNormalized(faceLandmarks[i*6 + 2]); + face.right_eye[1] = + client->arrayYToNormalized(faceLandmarks[i*6 + 3]); + face.mouth[0] = + client->arrayXToNormalized(faceLandmarks[i*6 + 4]); + face.mouth[1] = + client->arrayYToNormalized(faceLandmarks[i*6 + 5]); } else { face.id = 0; face.left_eye[0] = face.left_eye[1] = -2000; @@ -2106,9 +2162,9 @@ status_t Camera2Client::processFrameFaceDetect(const CameraMetadata &frame) { metadata.faces = faces.editArray(); { - Mutex::Autolock iccl(mICameraClientLock); - if (mCameraClient != NULL) { - mCameraClient->dataCallback(CAMERA_MSG_PREVIEW_METADATA, + Mutex::Autolock iccl(client->mICameraClientLock); + if (client->mCameraClient != NULL) { + client->mCameraClient->dataCallback(CAMERA_MSG_PREVIEW_METADATA, NULL, &metadata); } } diff --git a/services/camera/libcameraservice/Camera2Client.h b/services/camera/libcameraservice/Camera2Client.h index e03aaba..e2f5cc4 100644 --- a/services/camera/libcameraservice/Camera2Client.h +++ b/services/camera/libcameraservice/Camera2Client.h @@ -33,8 +33,7 @@ namespace android { */ class Camera2Client : public CameraService::Client, - public Camera2Device::NotificationListener, - public Camera2Device::FrameListener + public Camera2Device::NotificationListener { public: // ICamera interface (see ICamera for details) @@ -83,8 +82,6 @@ public: virtual void notifyAutoExposure(uint8_t newState, int triggerId); virtual void notifyAutoWhitebalance(uint8_t newState, int triggerId); - virtual void onNewFrameAvailable(); - private: enum State { DISCONNECTED, @@ -299,9 +296,29 @@ private: // Used with stream IDs static const int NO_STREAM = -1; - /* Output frame metadata processing methods */ + /* Output frame metadata processing thread. This thread waits for new + * frames from the device, and analyzes them as necessary. + */ + class FrameProcessor: public Thread { + public: + FrameProcessor(wp client); + ~FrameProcessor(); + + void dump(int fd, const Vector& args); + private: + static const nsecs_t kWaitDuration = 10000000; // 10 ms + wp mClient; + + virtual bool threadLoop(); + + void processNewFrames(sp &client); + status_t processFaceDetect(const CameraMetadata &frame, + sp &client); + + CameraMetadata mLastFrame; + }; - status_t processFrameFaceDetect(const CameraMetadata &frame); + sp mFrameProcessor; /* Preview related members */ diff --git a/services/camera/libcameraservice/Camera2Device.cpp b/services/camera/libcameraservice/Camera2Device.cpp index 3390ff3..35c4e74 100644 --- a/services/camera/libcameraservice/Camera2Device.cpp +++ b/services/camera/libcameraservice/Camera2Device.cpp @@ -33,12 +33,12 @@ Camera2Device::Camera2Device(int id): mId(id), mDevice(NULL) { - ALOGV("%s: E", __FUNCTION__); + ALOGV("%s: Created device for camera %d", __FUNCTION__, id); } Camera2Device::~Camera2Device() { - ALOGV("%s: E", __FUNCTION__); + ALOGV("%s: Shutting down device for camera %d", __FUNCTION__, mId); if (mDevice) { status_t res; res = mDevice->common.close(&mDevice->common); @@ -49,11 +49,12 @@ Camera2Device::~Camera2Device() } mDevice = NULL; } + ALOGV("%s: Shutdown complete", __FUNCTION__); } status_t Camera2Device::initialize(camera_module_t *module) { - ALOGV("%s: E", __FUNCTION__); + ALOGV("%s: Initializing device for camera %d", __FUNCTION__, mId); status_t res; char name[10]; @@ -347,8 +348,8 @@ void Camera2Device::notificationCallback(int32_t msg_type, } } -status_t Camera2Device::setFrameListener(FrameListener *listener) { - return mFrameQueue.setListener(listener); +status_t Camera2Device::waitForNextFrame(nsecs_t timeout) { + return mFrameQueue.waitForBuffer(timeout); } status_t Camera2Device::getNextFrame(CameraMetadata *frame) { @@ -407,13 +408,6 @@ Camera2Device::NotificationListener::~NotificationListener() { } /** - * Camera2Device::FrameListener - */ - -Camera2Device::FrameListener::~FrameListener() { -} - -/** * Camera2Device::MetadataQueue */ @@ -422,8 +416,7 @@ Camera2Device::MetadataQueue::MetadataQueue(): mFrameCount(0), mCount(0), mStreamSlotCount(0), - mSignalConsumer(true), - mListener(NULL) + mSignalConsumer(true) { camera2_request_queue_src_ops::dequeue_request = consumer_dequeue; camera2_request_queue_src_ops::request_count = consumer_buffer_count; @@ -541,12 +534,6 @@ status_t Camera2Device::MetadataQueue::waitForBuffer(nsecs_t timeout) return OK; } -status_t Camera2Device::MetadataQueue::setListener(FrameListener *listener) { - Mutex::Autolock l(mMutex); - mListener = listener; - return OK; -} - status_t Camera2Device::MetadataQueue::setStreamSlot(camera_metadata_t *buf) { ALOGV("%s: E", __FUNCTION__); @@ -659,13 +646,6 @@ status_t Camera2Device::MetadataQueue::signalConsumerLocked() { res = mDevice->ops->notify_request_queue_not_empty(mDevice); mMutex.lock(); } - if (mListener != NULL) { - FrameListener *listener = mListener; - mMutex.unlock(); - ALOGVV("%s: Signaling listener", __FUNCTION__); - listener->onNewFrameAvailable(); - mMutex.lock(); - } return res; } diff --git a/services/camera/libcameraservice/Camera2Device.h b/services/camera/libcameraservice/Camera2Device.h index 06047b8..223d77a 100644 --- a/services/camera/libcameraservice/Camera2Device.h +++ b/services/camera/libcameraservice/Camera2Device.h @@ -129,19 +129,10 @@ class Camera2Device : public virtual RefBase { status_t setNotifyCallback(NotificationListener *listener); /** - * Abstract class for HAL frame available notifications + * Wait for a new frame to be produced, with timeout in nanoseconds. + * Returns TIMED_OUT when no frame produced within the specified duration */ - class FrameListener { - public: - virtual void onNewFrameAvailable() = 0; - protected: - virtual ~FrameListener(); - }; - - /** - * Set a frame listener to be notified about new frames. - */ - status_t setFrameListener(FrameListener *listener); + status_t waitForNextFrame(nsecs_t timeout); /** * Get next metadata frame from the frame queue. Returns NULL if the queue @@ -206,7 +197,6 @@ class Camera2Device : public virtual RefBase { status_t dequeue(camera_metadata_t **buf, bool incrementCount = true); int getBufferCount(); status_t waitForBuffer(nsecs_t timeout); - status_t setListener(FrameListener *listener); // Set repeating buffer(s); if the queue is empty on a dequeue call, the // queue copies the contents of the stream slot into the queue, and then @@ -235,7 +225,6 @@ class Camera2Device : public virtual RefBase { List mStreamSlot; bool mSignalConsumer; - FrameListener *mListener; static MetadataQueue* getInstance( const camera2_frame_queue_dst_ops_t *q); -- cgit v1.1