diff options
355 files changed, 22264 insertions, 6227 deletions
diff --git a/camera/Android.mk b/camera/Android.mk index fa518ff..e633450 100644 --- a/camera/Android.mk +++ b/camera/Android.mk @@ -16,6 +16,9 @@ LOCAL_SRC_FILES:= \ ICameraRecordingProxyListener.cpp \ IProCameraUser.cpp \ IProCameraCallbacks.cpp \ + camera2/ICameraDeviceUser.cpp \ + camera2/ICameraDeviceCallbacks.cpp \ + camera2/CaptureRequest.cpp \ ProCamera.cpp \ CameraBase.cpp \ diff --git a/camera/Camera.cpp b/camera/Camera.cpp index 1b136de..22199fa 100644 --- a/camera/Camera.cpp +++ b/camera/Camera.cpp @@ -39,6 +39,9 @@ Camera::Camera(int cameraId) { } +CameraTraits<Camera>::TCamConnectService CameraTraits<Camera>::fnConnectService = + &ICameraService::connect; + // construct a camera client from an existing camera remote sp<Camera> Camera::create(const sp<ICamera>& camera) { @@ -97,13 +100,13 @@ status_t Camera::unlock() } // pass the buffered IGraphicBufferProducer to the camera service -status_t Camera::setPreviewTexture(const sp<IGraphicBufferProducer>& bufferProducer) +status_t Camera::setPreviewTarget(const sp<IGraphicBufferProducer>& bufferProducer) { - ALOGV("setPreviewTexture(%p)", bufferProducer.get()); + ALOGV("setPreviewTarget(%p)", bufferProducer.get()); sp <ICamera> c = mCamera; if (c == 0) return NO_INIT; ALOGD_IF(bufferProducer == 0, "app passed NULL surface"); - return c->setPreviewTexture(bufferProducer); + return c->setPreviewTarget(bufferProducer); } // start preview mode @@ -124,7 +127,7 @@ status_t Camera::storeMetaDataInBuffers(bool enabled) return c->storeMetaDataInBuffers(enabled); } -// start recording mode, must call setPreviewDisplay first +// start recording mode, must call setPreviewTarget first status_t Camera::startRecording() { ALOGV("startRecording"); @@ -255,6 +258,14 @@ void Camera::setPreviewCallbackFlags(int flag) mCamera->setPreviewCallbackFlag(flag); } +status_t Camera::setPreviewCallbackTarget( + const sp<IGraphicBufferProducer>& callbackProducer) +{ + sp <ICamera> c = mCamera; + if (c == 0) return NO_INIT; + return c->setPreviewCallbackTarget(callbackProducer); +} + // callback from camera service void Camera::notifyCallback(int32_t msgType, int32_t ext1, int32_t ext2) { diff --git a/camera/CameraBase.cpp b/camera/CameraBase.cpp index c25c5fd..55376b0 100644 --- a/camera/CameraBase.cpp +++ b/camera/CameraBase.cpp @@ -92,20 +92,25 @@ const sp<ICameraService>& CameraBase<TCam, TCamTraits>::getCameraService() template <typename TCam, typename TCamTraits> sp<TCam> CameraBase<TCam, TCamTraits>::connect(int cameraId, - const String16& clientPackageName, + const String16& clientPackageName, int clientUid) { ALOGV("%s: connect", __FUNCTION__); sp<TCam> c = new TCam(cameraId); sp<TCamCallbacks> cl = c; + status_t status = NO_ERROR; const sp<ICameraService>& cs = getCameraService(); + if (cs != 0) { - c->mCamera = cs->connect(cl, cameraId, clientPackageName, clientUid); + TCamConnectService fnConnectService = TCamTraits::fnConnectService; + status = (cs.get()->*fnConnectService)(cl, cameraId, clientPackageName, clientUid, + /*out*/ c->mCamera); } - if (c->mCamera != 0) { + if (status == OK && c->mCamera != 0) { c->mCamera->asBinder()->linkToDeath(c); c->mStatus = NO_ERROR; } else { + ALOGW("An error occurred while connecting to camera: %d", cameraId); c.clear(); } return c; diff --git a/camera/CameraMetadata.cpp b/camera/CameraMetadata.cpp index a8f9eff..f447c5b 100644 --- a/camera/CameraMetadata.cpp +++ b/camera/CameraMetadata.cpp @@ -21,9 +21,13 @@ #include <utils/Errors.h> #include <camera/CameraMetadata.h> +#include <binder/Parcel.h> namespace android { +typedef Parcel::WritableBlob WritableBlob; +typedef Parcel::ReadableBlob ReadableBlob; + CameraMetadata::CameraMetadata() : mBuffer(NULL), mLocked(false) { } @@ -408,4 +412,175 @@ status_t CameraMetadata::resizeIfNeeded(size_t extraEntries, size_t extraData) { return OK; } +status_t CameraMetadata::readFromParcel(const Parcel& data, + camera_metadata_t** out) { + + status_t err = OK; + + camera_metadata_t* metadata = NULL; + + if (out) { + *out = NULL; + } + + // arg0 = metadataSize (int32) + int32_t metadataSizeTmp = -1; + if ((err = data.readInt32(&metadataSizeTmp)) != OK) { + ALOGE("%s: Failed to read metadata size (error %d %s)", + __FUNCTION__, err, strerror(-err)); + return err; + } + const size_t metadataSize = static_cast<size_t>(metadataSizeTmp); + + if (metadataSize == 0) { + ALOGV("%s: Read 0-sized metadata", __FUNCTION__); + return OK; + } + + // NOTE: this doesn't make sense to me. shouldnt the blob + // know how big it is? why do we have to specify the size + // to Parcel::readBlob ? + + ReadableBlob blob; + // arg1 = metadata (blob) + do { + if ((err = data.readBlob(metadataSize, &blob)) != OK) { + ALOGE("%s: Failed to read metadata blob (sized %d). Possible " + " serialization bug. Error %d %s", + __FUNCTION__, metadataSize, err, strerror(-err)); + break; + } + const camera_metadata_t* tmp = + reinterpret_cast<const camera_metadata_t*>(blob.data()); + + metadata = allocate_copy_camera_metadata_checked(tmp, metadataSize); + if (metadata == NULL) { + // We consider that allocation only fails if the validation + // also failed, therefore the readFromParcel was a failure. + err = BAD_VALUE; + } + } while(0); + blob.release(); + + if (out) { + ALOGV("%s: Set out metadata to %p", __FUNCTION__, metadata); + *out = metadata; + } else if (metadata != NULL) { + ALOGV("%s: Freed camera metadata at %p", __FUNCTION__, metadata); + free_camera_metadata(metadata); + } + + return err; +} + +status_t CameraMetadata::writeToParcel(Parcel& data, + const camera_metadata_t* metadata) { + status_t res = OK; + + // arg0 = metadataSize (int32) + + if (metadata == NULL) { + return data.writeInt32(0); + } + + const size_t metadataSize = get_camera_metadata_compact_size(metadata); + res = data.writeInt32(static_cast<int32_t>(metadataSize)); + if (res != OK) { + return res; + } + + // arg1 = metadata (blob) + WritableBlob blob; + do { + res = data.writeBlob(metadataSize, &blob); + if (res != OK) { + break; + } + copy_camera_metadata(blob.data(), metadataSize, metadata); + + IF_ALOGV() { + if (validate_camera_metadata_structure( + (const camera_metadata_t*)blob.data(), + &metadataSize) != OK) { + ALOGV("%s: Failed to validate metadata %p after writing blob", + __FUNCTION__, blob.data()); + } else { + ALOGV("%s: Metadata written to blob. Validation success", + __FUNCTION__); + } + } + + // Not too big of a problem since receiving side does hard validation + // Don't check the size since the compact size could be larger + if (validate_camera_metadata_structure(metadata, /*size*/NULL) != OK) { + ALOGW("%s: Failed to validate metadata %p before writing blob", + __FUNCTION__, metadata); + } + + } while(false); + blob.release(); + + return res; +} + +status_t CameraMetadata::readFromParcel(Parcel *parcel) { + + ALOGV("%s: parcel = %p", __FUNCTION__, parcel); + + status_t res = OK; + + if (parcel == NULL) { + ALOGE("%s: parcel is null", __FUNCTION__); + return BAD_VALUE; + } + + if (mLocked) { + ALOGE("%s: CameraMetadata is locked", __FUNCTION__); + return INVALID_OPERATION; + } + + camera_metadata *buffer = NULL; + // TODO: reading should return a status code, in case validation fails + res = CameraMetadata::readFromParcel(*parcel, &buffer); + + if (res != NO_ERROR) { + ALOGE("%s: Failed to read from parcel. Metadata is unchanged.", + __FUNCTION__); + return res; + } + + clear(); + mBuffer = buffer; + + return OK; +} + +status_t CameraMetadata::writeToParcel(Parcel *parcel) const { + + ALOGV("%s: parcel = %p", __FUNCTION__, parcel); + + if (parcel == NULL) { + ALOGE("%s: parcel is null", __FUNCTION__); + return BAD_VALUE; + } + + return CameraMetadata::writeToParcel(*parcel, mBuffer); +} + +void CameraMetadata::swap(CameraMetadata& other) { + if (mLocked) { + ALOGE("%s: CameraMetadata is locked", __FUNCTION__); + return; + } else if (other.mLocked) { + ALOGE("%s: Other CameraMetadata is locked", __FUNCTION__); + return; + } + + camera_metadata* thisBuf = mBuffer; + camera_metadata* otherBuf = other.mBuffer; + + other.mBuffer = thisBuf; + mBuffer = otherBuf; +} + }; // namespace android diff --git a/camera/ICamera.cpp b/camera/ICamera.cpp index 8900867..8c6e1f7 100644 --- a/camera/ICamera.cpp +++ b/camera/ICamera.cpp @@ -29,8 +29,9 @@ namespace android { enum { DISCONNECT = IBinder::FIRST_CALL_TRANSACTION, - SET_PREVIEW_TEXTURE, + SET_PREVIEW_TARGET, SET_PREVIEW_CALLBACK_FLAG, + SET_PREVIEW_CALLBACK_TARGET, START_PREVIEW, STOP_PREVIEW, AUTO_FOCUS, @@ -65,17 +66,18 @@ public: Parcel data, reply; data.writeInterfaceToken(ICamera::getInterfaceDescriptor()); remote()->transact(DISCONNECT, data, &reply); + reply.readExceptionCode(); } // pass the buffered IGraphicBufferProducer to the camera service - status_t setPreviewTexture(const sp<IGraphicBufferProducer>& bufferProducer) + status_t setPreviewTarget(const sp<IGraphicBufferProducer>& bufferProducer) { - ALOGV("setPreviewTexture"); + ALOGV("setPreviewTarget"); Parcel data, reply; data.writeInterfaceToken(ICamera::getInterfaceDescriptor()); sp<IBinder> b(bufferProducer->asBinder()); data.writeStrongBinder(b); - remote()->transact(SET_PREVIEW_TEXTURE, data, &reply); + remote()->transact(SET_PREVIEW_TARGET, data, &reply); return reply.readInt32(); } @@ -90,7 +92,19 @@ public: remote()->transact(SET_PREVIEW_CALLBACK_FLAG, data, &reply); } - // start preview mode, must call setPreviewDisplay first + status_t setPreviewCallbackTarget( + const sp<IGraphicBufferProducer>& callbackProducer) + { + ALOGV("setPreviewCallbackTarget"); + Parcel data, reply; + data.writeInterfaceToken(ICamera::getInterfaceDescriptor()); + sp<IBinder> b(callbackProducer->asBinder()); + data.writeStrongBinder(b); + remote()->transact(SET_PREVIEW_CALLBACK_TARGET, data, &reply); + return reply.readInt32(); + } + + // start preview mode, must call setPreviewTarget first status_t startPreview() { ALOGV("startPreview"); @@ -100,7 +114,7 @@ public: return reply.readInt32(); } - // start recording mode, must call setPreviewDisplay first + // start recording mode, must call setPreviewTarget first status_t startRecording() { ALOGV("startRecording"); @@ -268,14 +282,15 @@ status_t BnCamera::onTransact( ALOGV("DISCONNECT"); CHECK_INTERFACE(ICamera, data, reply); disconnect(); + reply->writeNoException(); return NO_ERROR; } break; - case SET_PREVIEW_TEXTURE: { - ALOGV("SET_PREVIEW_TEXTURE"); + case SET_PREVIEW_TARGET: { + ALOGV("SET_PREVIEW_TARGET"); CHECK_INTERFACE(ICamera, data, reply); sp<IGraphicBufferProducer> st = interface_cast<IGraphicBufferProducer>(data.readStrongBinder()); - reply->writeInt32(setPreviewTexture(st)); + reply->writeInt32(setPreviewTarget(st)); return NO_ERROR; } break; case SET_PREVIEW_CALLBACK_FLAG: { @@ -285,6 +300,14 @@ status_t BnCamera::onTransact( setPreviewCallbackFlag(callback_flag); return NO_ERROR; } break; + case SET_PREVIEW_CALLBACK_TARGET: { + ALOGV("SET_PREVIEW_CALLBACK_TARGET"); + CHECK_INTERFACE(ICamera, data, reply); + sp<IGraphicBufferProducer> cp = + interface_cast<IGraphicBufferProducer>(data.readStrongBinder()); + reply->writeInt32(setPreviewCallbackTarget(cp)); + return NO_ERROR; + } case START_PREVIEW: { ALOGV("START_PREVIEW"); CHECK_INTERFACE(ICamera, data, reply); diff --git a/camera/ICameraService.cpp b/camera/ICameraService.cpp index 134f7f0..5fc89fb 100644 --- a/camera/ICameraService.cpp +++ b/camera/ICameraService.cpp @@ -15,6 +15,9 @@ ** limitations under the License. */ +#define LOG_TAG "BpCameraService" +#include <utils/Log.h> + #include <stdint.h> #include <sys/types.h> @@ -28,9 +31,59 @@ #include <camera/IProCameraCallbacks.h> #include <camera/ICamera.h> #include <camera/ICameraClient.h> +#include <camera/camera2/ICameraDeviceUser.h> +#include <camera/camera2/ICameraDeviceCallbacks.h> +#include <camera/CameraMetadata.h> namespace android { +namespace { + +enum { + EX_SECURITY = -1, + EX_BAD_PARCELABLE = -2, + EX_ILLEGAL_ARGUMENT = -3, + EX_NULL_POINTER = -4, + EX_ILLEGAL_STATE = -5, + EX_HAS_REPLY_HEADER = -128, // special; see below +}; + +static bool readExceptionCode(Parcel& reply) { + int32_t exceptionCode = reply.readExceptionCode(); + + if (exceptionCode != 0) { + const char* errorMsg; + switch(exceptionCode) { + case EX_SECURITY: + errorMsg = "Security"; + break; + case EX_BAD_PARCELABLE: + errorMsg = "BadParcelable"; + break; + case EX_NULL_POINTER: + errorMsg = "NullPointer"; + break; + case EX_ILLEGAL_STATE: + errorMsg = "IllegalState"; + break; + // Binder should be handling this code inside Parcel::readException + // but lets have a to-string here anyway just in case. + case EX_HAS_REPLY_HEADER: + errorMsg = "HasReplyHeader"; + break; + default: + errorMsg = "Unknown"; + } + + ALOGE("Binder transmission error %s (%d)", errorMsg, exceptionCode); + return true; + } + + return false; +} + +}; + class BpCameraService: public BpInterface<ICameraService> { public: @@ -45,6 +98,8 @@ public: Parcel data, reply; data.writeInterfaceToken(ICameraService::getInterfaceDescriptor()); remote()->transact(BnCameraService::GET_NUMBER_OF_CAMERAS, data, &reply); + + if (readExceptionCode(reply)) return 0; return reply.readInt32(); } @@ -55,14 +110,44 @@ public: data.writeInterfaceToken(ICameraService::getInterfaceDescriptor()); data.writeInt32(cameraId); remote()->transact(BnCameraService::GET_CAMERA_INFO, data, &reply); - cameraInfo->facing = reply.readInt32(); - cameraInfo->orientation = reply.readInt32(); - return reply.readInt32(); + + if (readExceptionCode(reply)) return -EPROTO; + status_t result = reply.readInt32(); + if (reply.readInt32() != 0) { + cameraInfo->facing = reply.readInt32(); + cameraInfo->orientation = reply.readInt32(); + } + return result; + } + + // get camera characteristics (static metadata) + virtual status_t getCameraCharacteristics(int cameraId, + CameraMetadata* cameraInfo) { + Parcel data, reply; + data.writeInterfaceToken(ICameraService::getInterfaceDescriptor()); + data.writeInt32(cameraId); + remote()->transact(BnCameraService::GET_CAMERA_CHARACTERISTICS, data, &reply); + + if (readExceptionCode(reply)) return -EPROTO; + status_t result = reply.readInt32(); + + CameraMetadata out; + if (reply.readInt32() != 0) { + out.readFromParcel(&reply); + } + + if (cameraInfo != NULL) { + cameraInfo->swap(out); + } + + return result; } - // connect to camera service - virtual sp<ICamera> connect(const sp<ICameraClient>& cameraClient, int cameraId, - const String16 &clientPackageName, int clientUid) + // connect to camera service (android.hardware.Camera) + virtual status_t connect(const sp<ICameraClient>& cameraClient, int cameraId, + const String16 &clientPackageName, int clientUid, + /*out*/ + sp<ICamera>& device) { Parcel data, reply; data.writeInterfaceToken(ICameraService::getInterfaceDescriptor()); @@ -71,12 +156,20 @@ public: data.writeString16(clientPackageName); data.writeInt32(clientUid); remote()->transact(BnCameraService::CONNECT, data, &reply); - return interface_cast<ICamera>(reply.readStrongBinder()); + + if (readExceptionCode(reply)) return -EPROTO; + status_t status = reply.readInt32(); + if (reply.readInt32() != 0) { + device = interface_cast<ICamera>(reply.readStrongBinder()); + } + return status; } // connect to camera service (pro client) - virtual sp<IProCameraUser> connect(const sp<IProCameraCallbacks>& cameraCb, int cameraId, - const String16 &clientPackageName, int clientUid) + virtual status_t connectPro(const sp<IProCameraCallbacks>& cameraCb, int cameraId, + const String16 &clientPackageName, int clientUid, + /*out*/ + sp<IProCameraUser>& device) { Parcel data, reply; data.writeInterfaceToken(ICameraService::getInterfaceDescriptor()); @@ -85,7 +178,38 @@ public: data.writeString16(clientPackageName); data.writeInt32(clientUid); remote()->transact(BnCameraService::CONNECT_PRO, data, &reply); - return interface_cast<IProCameraUser>(reply.readStrongBinder()); + + if (readExceptionCode(reply)) return -EPROTO; + status_t status = reply.readInt32(); + if (reply.readInt32() != 0) { + device = interface_cast<IProCameraUser>(reply.readStrongBinder()); + } + return status; + } + + // connect to camera service (android.hardware.camera2.CameraDevice) + virtual status_t connectDevice( + const sp<ICameraDeviceCallbacks>& cameraCb, + int cameraId, + const String16& clientPackageName, + int clientUid, + /*out*/ + sp<ICameraDeviceUser>& device) + { + Parcel data, reply; + data.writeInterfaceToken(ICameraService::getInterfaceDescriptor()); + data.writeStrongBinder(cameraCb->asBinder()); + data.writeInt32(cameraId); + data.writeString16(clientPackageName); + data.writeInt32(clientUid); + remote()->transact(BnCameraService::CONNECT_DEVICE, data, &reply); + + if (readExceptionCode(reply)) return -EPROTO; + status_t status = reply.readInt32(); + if (reply.readInt32() != 0) { + device = interface_cast<ICameraDeviceUser>(reply.readStrongBinder()); + } + return status; } virtual status_t addListener(const sp<ICameraServiceListener>& listener) @@ -94,6 +218,8 @@ public: data.writeInterfaceToken(ICameraService::getInterfaceDescriptor()); data.writeStrongBinder(listener->asBinder()); remote()->transact(BnCameraService::ADD_LISTENER, data, &reply); + + if (readExceptionCode(reply)) return -EPROTO; return reply.readInt32(); } @@ -103,6 +229,8 @@ public: data.writeInterfaceToken(ICameraService::getInterfaceDescriptor()); data.writeStrongBinder(listener->asBinder()); remote()->transact(BnCameraService::REMOVE_LISTENER, data, &reply); + + if (readExceptionCode(reply)) return -EPROTO; return reply.readInt32(); } }; @@ -117,17 +245,34 @@ status_t BnCameraService::onTransact( switch(code) { case GET_NUMBER_OF_CAMERAS: { CHECK_INTERFACE(ICameraService, data, reply); + reply->writeNoException(); reply->writeInt32(getNumberOfCameras()); return NO_ERROR; } break; case GET_CAMERA_INFO: { CHECK_INTERFACE(ICameraService, data, reply); - CameraInfo cameraInfo; + CameraInfo cameraInfo = CameraInfo(); memset(&cameraInfo, 0, sizeof(cameraInfo)); status_t result = getCameraInfo(data.readInt32(), &cameraInfo); + reply->writeNoException(); + reply->writeInt32(result); + + // Fake a parcelable object here + reply->writeInt32(1); // means the parcelable is included reply->writeInt32(cameraInfo.facing); reply->writeInt32(cameraInfo.orientation); + return NO_ERROR; + } break; + case GET_CAMERA_CHARACTERISTICS: { + CHECK_INTERFACE(ICameraService, data, reply); + CameraMetadata info; + status_t result = getCameraCharacteristics(data.readInt32(), &info); + reply->writeNoException(); reply->writeInt32(result); + + // out-variables are after exception and return value + reply->writeInt32(1); // means the parcelable is included + info.writeToParcel(reply); return NO_ERROR; } break; case CONNECT: { @@ -137,26 +282,64 @@ status_t BnCameraService::onTransact( int32_t cameraId = data.readInt32(); const String16 clientName = data.readString16(); int32_t clientUid = data.readInt32(); - sp<ICamera> camera = connect(cameraClient, cameraId, - clientName, clientUid); - reply->writeStrongBinder(camera->asBinder()); + sp<ICamera> camera; + status_t status = connect(cameraClient, cameraId, + clientName, clientUid, /*out*/ camera); + reply->writeNoException(); + reply->writeInt32(status); + if (camera != NULL) { + reply->writeInt32(1); + reply->writeStrongBinder(camera->asBinder()); + } else { + reply->writeInt32(0); + } return NO_ERROR; } break; case CONNECT_PRO: { CHECK_INTERFACE(ICameraService, data, reply); - sp<IProCameraCallbacks> cameraClient = interface_cast<IProCameraCallbacks>(data.readStrongBinder()); + sp<IProCameraCallbacks> cameraClient = + interface_cast<IProCameraCallbacks>(data.readStrongBinder()); + int32_t cameraId = data.readInt32(); + const String16 clientName = data.readString16(); + int32_t clientUid = data.readInt32(); + sp<IProCameraUser> camera; + status_t status = connectPro(cameraClient, cameraId, + clientName, clientUid, /*out*/ camera); + reply->writeNoException(); + reply->writeInt32(status); + if (camera != NULL) { + reply->writeInt32(1); + reply->writeStrongBinder(camera->asBinder()); + } else { + reply->writeInt32(0); + } + return NO_ERROR; + } break; + case CONNECT_DEVICE: { + CHECK_INTERFACE(ICameraService, data, reply); + sp<ICameraDeviceCallbacks> cameraClient = + interface_cast<ICameraDeviceCallbacks>(data.readStrongBinder()); int32_t cameraId = data.readInt32(); const String16 clientName = data.readString16(); int32_t clientUid = data.readInt32(); - sp<IProCameraUser> camera = connect(cameraClient, cameraId, - clientName, clientUid); - reply->writeStrongBinder(camera->asBinder()); + sp<ICameraDeviceUser> camera; + status_t status = connectDevice(cameraClient, cameraId, + clientName, clientUid, /*out*/ camera); + reply->writeNoException(); + reply->writeInt32(status); + if (camera != NULL) { + reply->writeInt32(1); + reply->writeStrongBinder(camera->asBinder()); + } else { + reply->writeInt32(0); + } return NO_ERROR; } break; case ADD_LISTENER: { CHECK_INTERFACE(ICameraService, data, reply); sp<ICameraServiceListener> listener = interface_cast<ICameraServiceListener>(data.readStrongBinder()); + reply->writeNoException(); reply->writeInt32(addListener(listener)); return NO_ERROR; } break; @@ -164,6 +347,7 @@ status_t BnCameraService::onTransact( CHECK_INTERFACE(ICameraService, data, reply); sp<ICameraServiceListener> listener = interface_cast<ICameraServiceListener>(data.readStrongBinder()); + reply->writeNoException(); reply->writeInt32(removeListener(listener)); return NO_ERROR; } break; diff --git a/camera/ICameraServiceListener.cpp b/camera/ICameraServiceListener.cpp index 640ee35..b2f1729 100644 --- a/camera/ICameraServiceListener.cpp +++ b/camera/ICameraServiceListener.cpp @@ -54,6 +54,8 @@ public: data, &reply, IBinder::FLAG_ONEWAY); + + reply.readExceptionCode(); } }; @@ -73,6 +75,7 @@ status_t BnCameraServiceListener::onTransact( int32_t cameraId = data.readInt32(); onStatusChanged(status, cameraId); + reply->writeNoException(); return NO_ERROR; } break; diff --git a/camera/IProCameraCallbacks.cpp b/camera/IProCameraCallbacks.cpp index b9cd14d..bd3d420 100644 --- a/camera/IProCameraCallbacks.cpp +++ b/camera/IProCameraCallbacks.cpp @@ -28,7 +28,7 @@ #include <camera/IProCameraCallbacks.h> -#include <system/camera_metadata.h> +#include "camera/CameraMetadata.h" namespace android { @@ -38,9 +38,6 @@ enum { RESULT_RECEIVED, }; -void readMetadata(const Parcel& data, camera_metadata_t** out); -void writeMetadata(Parcel& data, camera_metadata_t* metadata); - class BpProCameraCallbacks: public BpInterface<IProCameraCallbacks> { public: @@ -70,12 +67,12 @@ public: IBinder::FLAG_ONEWAY); } - void onResultReceived(int32_t frameId, camera_metadata* result) { + void onResultReceived(int32_t requestId, camera_metadata* result) { ALOGV("onResultReceived"); Parcel data, reply; data.writeInterfaceToken(IProCameraCallbacks::getInterfaceDescriptor()); - data.writeInt32(frameId); - writeMetadata(data, result); + data.writeInt32(requestId); + CameraMetadata::writeToParcel(data, result); remote()->transact(RESULT_RECEIVED, data, &reply, IBinder::FLAG_ONEWAY); } }; @@ -110,10 +107,10 @@ status_t BnProCameraCallbacks::onTransact( case RESULT_RECEIVED: { ALOGV("RESULT_RECEIVED"); CHECK_INTERFACE(IProCameraCallbacks, data, reply); - int32_t frameId = data.readInt32(); + int32_t requestId = data.readInt32(); camera_metadata_t *result = NULL; - readMetadata(data, &result); - onResultReceived(frameId, result); + CameraMetadata::readFromParcel(data, &result); + onResultReceived(requestId, result); return NO_ERROR; break; } diff --git a/camera/IProCameraUser.cpp b/camera/IProCameraUser.cpp index 4c4dec3..8f22124 100644 --- a/camera/IProCameraUser.cpp +++ b/camera/IProCameraUser.cpp @@ -15,7 +15,7 @@ ** limitations under the License. */ -//#define LOG_NDEBUG 0 +// #define LOG_NDEBUG 0 #define LOG_TAG "IProCameraUser" #include <utils/Log.h> #include <stdint.h> @@ -24,13 +24,10 @@ #include <camera/IProCameraUser.h> #include <gui/IGraphicBufferProducer.h> #include <gui/Surface.h> -#include <system/camera_metadata.h> +#include "camera/CameraMetadata.h" namespace android { -typedef Parcel::WritableBlob WritableBlob; -typedef Parcel::ReadableBlob ReadableBlob; - enum { DISCONNECT = IBinder::FIRST_CALL_TRANSACTION, CONNECT, @@ -46,107 +43,6 @@ enum { GET_CAMERA_INFO, }; -/** - * Caller becomes the owner of the new metadata - * 'const Parcel' doesnt prevent us from calling the read functions. - * which is interesting since it changes the internal state - * - * NULL can be returned when no metadata was sent, OR if there was an issue - * unpacking the serialized data (i.e. bad parcel or invalid structure). - */ -void readMetadata(const Parcel& data, camera_metadata_t** out) { - - status_t err = OK; - - camera_metadata_t* metadata = NULL; - - if (out) { - *out = NULL; - } - - // arg0 = metadataSize (int32) - int32_t metadataSizeTmp = -1; - if ((err = data.readInt32(&metadataSizeTmp)) != OK) { - ALOGE("%s: Failed to read metadata size (error %d %s)", - __FUNCTION__, err, strerror(-err)); - return; - } - const size_t metadataSize = static_cast<size_t>(metadataSizeTmp); - - if (metadataSize == 0) { - return; - } - - // NOTE: this doesn't make sense to me. shouldnt the blob - // know how big it is? why do we have to specify the size - // to Parcel::readBlob ? - - ReadableBlob blob; - // arg1 = metadata (blob) - do { - if ((err = data.readBlob(metadataSize, &blob)) != OK) { - ALOGE("%s: Failed to read metadata blob (sized %d). Possible " - " serialization bug. Error %d %s", - __FUNCTION__, metadataSize, err, strerror(-err)); - break; - } - const camera_metadata_t* tmp = - reinterpret_cast<const camera_metadata_t*>(blob.data()); - - metadata = allocate_copy_camera_metadata_checked(tmp, metadataSize); - } while(0); - blob.release(); - - if (out) { - *out = metadata; - } else if (metadata != NULL) { - free_camera_metadata(metadata); - } -} - -/** - * Caller retains ownership of metadata - * - Write 2 (int32 + blob) args in the current position - */ -void writeMetadata(Parcel& data, camera_metadata_t* metadata) { - // arg0 = metadataSize (int32) - - if (metadata == NULL) { - data.writeInt32(0); - return; - } - - const size_t metadataSize = get_camera_metadata_compact_size(metadata); - data.writeInt32(static_cast<int32_t>(metadataSize)); - - // arg1 = metadata (blob) - WritableBlob blob; - { - data.writeBlob(metadataSize, &blob); - copy_camera_metadata(blob.data(), metadataSize, metadata); - - IF_ALOGV() { - if (validate_camera_metadata_structure( - (const camera_metadata_t*)blob.data(), - &metadataSize) != OK) { - ALOGV("%s: Failed to validate metadata %p after writing blob", - __FUNCTION__, blob.data()); - } else { - ALOGV("%s: Metadata written to blob. Validation success", - __FUNCTION__); - } - } - - // Not too big of a problem since receiving side does hard validation - if (validate_camera_metadata_structure(metadata, &metadataSize) != OK) { - ALOGW("%s: Failed to validate metadata %p before writing blob", - __FUNCTION__, metadata); - } - - } - blob.release(); -} - class BpProCameraUser: public BpInterface<IProCameraUser> { public: @@ -162,6 +58,7 @@ public: Parcel data, reply; data.writeInterfaceToken(IProCameraUser::getInterfaceDescriptor()); remote()->transact(DISCONNECT, data, &reply); + reply.readExceptionCode(); } virtual status_t connect(const sp<IProCameraCallbacks>& cameraClient) @@ -213,7 +110,7 @@ public: data.writeInterfaceToken(IProCameraUser::getInterfaceDescriptor()); // arg0+arg1 - writeMetadata(data, metadata); + CameraMetadata::writeToParcel(data, metadata); // arg2 = streaming (bool) data.writeInt32(streaming); @@ -274,7 +171,7 @@ public: data.writeInterfaceToken(IProCameraUser::getInterfaceDescriptor()); data.writeInt32(templateId); remote()->transact(CREATE_DEFAULT_REQUEST, data, &reply); - readMetadata(reply, /*out*/request); + CameraMetadata::readFromParcel(reply, /*out*/request); return reply.readInt32(); } @@ -285,7 +182,7 @@ public: data.writeInterfaceToken(IProCameraUser::getInterfaceDescriptor()); data.writeInt32(cameraId); remote()->transact(GET_CAMERA_INFO, data, &reply); - readMetadata(reply, /*out*/info); + CameraMetadata::readFromParcel(reply, /*out*/info); return reply.readInt32(); } @@ -307,6 +204,7 @@ status_t BnProCameraUser::onTransact( ALOGV("DISCONNECT"); CHECK_INTERFACE(IProCameraUser, data, reply); disconnect(); + reply->writeNoException(); return NO_ERROR; } break; case CONNECT: { @@ -341,7 +239,7 @@ status_t BnProCameraUser::onTransact( case SUBMIT_REQUEST: { CHECK_INTERFACE(IProCameraUser, data, reply); camera_metadata_t* metadata; - readMetadata(data, /*out*/&metadata); + CameraMetadata::readFromParcel(data, /*out*/&metadata); // arg2 = streaming (bool) bool streaming = data.readInt32(); @@ -393,7 +291,7 @@ status_t BnProCameraUser::onTransact( status_t ret; ret = createDefaultRequest(templateId, &request); - writeMetadata(*reply, request); + CameraMetadata::writeToParcel(*reply, request); reply->writeInt32(ret); free_camera_metadata(request); @@ -409,7 +307,7 @@ status_t BnProCameraUser::onTransact( status_t ret; ret = getCameraInfo(cameraId, &info); - writeMetadata(*reply, info); + CameraMetadata::writeToParcel(*reply, info); reply->writeInt32(ret); free_camera_metadata(info); diff --git a/camera/ProCamera.cpp b/camera/ProCamera.cpp index fec5461..ba5a48c 100644 --- a/camera/ProCamera.cpp +++ b/camera/ProCamera.cpp @@ -26,7 +26,6 @@ #include <binder/IMemory.h> #include <camera/ProCamera.h> -#include <camera/ICameraService.h> #include <camera/IProCameraUser.h> #include <camera/IProCameraCallbacks.h> @@ -47,6 +46,9 @@ ProCamera::ProCamera(int cameraId) { } +CameraTraits<ProCamera>::TCamConnectService CameraTraits<ProCamera>::fnConnectService = + &ICameraService::connectPro; + ProCamera::~ProCamera() { @@ -88,8 +90,8 @@ void ProCamera::onLockStatusChanged( } } -void ProCamera::onResultReceived(int32_t frameId, camera_metadata* result) { - ALOGV("%s: frameId = %d, result = %p", __FUNCTION__, frameId, result); +void ProCamera::onResultReceived(int32_t requestId, camera_metadata* result) { + ALOGV("%s: requestId = %d, result = %p", __FUNCTION__, requestId, result); sp<ProCameraListener> listener; { @@ -110,7 +112,7 @@ void ProCamera::onResultReceived(int32_t frameId, camera_metadata* result) { result = tmp.release(); if (listener != NULL) { - listener->onResultReceived(frameId, result); + listener->onResultReceived(requestId, result); } else { free_camera_metadata(result); } @@ -247,11 +249,11 @@ status_t ProCamera::createStreamCpu(int width, int height, int format, sp <IProCameraUser> c = mCamera; if (c == 0) return NO_INIT; - sp<CpuConsumer> cc = new CpuConsumer(heapCount, synchronousMode); + sp<BufferQueue> bq = new BufferQueue(); + sp<CpuConsumer> cc = new CpuConsumer(bq, heapCount/*, synchronousMode*/); cc->setName(String8("ProCamera::mCpuConsumer")); - sp<Surface> stc = new Surface( - cc->getProducerInterface()); + sp<Surface> stc = new Surface(bq); status_t s = createStream(width, height, format, stc->getIGraphicBufferProducer(), diff --git a/camera/camera2/CaptureRequest.cpp b/camera/camera2/CaptureRequest.cpp new file mode 100644 index 0000000..57e5319 --- /dev/null +++ b/camera/camera2/CaptureRequest.cpp @@ -0,0 +1,124 @@ +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + +// #define LOG_NDEBUG 0 +#define LOG_TAG "CameraRequest" +#include <utils/Log.h> + +#include <camera/camera2/CaptureRequest.h> + +#include <binder/Parcel.h> +#include <gui/Surface.h> + +namespace android { + +status_t CaptureRequest::readFromParcel(Parcel* parcel) { + if (parcel == NULL) { + ALOGE("%s: Null parcel", __FUNCTION__); + return BAD_VALUE; + } + + mMetadata.clear(); + mSurfaceList.clear(); + + status_t err; + + if ((err = mMetadata.readFromParcel(parcel)) != OK) { + ALOGE("%s: Failed to read metadata from parcel", __FUNCTION__); + return err; + } + ALOGV("%s: Read metadata from parcel", __FUNCTION__); + + int32_t size; + if ((err = parcel->readInt32(&size)) != OK) { + ALOGE("%s: Failed to read surface list size from parcel", __FUNCTION__); + return err; + } + ALOGV("%s: Read surface list size = %d", __FUNCTION__, size); + + // Do not distinguish null arrays from 0-sized arrays. + for (int i = 0; i < size; ++i) { + // Parcel.writeParcelableArray + size_t len; + const char16_t* className = parcel->readString16Inplace(&len); + ALOGV("%s: Read surface class = %s", __FUNCTION__, + className != NULL ? String8(className).string() : "<null>"); + + if (className == NULL) { + continue; + } + + // Surface.writeToParcel + String16 name = parcel->readString16(); + ALOGV("%s: Read surface name = %s", + __FUNCTION__, String8(name).string()); + sp<IBinder> binder(parcel->readStrongBinder()); + ALOGV("%s: Read surface binder = %p", + __FUNCTION__, binder.get()); + + sp<Surface> surface; + + if (binder != NULL) { + sp<IGraphicBufferProducer> gbp = + interface_cast<IGraphicBufferProducer>(binder); + surface = new Surface(gbp); + } + + mSurfaceList.push_back(surface); + } + + return OK; +} + +status_t CaptureRequest::writeToParcel(Parcel* parcel) const { + if (parcel == NULL) { + ALOGE("%s: Null parcel", __FUNCTION__); + return BAD_VALUE; + } + + status_t err; + + if ((err = mMetadata.writeToParcel(parcel)) != OK) { + return err; + } + + int32_t size = static_cast<int32_t>(mSurfaceList.size()); + + // Send 0-sized arrays when it's empty. Do not send null arrays. + parcel->writeInt32(size); + + for (int32_t i = 0; i < size; ++i) { + sp<Surface> surface = mSurfaceList[i]; + + sp<IBinder> binder; + if (surface != 0) { + binder = surface->getIGraphicBufferProducer()->asBinder(); + } + + // not sure if readParcelableArray does this, hard to tell from source + parcel->writeString16(String16("android.view.Surface")); + + // Surface.writeToParcel + parcel->writeString16(String16("unknown_name")); + // Surface.nativeWriteToParcel + parcel->writeStrongBinder(binder); + } + + return OK; +} + +}; // namespace android diff --git a/camera/camera2/ICameraDeviceCallbacks.cpp b/camera/camera2/ICameraDeviceCallbacks.cpp new file mode 100644 index 0000000..613358a --- /dev/null +++ b/camera/camera2/ICameraDeviceCallbacks.cpp @@ -0,0 +1,148 @@ +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "ICameraDeviceCallbacks" +#include <utils/Log.h> +#include <stdint.h> +#include <sys/types.h> + +#include <binder/Parcel.h> +#include <gui/IGraphicBufferProducer.h> +#include <gui/Surface.h> +#include <utils/Mutex.h> + +#include <camera/camera2/ICameraDeviceCallbacks.h> +#include "camera/CameraMetadata.h" + +namespace android { + +enum { + CAMERA_ERROR = IBinder::FIRST_CALL_TRANSACTION, + CAMERA_IDLE, + CAPTURE_STARTED, + RESULT_RECEIVED, +}; + +class BpCameraDeviceCallbacks: public BpInterface<ICameraDeviceCallbacks> +{ +public: + BpCameraDeviceCallbacks(const sp<IBinder>& impl) + : BpInterface<ICameraDeviceCallbacks>(impl) + { + } + + void onDeviceError(CameraErrorCode errorCode) + { + ALOGV("onDeviceError"); + Parcel data, reply; + data.writeInterfaceToken(ICameraDeviceCallbacks::getInterfaceDescriptor()); + data.writeInt32(static_cast<int32_t>(errorCode)); + remote()->transact(CAMERA_ERROR, data, &reply, IBinder::FLAG_ONEWAY); + data.writeNoException(); + } + + void onDeviceIdle() + { + ALOGV("onDeviceIdle"); + Parcel data, reply; + data.writeInterfaceToken(ICameraDeviceCallbacks::getInterfaceDescriptor()); + remote()->transact(CAMERA_IDLE, data, &reply, IBinder::FLAG_ONEWAY); + data.writeNoException(); + } + + void onCaptureStarted(int32_t requestId, int64_t timestamp) + { + ALOGV("onCaptureStarted"); + Parcel data, reply; + data.writeInterfaceToken(ICameraDeviceCallbacks::getInterfaceDescriptor()); + data.writeInt32(requestId); + data.writeInt64(timestamp); + remote()->transact(CAPTURE_STARTED, data, &reply, IBinder::FLAG_ONEWAY); + data.writeNoException(); + } + + + void onResultReceived(int32_t requestId, const CameraMetadata& result) { + ALOGV("onResultReceived"); + Parcel data, reply; + data.writeInterfaceToken(ICameraDeviceCallbacks::getInterfaceDescriptor()); + data.writeInt32(requestId); + data.writeInt32(1); // to mark presence of metadata object + result.writeToParcel(&data); + remote()->transact(RESULT_RECEIVED, data, &reply, IBinder::FLAG_ONEWAY); + data.writeNoException(); + } +}; + +IMPLEMENT_META_INTERFACE(CameraDeviceCallbacks, + "android.hardware.camera2.ICameraDeviceCallbacks"); + +// ---------------------------------------------------------------------- + +status_t BnCameraDeviceCallbacks::onTransact( + uint32_t code, const Parcel& data, Parcel* reply, uint32_t flags) +{ + ALOGV("onTransact - code = %d", code); + switch(code) { + case CAMERA_ERROR: { + ALOGV("onDeviceError"); + CHECK_INTERFACE(ICameraDeviceCallbacks, data, reply); + CameraErrorCode errorCode = + static_cast<CameraErrorCode>(data.readInt32()); + onDeviceError(errorCode); + data.readExceptionCode(); + return NO_ERROR; + } break; + case CAMERA_IDLE: { + ALOGV("onDeviceIdle"); + CHECK_INTERFACE(ICameraDeviceCallbacks, data, reply); + onDeviceIdle(); + data.readExceptionCode(); + return NO_ERROR; + } break; + case CAPTURE_STARTED: { + ALOGV("onCaptureStarted"); + CHECK_INTERFACE(ICameraDeviceCallbacks, data, reply); + int32_t requestId = data.readInt32(); + int64_t timestamp = data.readInt64(); + onCaptureStarted(requestId, timestamp); + data.readExceptionCode(); + return NO_ERROR; + } break; + case RESULT_RECEIVED: { + ALOGV("onResultReceived"); + CHECK_INTERFACE(ICameraDeviceCallbacks, data, reply); + int32_t requestId = data.readInt32(); + CameraMetadata result; + if (data.readInt32() != 0) { + result.readFromParcel(const_cast<Parcel*>(&data)); + } else { + ALOGW("No metadata object is present in result"); + } + onResultReceived(requestId, result); + data.readExceptionCode(); + return NO_ERROR; + } break; + default: + return BBinder::onTransact(code, data, reply, flags); + } +} + +// ---------------------------------------------------------------------------- + +}; // namespace android diff --git a/camera/camera2/ICameraDeviceUser.cpp b/camera/camera2/ICameraDeviceUser.cpp new file mode 100644 index 0000000..1e5822f --- /dev/null +++ b/camera/camera2/ICameraDeviceUser.cpp @@ -0,0 +1,352 @@ +/* +** +** Copyright 2013, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + +// #define LOG_NDEBUG 0 +#define LOG_TAG "ICameraDeviceUser" +#include <utils/Log.h> +#include <stdint.h> +#include <sys/types.h> +#include <binder/Parcel.h> +#include <camera/camera2/ICameraDeviceUser.h> +#include <gui/IGraphicBufferProducer.h> +#include <gui/Surface.h> +#include <camera/CameraMetadata.h> +#include <camera/camera2/CaptureRequest.h> + +namespace android { + +typedef Parcel::WritableBlob WritableBlob; +typedef Parcel::ReadableBlob ReadableBlob; + +enum { + DISCONNECT = IBinder::FIRST_CALL_TRANSACTION, + SUBMIT_REQUEST, + CANCEL_REQUEST, + DELETE_STREAM, + CREATE_STREAM, + CREATE_DEFAULT_REQUEST, + GET_CAMERA_INFO, + WAIT_UNTIL_IDLE, + FLUSH +}; + +namespace { + // Read empty strings without printing a false error message. + String16 readMaybeEmptyString16(const Parcel& parcel) { + size_t len; + const char16_t* str = parcel.readString16Inplace(&len); + if (str != NULL) { + return String16(str, len); + } else { + return String16(); + } + } +}; + +class BpCameraDeviceUser : public BpInterface<ICameraDeviceUser> +{ +public: + BpCameraDeviceUser(const sp<IBinder>& impl) + : BpInterface<ICameraDeviceUser>(impl) + { + } + + // disconnect from camera service + void disconnect() + { + ALOGV("disconnect"); + Parcel data, reply; + data.writeInterfaceToken(ICameraDeviceUser::getInterfaceDescriptor()); + remote()->transact(DISCONNECT, data, &reply); + reply.readExceptionCode(); + } + + virtual int submitRequest(sp<CaptureRequest> request, bool streaming) + { + Parcel data, reply; + data.writeInterfaceToken(ICameraDeviceUser::getInterfaceDescriptor()); + + // arg0 = CaptureRequest + if (request != 0) { + data.writeInt32(1); + request->writeToParcel(&data); + } else { + data.writeInt32(0); + } + + // arg1 = streaming (bool) + data.writeInt32(streaming); + + remote()->transact(SUBMIT_REQUEST, data, &reply); + + reply.readExceptionCode(); + return reply.readInt32(); + } + + virtual status_t cancelRequest(int requestId) + { + Parcel data, reply; + data.writeInterfaceToken(ICameraDeviceUser::getInterfaceDescriptor()); + data.writeInt32(requestId); + + remote()->transact(CANCEL_REQUEST, data, &reply); + + reply.readExceptionCode(); + return reply.readInt32(); + } + + virtual status_t deleteStream(int streamId) + { + Parcel data, reply; + data.writeInterfaceToken(ICameraDeviceUser::getInterfaceDescriptor()); + data.writeInt32(streamId); + + remote()->transact(DELETE_STREAM, data, &reply); + + reply.readExceptionCode(); + return reply.readInt32(); + } + + virtual status_t createStream(int width, int height, int format, + const sp<IGraphicBufferProducer>& bufferProducer) + { + Parcel data, reply; + data.writeInterfaceToken(ICameraDeviceUser::getInterfaceDescriptor()); + data.writeInt32(width); + data.writeInt32(height); + data.writeInt32(format); + + data.writeInt32(1); // marker that bufferProducer is not null + data.writeString16(String16("unknown_name")); // name of surface + sp<IBinder> b(bufferProducer->asBinder()); + data.writeStrongBinder(b); + + remote()->transact(CREATE_STREAM, data, &reply); + + reply.readExceptionCode(); + return reply.readInt32(); + } + + // Create a request object from a template. + virtual status_t createDefaultRequest(int templateId, + /*out*/ + CameraMetadata* request) + { + Parcel data, reply; + data.writeInterfaceToken(ICameraDeviceUser::getInterfaceDescriptor()); + data.writeInt32(templateId); + remote()->transact(CREATE_DEFAULT_REQUEST, data, &reply); + + reply.readExceptionCode(); + status_t result = reply.readInt32(); + + CameraMetadata out; + if (reply.readInt32() != 0) { + out.readFromParcel(&reply); + } + + if (request != NULL) { + request->swap(out); + } + return result; + } + + + virtual status_t getCameraInfo(CameraMetadata* info) + { + Parcel data, reply; + data.writeInterfaceToken(ICameraDeviceUser::getInterfaceDescriptor()); + remote()->transact(GET_CAMERA_INFO, data, &reply); + + reply.readExceptionCode(); + status_t result = reply.readInt32(); + + CameraMetadata out; + if (reply.readInt32() != 0) { + out.readFromParcel(&reply); + } + + if (info != NULL) { + info->swap(out); + } + + return result; + } + + virtual status_t waitUntilIdle() + { + ALOGV("waitUntilIdle"); + Parcel data, reply; + data.writeInterfaceToken(ICameraDeviceUser::getInterfaceDescriptor()); + remote()->transact(WAIT_UNTIL_IDLE, data, &reply); + reply.readExceptionCode(); + return reply.readInt32(); + } + + virtual status_t flush() + { + ALOGV("flush"); + Parcel data, reply; + data.writeInterfaceToken(ICameraDeviceUser::getInterfaceDescriptor()); + remote()->transact(FLUSH, data, &reply); + reply.readExceptionCode(); + return reply.readInt32(); + } + +private: + + +}; + +IMPLEMENT_META_INTERFACE(CameraDeviceUser, + "android.hardware.camera2.ICameraDeviceUser"); + +// ---------------------------------------------------------------------- + +status_t BnCameraDeviceUser::onTransact( + uint32_t code, const Parcel& data, Parcel* reply, uint32_t flags) +{ + switch(code) { + case DISCONNECT: { + ALOGV("DISCONNECT"); + CHECK_INTERFACE(ICameraDeviceUser, data, reply); + disconnect(); + reply->writeNoException(); + return NO_ERROR; + } break; + case SUBMIT_REQUEST: { + CHECK_INTERFACE(ICameraDeviceUser, data, reply); + + // arg0 = request + sp<CaptureRequest> request; + if (data.readInt32() != 0) { + request = new CaptureRequest(); + request->readFromParcel(const_cast<Parcel*>(&data)); + } + + // arg1 = streaming (bool) + bool streaming = data.readInt32(); + + // return code: requestId (int32) + reply->writeNoException(); + reply->writeInt32(submitRequest(request, streaming)); + + return NO_ERROR; + } break; + case CANCEL_REQUEST: { + CHECK_INTERFACE(ICameraDeviceUser, data, reply); + int requestId = data.readInt32(); + reply->writeNoException(); + reply->writeInt32(cancelRequest(requestId)); + return NO_ERROR; + } break; + case DELETE_STREAM: { + CHECK_INTERFACE(ICameraDeviceUser, data, reply); + int streamId = data.readInt32(); + reply->writeNoException(); + reply->writeInt32(deleteStream(streamId)); + return NO_ERROR; + } break; + case CREATE_STREAM: { + CHECK_INTERFACE(ICameraDeviceUser, data, reply); + int width, height, format; + + width = data.readInt32(); + ALOGV("%s: CREATE_STREAM: width = %d", __FUNCTION__, width); + height = data.readInt32(); + ALOGV("%s: CREATE_STREAM: height = %d", __FUNCTION__, height); + format = data.readInt32(); + ALOGV("%s: CREATE_STREAM: format = %d", __FUNCTION__, format); + + sp<IGraphicBufferProducer> bp; + if (data.readInt32() != 0) { + String16 name = readMaybeEmptyString16(data); + bp = interface_cast<IGraphicBufferProducer>( + data.readStrongBinder()); + + ALOGV("%s: CREATE_STREAM: bp = %p, name = %s", __FUNCTION__, + bp.get(), String8(name).string()); + } else { + ALOGV("%s: CREATE_STREAM: bp = unset, name = unset", + __FUNCTION__); + } + + status_t ret; + ret = createStream(width, height, format, bp); + + reply->writeNoException(); + ALOGV("%s: CREATE_STREAM: write noException", __FUNCTION__); + reply->writeInt32(ret); + ALOGV("%s: CREATE_STREAM: write ret = %d", __FUNCTION__, ret); + + return NO_ERROR; + } break; + + case CREATE_DEFAULT_REQUEST: { + CHECK_INTERFACE(ICameraDeviceUser, data, reply); + + int templateId = data.readInt32(); + + CameraMetadata request; + status_t ret; + ret = createDefaultRequest(templateId, &request); + + reply->writeNoException(); + reply->writeInt32(ret); + + // out-variables are after exception and return value + reply->writeInt32(1); // to mark presence of metadata object + request.writeToParcel(const_cast<Parcel*>(reply)); + + return NO_ERROR; + } break; + case GET_CAMERA_INFO: { + CHECK_INTERFACE(ICameraDeviceUser, data, reply); + + CameraMetadata info; + status_t ret; + ret = getCameraInfo(&info); + + reply->writeNoException(); + reply->writeInt32(ret); + + // out-variables are after exception and return value + reply->writeInt32(1); // to mark presence of metadata object + info.writeToParcel(reply); + + return NO_ERROR; + } break; + case WAIT_UNTIL_IDLE: { + CHECK_INTERFACE(ICameraDeviceUser, data, reply); + reply->writeNoException(); + reply->writeInt32(waitUntilIdle()); + return NO_ERROR; + } break; + case FLUSH: { + CHECK_INTERFACE(ICameraDeviceUser, data, reply); + reply->writeNoException(); + reply->writeInt32(flush()); + return NO_ERROR; + } + default: + return BBinder::onTransact(code, data, reply, flags); + } +} + +// ---------------------------------------------------------------------------- + +}; // namespace android diff --git a/camera/tests/ProCameraTests.cpp b/camera/tests/ProCameraTests.cpp index f203949..1f5867a 100644 --- a/camera/tests/ProCameraTests.cpp +++ b/camera/tests/ProCameraTests.cpp @@ -271,7 +271,6 @@ protected: CpuConsumer::LockedBuffer buf; status_t ret; - EXPECT_OK(ret); if (OK == (ret = consumer->lockNextBuffer(&buf))) { dout << "Frame received on streamId = " << streamId << @@ -285,9 +284,9 @@ protected: } } - virtual void onResultReceived(int32_t frameId, + virtual void onResultReceived(int32_t requestId, camera_metadata* request) { - dout << "Result received frameId = " << frameId + dout << "Result received requestId = " << requestId << ", requestPtr = " << (void*)request << std::endl; QueueEvent(RESULT_RECEIVED); free_camera_metadata(request); @@ -482,7 +481,7 @@ protected: * Creating a streaming request for these output streams from a template, * and submit it */ - void createSubmitRequestForStreams(uint8_t* streamIds, size_t count, int requestCount=-1) { + void createSubmitRequestForStreams(int32_t* streamIds, size_t count, int requestCount=-1) { ASSERT_NE((void*)NULL, streamIds); ASSERT_LT(0u, count); @@ -629,7 +628,7 @@ TEST_F(ProCameraTest, DISABLED_StreamingImageSingle) { EXPECT_OK(mCamera->exclusiveTryLock()); - uint8_t streams[] = { depthStreamId }; + int32_t streams[] = { depthStreamId }; ASSERT_NO_FATAL_FAILURE(createSubmitRequestForStreams( streams, /*count*/1)); @@ -706,7 +705,7 @@ TEST_F(ProCameraTest, DISABLED_StreamingImageDual) { // set the output streams to just this stream ID // wow what a verbose API. - uint8_t allStreams[] = { streamId, depthStreamId }; + int32_t allStreams[] = { streamId, depthStreamId }; // IMPORTANT. bad things will happen if its not a uint8. size_t streamCount = sizeof(allStreams) / sizeof(allStreams[0]); camera_metadata_entry_t entry; @@ -735,7 +734,7 @@ TEST_F(ProCameraTest, DISABLED_StreamingImageDual) { free_camera_metadata(request); - for (int i = 0; i < streamCount; ++i) { + for (size_t i = 0; i < streamCount; ++i) { EXPECT_OK(mCamera->deleteStream(allStreams[i])); } EXPECT_OK(mCamera->exclusiveUnlock()); @@ -777,7 +776,7 @@ TEST_F(ProCameraTest, CpuConsumerSingle) { // set the output streams to just this stream ID - uint8_t allStreams[] = { streamId }; + int32_t allStreams[] = { streamId }; camera_metadata_entry_t entry; uint32_t tag = static_cast<uint32_t>(ANDROID_REQUEST_OUTPUT_STREAMS); int find = find_camera_metadata_entry(request, tag, &entry); @@ -848,7 +847,7 @@ TEST_F(ProCameraTest, CpuConsumerDual) { // set the output streams to just this stream ID // wow what a verbose API. - uint8_t allStreams[] = { streamId, depthStreamId }; + int32_t allStreams[] = { streamId, depthStreamId }; size_t streamCount = 2; camera_metadata_entry_t entry; uint32_t tag = static_cast<uint32_t>(ANDROID_REQUEST_OUTPUT_STREAMS); @@ -923,7 +922,7 @@ TEST_F(ProCameraTest, ResultReceiver) { // set the output streams to just this stream ID - uint8_t allStreams[] = { streamId }; + int32_t allStreams[] = { streamId }; size_t streamCount = 1; camera_metadata_entry_t entry; uint32_t tag = static_cast<uint32_t>(ANDROID_REQUEST_OUTPUT_STREAMS); @@ -974,7 +973,7 @@ TEST_F(ProCameraTest, DISABLED_WaitForResult) { EXPECT_OK(mCamera->exclusiveTryLock()); - uint8_t streams[] = { streamId }; + int32_t streams[] = { streamId }; ASSERT_NO_FATAL_FAILURE(createSubmitRequestForStreams(streams, /*count*/1)); // Consume a couple of results @@ -1002,7 +1001,7 @@ TEST_F(ProCameraTest, WaitForSingleStreamBuffer) { EXPECT_OK(mCamera->exclusiveTryLock()); - uint8_t streams[] = { streamId }; + int32_t streams[] = { streamId }; ASSERT_NO_FATAL_FAILURE(createSubmitRequestForStreams(streams, /*count*/1, /*requests*/TEST_CPU_FRAME_COUNT)); @@ -1049,7 +1048,7 @@ TEST_F(ProCameraTest, DISABLED_WaitForDualStreamBuffer) { EXPECT_OK(mCamera->exclusiveTryLock()); - uint8_t streams[] = { streamId, depthStreamId }; + int32_t streams[] = { streamId, depthStreamId }; ASSERT_NO_FATAL_FAILURE(createSubmitRequestForStreams(streams, /*count*/2, /*requests*/REQUEST_COUNT)); @@ -1128,7 +1127,7 @@ TEST_F(ProCameraTest, WaitForSingleStreamBufferAndDropFramesSync) { EXPECT_OK(mCamera->exclusiveTryLock()); - uint8_t streams[] = { streamId }; + int32_t streams[] = { streamId }; ASSERT_NO_FATAL_FAILURE(createSubmitRequestForStreams(streams, /*count*/1, /*requests*/NUM_REQUESTS)); @@ -1172,7 +1171,6 @@ TEST_F(ProCameraTest, WaitForSingleStreamBufferAndDropFramesAsync) { } const int NUM_REQUESTS = 20 * TEST_CPU_FRAME_COUNT; - const int CONSECUTIVE_FAILS_ASSUME_TIME_OUT = 5; int streamId = -1; sp<CpuConsumer> consumer; @@ -1183,7 +1181,7 @@ TEST_F(ProCameraTest, WaitForSingleStreamBufferAndDropFramesAsync) { EXPECT_OK(mCamera->exclusiveTryLock()); - uint8_t streams[] = { streamId }; + int32_t streams[] = { streamId }; ASSERT_NO_FATAL_FAILURE(createSubmitRequestForStreams(streams, /*count*/1, /*requests*/NUM_REQUESTS)); @@ -1278,4 +1276,3 @@ TEST_F(ProCameraTest, ServiceListenersFunctional) { } } } - diff --git a/cmds/screenrecord/Android.mk b/cmds/screenrecord/Android.mk new file mode 100644 index 0000000..b4a5947 --- /dev/null +++ b/cmds/screenrecord/Android.mk @@ -0,0 +1,38 @@ +# Copyright 2013 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +LOCAL_PATH:= $(call my-dir) + +include $(CLEAR_VARS) + +LOCAL_SRC_FILES := \ + screenrecord.cpp \ + +LOCAL_SHARED_LIBRARIES := \ + libstagefright libmedia libutils libbinder libstagefright_foundation \ + libjpeg libgui libcutils liblog + +LOCAL_C_INCLUDES := \ + frameworks/av/media/libstagefright \ + frameworks/av/media/libstagefright/include \ + $(TOP)/frameworks/native/include/media/openmax \ + external/jpeg + +LOCAL_CFLAGS += -Wno-multichar + +LOCAL_MODULE_TAGS := optional + +LOCAL_MODULE:= screenrecord + +include $(BUILD_EXECUTABLE) diff --git a/cmds/screenrecord/screenrecord.cpp b/cmds/screenrecord/screenrecord.cpp new file mode 100644 index 0000000..49999b5 --- /dev/null +++ b/cmds/screenrecord/screenrecord.cpp @@ -0,0 +1,739 @@ +/* + * Copyright 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "ScreenRecord" +//#define LOG_NDEBUG 0 +#include <utils/Log.h> + +#include <binder/IPCThreadState.h> +#include <utils/Errors.h> +#include <utils/Thread.h> +#include <utils/Timers.h> + +#include <gui/Surface.h> +#include <gui/SurfaceComposerClient.h> +#include <gui/ISurfaceComposer.h> +#include <ui/DisplayInfo.h> +#include <media/openmax/OMX_IVCommon.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/MediaCodec.h> +#include <media/stagefright/MediaErrors.h> +#include <media/stagefright/MediaMuxer.h> +#include <media/ICrypto.h> + +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <stdio.h> +#include <fcntl.h> +#include <signal.h> +#include <getopt.h> +#include <sys/wait.h> + +using namespace android; + +static const uint32_t kMinBitRate = 100000; // 0.1Mbps +static const uint32_t kMaxBitRate = 100 * 1000000; // 100Mbps +static const uint32_t kMaxTimeLimitSec = 180; // 3 minutes +static const uint32_t kFallbackWidth = 1280; // 720p +static const uint32_t kFallbackHeight = 720; + +// Command-line parameters. +static bool gVerbose = false; // chatty on stdout +static bool gRotate = false; // rotate 90 degrees +static bool gSizeSpecified = false; // was size explicitly requested? +static uint32_t gVideoWidth = 0; // default width+height +static uint32_t gVideoHeight = 0; +static uint32_t gBitRate = 4000000; // 4Mbps +static uint32_t gTimeLimitSec = kMaxTimeLimitSec; + +// Set by signal handler to stop recording. +static bool gStopRequested; + +// Previous signal handler state, restored after first hit. +static struct sigaction gOrigSigactionINT; +static struct sigaction gOrigSigactionHUP; + + +/* + * Catch keyboard interrupt signals. On receipt, the "stop requested" + * flag is raised, and the original handler is restored (so that, if + * we get stuck finishing, a second Ctrl-C will kill the process). + */ +static void signalCatcher(int signum) +{ + gStopRequested = true; + switch (signum) { + case SIGINT: + case SIGHUP: + sigaction(SIGINT, &gOrigSigactionINT, NULL); + sigaction(SIGHUP, &gOrigSigactionHUP, NULL); + break; + default: + abort(); + break; + } +} + +/* + * Configures signal handlers. The previous handlers are saved. + * + * If the command is run from an interactive adb shell, we get SIGINT + * when Ctrl-C is hit. If we're run from the host, the local adb process + * gets the signal, and we get a SIGHUP when the terminal disconnects. + */ +static status_t configureSignals() +{ + struct sigaction act; + memset(&act, 0, sizeof(act)); + act.sa_handler = signalCatcher; + if (sigaction(SIGINT, &act, &gOrigSigactionINT) != 0) { + status_t err = -errno; + fprintf(stderr, "Unable to configure SIGINT handler: %s\n", + strerror(errno)); + return err; + } + if (sigaction(SIGHUP, &act, &gOrigSigactionHUP) != 0) { + status_t err = -errno; + fprintf(stderr, "Unable to configure SIGHUP handler: %s\n", + strerror(errno)); + return err; + } + return NO_ERROR; +} + +/* + * Returns "true" if the device is rotated 90 degrees. + */ +static bool isDeviceRotated(int orientation) { + return orientation != DISPLAY_ORIENTATION_0 && + orientation != DISPLAY_ORIENTATION_180; +} + +/* + * Configures and starts the MediaCodec encoder. Obtains an input surface + * from the codec. + */ +static status_t prepareEncoder(float displayFps, sp<MediaCodec>* pCodec, + sp<IGraphicBufferProducer>* pBufferProducer) { + status_t err; + + if (gVerbose) { + printf("Configuring recorder for %dx%d video at %.2fMbps\n", + gVideoWidth, gVideoHeight, gBitRate / 1000000.0); + } + + sp<AMessage> format = new AMessage; + format->setInt32("width", gVideoWidth); + format->setInt32("height", gVideoHeight); + format->setString("mime", "video/avc"); + format->setInt32("color-format", OMX_COLOR_FormatAndroidOpaque); + format->setInt32("bitrate", gBitRate); + format->setFloat("frame-rate", displayFps); + format->setInt32("i-frame-interval", 10); + + sp<ALooper> looper = new ALooper; + looper->setName("screenrecord_looper"); + looper->start(); + ALOGV("Creating codec"); + sp<MediaCodec> codec = MediaCodec::CreateByType(looper, "video/avc", true); + if (codec == NULL) { + fprintf(stderr, "ERROR: unable to create video/avc codec instance\n"); + return UNKNOWN_ERROR; + } + err = codec->configure(format, NULL, NULL, + MediaCodec::CONFIGURE_FLAG_ENCODE); + if (err != NO_ERROR) { + codec->release(); + codec.clear(); + + fprintf(stderr, "ERROR: unable to configure codec (err=%d)\n", err); + return err; + } + + ALOGV("Creating buffer producer"); + sp<IGraphicBufferProducer> bufferProducer; + err = codec->createInputSurface(&bufferProducer); + if (err != NO_ERROR) { + codec->release(); + codec.clear(); + + fprintf(stderr, + "ERROR: unable to create encoder input surface (err=%d)\n", err); + return err; + } + + ALOGV("Starting codec"); + err = codec->start(); + if (err != NO_ERROR) { + codec->release(); + codec.clear(); + + fprintf(stderr, "ERROR: unable to start codec (err=%d)\n", err); + return err; + } + + ALOGV("Codec prepared"); + *pCodec = codec; + *pBufferProducer = bufferProducer; + return 0; +} + +/* + * Configures the virtual display. When this completes, virtual display + * frames will start being sent to the encoder's surface. + */ +static status_t prepareVirtualDisplay(const DisplayInfo& mainDpyInfo, + const sp<IGraphicBufferProducer>& bufferProducer, + sp<IBinder>* pDisplayHandle) { + status_t err; + + // Set the region of the layer stack we're interested in, which in our + // case is "all of it". If the app is rotated (so that the width of the + // app is based on the height of the display), reverse width/height. + bool deviceRotated = isDeviceRotated(mainDpyInfo.orientation); + uint32_t sourceWidth, sourceHeight; + if (!deviceRotated) { + sourceWidth = mainDpyInfo.w; + sourceHeight = mainDpyInfo.h; + } else { + ALOGV("using rotated width/height"); + sourceHeight = mainDpyInfo.w; + sourceWidth = mainDpyInfo.h; + } + Rect layerStackRect(sourceWidth, sourceHeight); + + // We need to preserve the aspect ratio of the display. + float displayAspect = (float) sourceHeight / (float) sourceWidth; + + + // Set the way we map the output onto the display surface (which will + // be e.g. 1280x720 for a 720p video). The rect is interpreted + // post-rotation, so if the display is rotated 90 degrees we need to + // "pre-rotate" it by flipping width/height, so that the orientation + // adjustment changes it back. + // + // We might want to encode a portrait display as landscape to use more + // of the screen real estate. (If players respect a 90-degree rotation + // hint, we can essentially get a 720x1280 video instead of 1280x720.) + // In that case, we swap the configured video width/height and then + // supply a rotation value to the display projection. + uint32_t videoWidth, videoHeight; + uint32_t outWidth, outHeight; + if (!gRotate) { + videoWidth = gVideoWidth; + videoHeight = gVideoHeight; + } else { + videoWidth = gVideoHeight; + videoHeight = gVideoWidth; + } + if (videoHeight > (uint32_t)(videoWidth * displayAspect)) { + // limited by narrow width; reduce height + outWidth = videoWidth; + outHeight = (uint32_t)(videoWidth * displayAspect); + } else { + // limited by short height; restrict width + outHeight = videoHeight; + outWidth = (uint32_t)(videoHeight / displayAspect); + } + uint32_t offX, offY; + offX = (videoWidth - outWidth) / 2; + offY = (videoHeight - outHeight) / 2; + Rect displayRect(offX, offY, offX + outWidth, offY + outHeight); + + if (gVerbose) { + if (gRotate) { + printf("Rotated content area is %ux%u at offset x=%d y=%d\n", + outHeight, outWidth, offY, offX); + } else { + printf("Content area is %ux%u at offset x=%d y=%d\n", + outWidth, outHeight, offX, offY); + } + } + + + sp<IBinder> dpy = SurfaceComposerClient::createDisplay( + String8("ScreenRecorder"), false /* secure */); + + SurfaceComposerClient::openGlobalTransaction(); + SurfaceComposerClient::setDisplaySurface(dpy, bufferProducer); + SurfaceComposerClient::setDisplayProjection(dpy, + gRotate ? DISPLAY_ORIENTATION_90 : DISPLAY_ORIENTATION_0, + layerStackRect, displayRect); + SurfaceComposerClient::setDisplayLayerStack(dpy, 0); // default stack + SurfaceComposerClient::closeGlobalTransaction(); + + *pDisplayHandle = dpy; + + return NO_ERROR; +} + +/* + * Runs the MediaCodec encoder, sending the output to the MediaMuxer. The + * input frames are coming from the virtual display as fast as SurfaceFlinger + * wants to send them. + * + * The muxer must *not* have been started before calling. + */ +static status_t runEncoder(const sp<MediaCodec>& encoder, + const sp<MediaMuxer>& muxer) { + static int kTimeout = 250000; // be responsive on signal + status_t err; + ssize_t trackIdx = -1; + uint32_t debugNumFrames = 0; + int64_t startWhenNsec = systemTime(CLOCK_MONOTONIC); + int64_t endWhenNsec = startWhenNsec + seconds_to_nanoseconds(gTimeLimitSec); + + Vector<sp<ABuffer> > buffers; + err = encoder->getOutputBuffers(&buffers); + if (err != NO_ERROR) { + fprintf(stderr, "Unable to get output buffers (err=%d)\n", err); + return err; + } + + // This is set by the signal handler. + gStopRequested = false; + + // Run until we're signaled. + while (!gStopRequested) { + size_t bufIndex, offset, size; + int64_t ptsUsec; + uint32_t flags; + + if (systemTime(CLOCK_MONOTONIC) > endWhenNsec) { + if (gVerbose) { + printf("Time limit reached\n"); + } + break; + } + + ALOGV("Calling dequeueOutputBuffer"); + err = encoder->dequeueOutputBuffer(&bufIndex, &offset, &size, &ptsUsec, + &flags, kTimeout); + ALOGV("dequeueOutputBuffer returned %d", err); + switch (err) { + case NO_ERROR: + // got a buffer + if ((flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) != 0) { + // ignore this -- we passed the CSD into MediaMuxer when + // we got the format change notification + ALOGV("Got codec config buffer (%u bytes); ignoring", size); + size = 0; + } + if (size != 0) { + ALOGV("Got data in buffer %d, size=%d, pts=%lld", + bufIndex, size, ptsUsec); + CHECK(trackIdx != -1); + + // If the virtual display isn't providing us with timestamps, + // use the current time. + if (ptsUsec == 0) { + ptsUsec = systemTime(SYSTEM_TIME_MONOTONIC) / 1000; + } + + // The MediaMuxer docs are unclear, but it appears that we + // need to pass either the full set of BufferInfo flags, or + // (flags & BUFFER_FLAG_SYNCFRAME). + err = muxer->writeSampleData(buffers[bufIndex], trackIdx, + ptsUsec, flags); + if (err != NO_ERROR) { + fprintf(stderr, "Failed writing data to muxer (err=%d)\n", + err); + return err; + } + debugNumFrames++; + } + err = encoder->releaseOutputBuffer(bufIndex); + if (err != NO_ERROR) { + fprintf(stderr, "Unable to release output buffer (err=%d)\n", + err); + return err; + } + if ((flags & MediaCodec::BUFFER_FLAG_EOS) != 0) { + // Not expecting EOS from SurfaceFlinger. Go with it. + ALOGD("Received end-of-stream"); + gStopRequested = false; + } + break; + case -EAGAIN: // INFO_TRY_AGAIN_LATER + ALOGV("Got -EAGAIN, looping"); + break; + case INFO_FORMAT_CHANGED: // INFO_OUTPUT_FORMAT_CHANGED + { + // format includes CSD, which we must provide to muxer + ALOGV("Encoder format changed"); + sp<AMessage> newFormat; + encoder->getOutputFormat(&newFormat); + trackIdx = muxer->addTrack(newFormat); + ALOGV("Starting muxer"); + err = muxer->start(); + if (err != NO_ERROR) { + fprintf(stderr, "Unable to start muxer (err=%d)\n", err); + return err; + } + } + break; + case INFO_OUTPUT_BUFFERS_CHANGED: // INFO_OUTPUT_BUFFERS_CHANGED + // not expected for an encoder; handle it anyway + ALOGV("Encoder buffers changed"); + err = encoder->getOutputBuffers(&buffers); + if (err != NO_ERROR) { + fprintf(stderr, + "Unable to get new output buffers (err=%d)\n", err); + return err; + } + break; + case INVALID_OPERATION: + fprintf(stderr, "Request for encoder buffer failed\n"); + return err; + default: + fprintf(stderr, + "Got weird result %d from dequeueOutputBuffer\n", err); + return err; + } + } + + ALOGV("Encoder stopping (req=%d)", gStopRequested); + if (gVerbose) { + printf("Encoder stopping; recorded %u frames in %lld seconds\n", + debugNumFrames, + nanoseconds_to_seconds(systemTime(CLOCK_MONOTONIC) - startWhenNsec)); + } + return NO_ERROR; +} + +/* + * Main "do work" method. + * + * Configures codec, muxer, and virtual display, then starts moving bits + * around. + */ +static status_t recordScreen(const char* fileName) { + status_t err; + + // Configure signal handler. + err = configureSignals(); + if (err != NO_ERROR) return err; + + // Start Binder thread pool. MediaCodec needs to be able to receive + // messages from mediaserver. + sp<ProcessState> self = ProcessState::self(); + self->startThreadPool(); + + // Get main display parameters. + sp<IBinder> mainDpy = SurfaceComposerClient::getBuiltInDisplay( + ISurfaceComposer::eDisplayIdMain); + DisplayInfo mainDpyInfo; + err = SurfaceComposerClient::getDisplayInfo(mainDpy, &mainDpyInfo); + if (err != NO_ERROR) { + fprintf(stderr, "ERROR: unable to get display characteristics\n"); + return err; + } + if (gVerbose) { + printf("Main display is %dx%d @%.2ffps (orientation=%u)\n", + mainDpyInfo.w, mainDpyInfo.h, mainDpyInfo.fps, + mainDpyInfo.orientation); + } + + bool rotated = isDeviceRotated(mainDpyInfo.orientation); + if (gVideoWidth == 0) { + gVideoWidth = rotated ? mainDpyInfo.h : mainDpyInfo.w; + } + if (gVideoHeight == 0) { + gVideoHeight = rotated ? mainDpyInfo.w : mainDpyInfo.h; + } + + // Configure and start the encoder. + sp<MediaCodec> encoder; + sp<IGraphicBufferProducer> bufferProducer; + err = prepareEncoder(mainDpyInfo.fps, &encoder, &bufferProducer); + + if (err != NO_ERROR && !gSizeSpecified) { + // fallback is defined for landscape; swap if we're in portrait + bool needSwap = gVideoWidth < gVideoHeight; + uint32_t newWidth = needSwap ? kFallbackHeight : kFallbackWidth; + uint32_t newHeight = needSwap ? kFallbackWidth : kFallbackHeight; + if (gVideoWidth != newWidth && gVideoHeight != newHeight) { + ALOGV("Retrying with 720p"); + fprintf(stderr, "WARNING: failed at %dx%d, retrying at %dx%d\n", + gVideoWidth, gVideoHeight, newWidth, newHeight); + gVideoWidth = newWidth; + gVideoHeight = newHeight; + err = prepareEncoder(mainDpyInfo.fps, &encoder, &bufferProducer); + } + } + if (err != NO_ERROR) { + return err; + } + + // Configure virtual display. + sp<IBinder> dpy; + err = prepareVirtualDisplay(mainDpyInfo, bufferProducer, &dpy); + if (err != NO_ERROR) { + encoder->release(); + encoder.clear(); + + return err; + } + + // Configure, but do not start, muxer. + sp<MediaMuxer> muxer = new MediaMuxer(fileName, + MediaMuxer::OUTPUT_FORMAT_MPEG_4); + if (gRotate) { + muxer->setOrientationHint(90); + } + + // Main encoder loop. + err = runEncoder(encoder, muxer); + if (err != NO_ERROR) { + encoder->release(); + encoder.clear(); + + return err; + } + + if (gVerbose) { + printf("Stopping encoder and muxer\n"); + } + + // Shut everything down, starting with the producer side. + bufferProducer = NULL; + SurfaceComposerClient::destroyDisplay(dpy); + + encoder->stop(); + muxer->stop(); + encoder->release(); + + return 0; +} + +/* + * Sends a broadcast to the media scanner to tell it about the new video. + * + * This is optional, but nice to have. + */ +static status_t notifyMediaScanner(const char* fileName) { + pid_t pid = fork(); + if (pid < 0) { + int err = errno; + ALOGW("fork() failed: %s", strerror(err)); + return -err; + } else if (pid > 0) { + // parent; wait for the child, mostly to make the verbose-mode output + // look right, but also to check for and log failures + int status; + pid_t actualPid = TEMP_FAILURE_RETRY(waitpid(pid, &status, 0)); + if (actualPid != pid) { + ALOGW("waitpid() returned %d (errno=%d)", actualPid, errno); + } else if (status != 0) { + ALOGW("'am broadcast' exited with status=%d", status); + } else { + ALOGV("'am broadcast' exited successfully"); + } + } else { + const char* kCommand = "/system/bin/am"; + + // child; we're single-threaded, so okay to alloc + String8 fileUrl("file://"); + fileUrl.append(fileName); + const char* const argv[] = { + kCommand, + "broadcast", + "-a", + "android.intent.action.MEDIA_SCANNER_SCAN_FILE", + "-d", + fileUrl.string(), + NULL + }; + if (gVerbose) { + printf("Executing:"); + for (int i = 0; argv[i] != NULL; i++) { + printf(" %s", argv[i]); + } + putchar('\n'); + } else { + // non-verbose, suppress 'am' output + ALOGV("closing stdout/stderr in child"); + int fd = open("/dev/null", O_WRONLY); + if (fd >= 0) { + dup2(fd, STDOUT_FILENO); + dup2(fd, STDERR_FILENO); + close(fd); + } + } + execv(kCommand, const_cast<char* const*>(argv)); + ALOGE("execv(%s) failed: %s\n", kCommand, strerror(errno)); + exit(1); + } + return NO_ERROR; +} + +/* + * Parses a string of the form "1280x720". + * + * Returns true on success. + */ +static bool parseWidthHeight(const char* widthHeight, uint32_t* pWidth, + uint32_t* pHeight) { + long width, height; + char* end; + + // Must specify base 10, or "0x0" gets parsed differently. + width = strtol(widthHeight, &end, 10); + if (end == widthHeight || *end != 'x' || *(end+1) == '\0') { + // invalid chars in width, or missing 'x', or missing height + return false; + } + height = strtol(end + 1, &end, 10); + if (*end != '\0') { + // invalid chars in height + return false; + } + + *pWidth = width; + *pHeight = height; + return true; +} + +/* + * Dumps usage on stderr. + */ +static void usage() { + fprintf(stderr, + "Usage: screenrecord [options] <filename>\n" + "\n" + "Records the device's display to a .mp4 file.\n" + "\n" + "Options:\n" + "--size WIDTHxHEIGHT\n" + " Set the video size, e.g. \"1280x720\". Default is the device's main\n" + " display resolution (if supported), 1280x720 if not. For best results,\n" + " use a size supported by the AVC encoder.\n" + "--bit-rate RATE\n" + " Set the video bit rate, in megabits per second. Default %dMbps.\n" + "--time-limit TIME\n" + " Set the maximum recording time, in seconds. Default / maximum is %d.\n" + "--rotate\n" + " Rotate the output 90 degrees.\n" + "--verbose\n" + " Display interesting information on stdout.\n" + "--help\n" + " Show this message.\n" + "\n" + "Recording continues until Ctrl-C is hit or the time limit is reached.\n" + "\n", + gBitRate / 1000000, gTimeLimitSec + ); +} + +/* + * Parses args and kicks things off. + */ +int main(int argc, char* const argv[]) { + static const struct option longOptions[] = { + { "help", no_argument, NULL, 'h' }, + { "verbose", no_argument, NULL, 'v' }, + { "size", required_argument, NULL, 's' }, + { "bit-rate", required_argument, NULL, 'b' }, + { "time-limit", required_argument, NULL, 't' }, + { "rotate", no_argument, NULL, 'r' }, + { NULL, 0, NULL, 0 } + }; + + while (true) { + int optionIndex = 0; + int ic = getopt_long(argc, argv, "", longOptions, &optionIndex); + if (ic == -1) { + break; + } + + switch (ic) { + case 'h': + usage(); + return 0; + case 'v': + gVerbose = true; + break; + case 's': + if (!parseWidthHeight(optarg, &gVideoWidth, &gVideoHeight)) { + fprintf(stderr, "Invalid size '%s', must be width x height\n", + optarg); + return 2; + } + if (gVideoWidth == 0 || gVideoHeight == 0) { + fprintf(stderr, + "Invalid size %ux%u, width and height may not be zero\n", + gVideoWidth, gVideoHeight); + return 2; + } + gSizeSpecified = true; + break; + case 'b': + gBitRate = atoi(optarg); + if (gBitRate < kMinBitRate || gBitRate > kMaxBitRate) { + fprintf(stderr, + "Bit rate %dbps outside acceptable range [%d,%d]\n", + gBitRate, kMinBitRate, kMaxBitRate); + return 2; + } + break; + case 't': + gTimeLimitSec = atoi(optarg); + if (gTimeLimitSec == 0 || gTimeLimitSec > kMaxTimeLimitSec) { + fprintf(stderr, + "Time limit %ds outside acceptable range [1,%d]\n", + gTimeLimitSec, kMaxTimeLimitSec); + return 2; + } + break; + case 'r': + gRotate = true; + break; + default: + if (ic != '?') { + fprintf(stderr, "getopt_long returned unexpected value 0x%x\n", ic); + } + return 2; + } + } + + if (optind != argc - 1) { + fprintf(stderr, "Must specify output file (see --help).\n"); + return 2; + } + + // MediaMuxer tries to create the file in the constructor, but we don't + // learn about the failure until muxer.start(), which returns a generic + // error code without logging anything. We attempt to create the file + // now for better diagnostics. + const char* fileName = argv[optind]; + int fd = open(fileName, O_CREAT | O_RDWR, 0644); + if (fd < 0) { + fprintf(stderr, "Unable to open '%s': %s\n", fileName, strerror(errno)); + return 1; + } + close(fd); + + status_t err = recordScreen(fileName); + if (err == NO_ERROR) { + // Try to notify the media scanner. Not fatal if this fails. + notifyMediaScanner(fileName); + } + ALOGD(err == NO_ERROR ? "success" : "failed"); + return (int) err; +} diff --git a/cmds/stagefright/Android.mk b/cmds/stagefright/Android.mk index 3844487..561ce02 100644 --- a/cmds/stagefright/Android.mk +++ b/cmds/stagefright/Android.mk @@ -19,7 +19,7 @@ LOCAL_C_INCLUDES:= \ LOCAL_CFLAGS += -Wno-multichar -LOCAL_MODULE_TAGS := debug +LOCAL_MODULE_TAGS := optional LOCAL_MODULE:= stagefright @@ -42,7 +42,7 @@ LOCAL_C_INCLUDES:= \ LOCAL_CFLAGS += -Wno-multichar -LOCAL_MODULE_TAGS := debug +LOCAL_MODULE_TAGS := optional LOCAL_MODULE:= record @@ -65,7 +65,7 @@ LOCAL_C_INCLUDES:= \ LOCAL_CFLAGS += -Wno-multichar -LOCAL_MODULE_TAGS := debug +LOCAL_MODULE_TAGS := optional LOCAL_MODULE:= recordvideo @@ -89,7 +89,7 @@ LOCAL_C_INCLUDES:= \ LOCAL_CFLAGS += -Wno-multichar -LOCAL_MODULE_TAGS := debug +LOCAL_MODULE_TAGS := optional LOCAL_MODULE:= audioloop @@ -112,7 +112,7 @@ LOCAL_C_INCLUDES:= \ LOCAL_CFLAGS += -Wno-multichar -LOCAL_MODULE_TAGS := debug +LOCAL_MODULE_TAGS := optional LOCAL_MODULE:= stream @@ -135,7 +135,7 @@ LOCAL_C_INCLUDES:= \ LOCAL_CFLAGS += -Wno-multichar -LOCAL_MODULE_TAGS := debug +LOCAL_MODULE_TAGS := optional LOCAL_MODULE:= sf2 @@ -159,7 +159,7 @@ LOCAL_C_INCLUDES:= \ LOCAL_CFLAGS += -Wno-multichar -LOCAL_MODULE_TAGS := debug +LOCAL_MODULE_TAGS := optional LOCAL_MODULE:= codec @@ -182,7 +182,7 @@ LOCAL_C_INCLUDES:= \ LOCAL_CFLAGS += -Wno-multichar -LOCAL_MODULE_TAGS := debug +LOCAL_MODULE_TAGS := optional LOCAL_MODULE:= muxer diff --git a/cmds/stagefright/stagefright.cpp b/cmds/stagefright/stagefright.cpp index 115b07c..030bf1b 100644 --- a/cmds/stagefright/stagefright.cpp +++ b/cmds/stagefright/stagefright.cpp @@ -30,8 +30,6 @@ #include <binder/ProcessState.h> #include <media/IMediaPlayerService.h> #include <media/stagefright/foundation/ALooper.h> -#include <media/stagefright/foundation/AMessage.h> -#include "include/LiveSession.h" #include "include/NuCachedSource2.h" #include <media/stagefright/AudioPlayer.h> #include <media/stagefright/DataSource.h> @@ -53,6 +51,7 @@ #include <fcntl.h> +#include <gui/GLConsumer.h> #include <gui/Surface.h> #include <gui/SurfaceComposerClient.h> @@ -618,7 +617,7 @@ static void dumpCodecProfiles(const sp<IOMX>& omx, bool queryDecoders) { MEDIA_MIMETYPE_AUDIO_AMR_NB, MEDIA_MIMETYPE_AUDIO_AMR_WB, MEDIA_MIMETYPE_AUDIO_MPEG, MEDIA_MIMETYPE_AUDIO_G711_MLAW, MEDIA_MIMETYPE_AUDIO_G711_ALAW, MEDIA_MIMETYPE_AUDIO_VORBIS, - MEDIA_MIMETYPE_VIDEO_VPX + MEDIA_MIMETYPE_VIDEO_VP8, MEDIA_MIMETYPE_VIDEO_VP9 }; const char *codecType = queryDecoders? "decoder" : "encoder"; @@ -678,7 +677,6 @@ int main(int argc, char **argv) { gDisplayHistogram = false; sp<ALooper> looper; - sp<LiveSession> liveSession; int res; while ((res = getopt(argc, argv, "han:lm:b:ptsrow:kxSTd:D:")) >= 0) { @@ -940,8 +938,9 @@ int main(int argc, char **argv) { } else { CHECK(useSurfaceTexAlloc); - sp<GLConsumer> texture = new GLConsumer(0 /* tex */); - gSurface = new Surface(texture->getBufferQueue()); + sp<BufferQueue> bq = new BufferQueue(); + sp<GLConsumer> texture = new GLConsumer(bq, 0 /* tex */); + gSurface = new Surface(bq); } CHECK_EQ((status_t)OK, @@ -961,9 +960,7 @@ int main(int argc, char **argv) { sp<DataSource> dataSource = DataSource::CreateFromURI(filename); - if (strncasecmp(filename, "sine:", 5) - && strncasecmp(filename, "httplive://", 11) - && dataSource == NULL) { + if (strncasecmp(filename, "sine:", 5) && dataSource == NULL) { fprintf(stderr, "Unable to create data source.\n"); return 1; } @@ -995,44 +992,21 @@ int main(int argc, char **argv) { mediaSources.push(mediaSource); } } else { - sp<MediaExtractor> extractor; + sp<MediaExtractor> extractor = MediaExtractor::Create(dataSource); - if (!strncasecmp("httplive://", filename, 11)) { - String8 uri("http://"); - uri.append(filename + 11); - - if (looper == NULL) { - looper = new ALooper; - looper->start(); - } - liveSession = new LiveSession(NULL /* notify */); - looper->registerHandler(liveSession); - - liveSession->connect(uri.string()); - dataSource = liveSession->getDataSource(); - - extractor = - MediaExtractor::Create( - dataSource, MEDIA_MIMETYPE_CONTAINER_MPEG2TS); - - syncInfoPresent = false; - } else { - extractor = MediaExtractor::Create(dataSource); - - if (extractor == NULL) { - fprintf(stderr, "could not create extractor.\n"); - return -1; - } + if (extractor == NULL) { + fprintf(stderr, "could not create extractor.\n"); + return -1; + } - sp<MetaData> meta = extractor->getMetaData(); + sp<MetaData> meta = extractor->getMetaData(); - if (meta != NULL) { - const char *mime; - CHECK(meta->findCString(kKeyMIMEType, &mime)); + if (meta != NULL) { + const char *mime; + CHECK(meta->findCString(kKeyMIMEType, &mime)); - if (!strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_MPEG2TS)) { - syncInfoPresent = false; - } + if (!strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_MPEG2TS)) { + syncInfoPresent = false; } } diff --git a/drm/common/IDrmManagerService.cpp b/drm/common/IDrmManagerService.cpp index 91fd91e..db41e0b 100644 --- a/drm/common/IDrmManagerService.cpp +++ b/drm/common/IDrmManagerService.cpp @@ -153,18 +153,6 @@ status_t BpDrmManagerService::setDrmServiceListener( return reply.readInt32(); } -status_t BpDrmManagerService::installDrmEngine(int uniqueId, const String8& drmEngineFile) { - ALOGV("Install DRM Engine"); - Parcel data, reply; - - data.writeInterfaceToken(IDrmManagerService::getInterfaceDescriptor()); - data.writeInt32(uniqueId); - data.writeString8(drmEngineFile); - - remote()->transact(INSTALL_DRM_ENGINE, data, &reply); - return reply.readInt32(); -} - DrmConstraints* BpDrmManagerService::getConstraints( int uniqueId, const String8* path, const int action) { ALOGV("Get Constraints"); @@ -855,19 +843,6 @@ status_t BnDrmManagerService::onTransact( return DRM_NO_ERROR; } - case INSTALL_DRM_ENGINE: - { - ALOGV("BnDrmManagerService::onTransact :INSTALL_DRM_ENGINE"); - CHECK_INTERFACE(IDrmManagerService, data, reply); - - const int uniqueId = data.readInt32(); - const String8 engineFile = data.readString8(); - status_t status = installDrmEngine(uniqueId, engineFile); - - reply->writeInt32(status); - return DRM_NO_ERROR; - } - case GET_CONSTRAINTS_FROM_CONTENT: { ALOGV("BnDrmManagerService::onTransact :GET_CONSTRAINTS_FROM_CONTENT"); diff --git a/drm/drmserver/DrmManager.cpp b/drm/drmserver/DrmManager.cpp index bfaf4bc..dccd23d 100644 --- a/drm/drmserver/DrmManager.cpp +++ b/drm/drmserver/DrmManager.cpp @@ -175,21 +175,6 @@ DrmMetadata* DrmManager::getMetadata(int uniqueId, const String8* path) { return NULL; } -status_t DrmManager::installDrmEngine(int uniqueId, const String8& absolutePath) { - Mutex::Autolock _l(mLock); - mPlugInManager.loadPlugIn(absolutePath); - - IDrmEngine& rDrmEngine = mPlugInManager.getPlugIn(absolutePath); - rDrmEngine.initialize(uniqueId); - rDrmEngine.setOnInfoListener(uniqueId, this); - - DrmSupportInfo* info = rDrmEngine.getSupportInfo(0); - mSupportInfoToPlugInIdMap.add(*info, absolutePath); - delete info; - - return DRM_NO_ERROR; -} - bool DrmManager::canHandle(int uniqueId, const String8& path, const String8& mimeType) { Mutex::Autolock _l(mLock); const String8 plugInId = getSupportedPlugInId(mimeType); diff --git a/drm/drmserver/DrmManagerService.cpp b/drm/drmserver/DrmManagerService.cpp index bbd3b7f..2b71904 100644 --- a/drm/drmserver/DrmManagerService.cpp +++ b/drm/drmserver/DrmManagerService.cpp @@ -87,11 +87,6 @@ status_t DrmManagerService::setDrmServiceListener( return DRM_NO_ERROR; } -status_t DrmManagerService::installDrmEngine(int uniqueId, const String8& drmEngineFile) { - ALOGV("Entering installDrmEngine"); - return mDrmManager->installDrmEngine(uniqueId, drmEngineFile); -} - DrmConstraints* DrmManagerService::getConstraints( int uniqueId, const String8* path, const int action) { ALOGV("Entering getConstraints from content"); diff --git a/drm/libdrmframework/DrmManagerClientImpl.cpp b/drm/libdrmframework/DrmManagerClientImpl.cpp index a970035..ffefd74 100644 --- a/drm/libdrmframework/DrmManagerClientImpl.cpp +++ b/drm/libdrmframework/DrmManagerClientImpl.cpp @@ -86,15 +86,6 @@ status_t DrmManagerClientImpl::setOnInfoListener( (NULL != infoListener.get()) ? this : NULL); } -status_t DrmManagerClientImpl::installDrmEngine( - int uniqueId, const String8& drmEngineFile) { - status_t status = DRM_ERROR_UNKNOWN; - if (EMPTY_STRING != drmEngineFile) { - status = getDrmManagerService()->installDrmEngine(uniqueId, drmEngineFile); - } - return status; -} - DrmConstraints* DrmManagerClientImpl::getConstraints( int uniqueId, const String8* path, const int action) { DrmConstraints *drmConstraints = NULL; diff --git a/drm/libdrmframework/include/DrmManager.h b/drm/libdrmframework/include/DrmManager.h index 8ab693f..e7cdd36 100644 --- a/drm/libdrmframework/include/DrmManager.h +++ b/drm/libdrmframework/include/DrmManager.h @@ -70,8 +70,6 @@ public: status_t setDrmServiceListener( int uniqueId, const sp<IDrmServiceListener>& drmServiceListener); - status_t installDrmEngine(int uniqueId, const String8& drmEngineFile); - DrmConstraints* getConstraints(int uniqueId, const String8* path, const int action); DrmMetadata* getMetadata(int uniqueId, const String8* path); diff --git a/drm/libdrmframework/include/DrmManagerClientImpl.h b/drm/libdrmframework/include/DrmManagerClientImpl.h index 9b4c9ae..3400cb1 100644 --- a/drm/libdrmframework/include/DrmManagerClientImpl.h +++ b/drm/libdrmframework/include/DrmManagerClientImpl.h @@ -410,17 +410,6 @@ public: status_t notify(const DrmInfoEvent& event); private: - /** - * Install new DRM Engine Plug-in at the runtime - * - * @param[in] uniqueId Unique identifier for a session - * @param[in] drmEngine Shared Object(so) File in which DRM Engine defined - * @return status_t - * Returns DRM_NO_ERROR for success, DRM_ERROR_UNKNOWN for failure - */ - status_t installDrmEngine(int uniqueId, const String8& drmEngineFile); - -private: Mutex mLock; sp<DrmManagerClient::OnInfoListener> mOnInfoListener; diff --git a/drm/libdrmframework/include/DrmManagerService.h b/drm/libdrmframework/include/DrmManagerService.h index 0dfdca6..8bc59b4 100644 --- a/drm/libdrmframework/include/DrmManagerService.h +++ b/drm/libdrmframework/include/DrmManagerService.h @@ -57,8 +57,6 @@ public: status_t setDrmServiceListener( int uniqueId, const sp<IDrmServiceListener>& drmServiceListener); - status_t installDrmEngine(int uniqueId, const String8& drmEngineFile); - DrmConstraints* getConstraints(int uniqueId, const String8* path, const int action); DrmMetadata* getMetadata(int uniqueId, const String8* path); diff --git a/drm/libdrmframework/include/IDrmManagerService.h b/drm/libdrmframework/include/IDrmManagerService.h index 5a4d70a..fe55650 100644 --- a/drm/libdrmframework/include/IDrmManagerService.h +++ b/drm/libdrmframework/include/IDrmManagerService.h @@ -93,8 +93,6 @@ public: virtual status_t setDrmServiceListener( int uniqueId, const sp<IDrmServiceListener>& infoListener) = 0; - virtual status_t installDrmEngine(int uniqueId, const String8& drmEngineFile) = 0; - virtual DrmConstraints* getConstraints( int uniqueId, const String8* path, const int action) = 0; @@ -185,8 +183,6 @@ public: virtual status_t setDrmServiceListener( int uniqueId, const sp<IDrmServiceListener>& infoListener); - virtual status_t installDrmEngine(int uniqueId, const String8& drmEngineFile); - virtual DrmConstraints* getConstraints(int uniqueId, const String8* path, const int action); virtual DrmMetadata* getMetadata(int uniqueId, const String8* path); diff --git a/drm/mediadrm/plugins/mock/MockDrmCryptoPlugin.cpp b/drm/mediadrm/plugins/mock/MockDrmCryptoPlugin.cpp index 06fc29d..f2cadf7 100644 --- a/drm/mediadrm/plugins/mock/MockDrmCryptoPlugin.cpp +++ b/drm/mediadrm/plugins/mock/MockDrmCryptoPlugin.cpp @@ -48,6 +48,14 @@ namespace android { return (!memcmp(uuid, mock_uuid, sizeof(uuid))); } + bool MockDrmFactory::isContentTypeSupported(const String8 &mimeType) + { + if (mimeType != "video/mp4") { + return false; + } + return true; + } + status_t MockDrmFactory::createDrmPlugin(const uint8_t uuid[16], DrmPlugin **plugin) { *plugin = new MockDrmPlugin(); diff --git a/drm/mediadrm/plugins/mock/MockDrmCryptoPlugin.h b/drm/mediadrm/plugins/mock/MockDrmCryptoPlugin.h index ca9eac7..2297f9b 100644 --- a/drm/mediadrm/plugins/mock/MockDrmCryptoPlugin.h +++ b/drm/mediadrm/plugins/mock/MockDrmCryptoPlugin.h @@ -32,6 +32,7 @@ namespace android { virtual ~MockDrmFactory() {} bool isCryptoSchemeSupported(const uint8_t uuid[16]); + bool isContentTypeSupported(const String8 &mimeType); status_t createDrmPlugin(const uint8_t uuid[16], DrmPlugin **plugin); }; diff --git a/include/camera/Camera.h b/include/camera/Camera.h index 37626a4..79682b8 100644 --- a/include/camera/Camera.h +++ b/include/camera/Camera.h @@ -51,8 +51,14 @@ struct CameraTraits<Camera> typedef CameraListener TCamListener; typedef ICamera TCamUser; typedef ICameraClient TCamCallbacks; + typedef status_t (ICameraService::*TCamConnectService)(const sp<ICameraClient>&, + int, const String16&, int, + /*out*/ + sp<ICamera>&); + static TCamConnectService fnConnectService; }; + class Camera : public CameraBase<Camera>, public BnCameraClient @@ -75,9 +81,9 @@ public: status_t unlock(); // pass the buffered IGraphicBufferProducer to the camera service - status_t setPreviewTexture(const sp<IGraphicBufferProducer>& bufferProducer); + status_t setPreviewTarget(const sp<IGraphicBufferProducer>& bufferProducer); - // start preview mode, must call setPreviewDisplay first + // start preview mode, must call setPreviewTarget first status_t startPreview(); // stop preview mode @@ -86,7 +92,7 @@ public: // get preview state bool previewEnabled(); - // start recording mode, must call setPreviewDisplay first + // start recording mode, must call setPreviewTarget first status_t startRecording(); // stop recording mode @@ -121,7 +127,15 @@ public: void setListener(const sp<CameraListener>& listener); void setRecordingProxyListener(const sp<ICameraRecordingProxyListener>& listener); + + // Configure preview callbacks to app. Only one of the older + // callbacks or the callback surface can be active at the same time; + // enabling one will disable the other if active. Flags can be + // disabled by calling it with CAMERA_FRAME_CALLBACK_FLAG_NOOP, and + // Target by calling it with a NULL interface. void setPreviewCallbackFlags(int preview_callback_flag); + status_t setPreviewCallbackTarget( + const sp<IGraphicBufferProducer>& callbackProducer); sp<ICameraRecordingProxy> getRecordingProxy(); diff --git a/include/camera/CameraBase.h b/include/camera/CameraBase.h index 9b08c0f..1b93157 100644 --- a/include/camera/CameraBase.h +++ b/include/camera/CameraBase.h @@ -54,9 +54,10 @@ template <typename TCam, typename TCamTraits = CameraTraits<TCam> > class CameraBase : public IBinder::DeathRecipient { public: - typedef typename TCamTraits::TCamListener TCamListener; - typedef typename TCamTraits::TCamUser TCamUser; - typedef typename TCamTraits::TCamCallbacks TCamCallbacks; + typedef typename TCamTraits::TCamListener TCamListener; + typedef typename TCamTraits::TCamUser TCamUser; + typedef typename TCamTraits::TCamCallbacks TCamCallbacks; + typedef typename TCamTraits::TCamConnectService TCamConnectService; static sp<TCam> connect(int cameraId, const String16& clientPackageName, diff --git a/include/camera/CameraMetadata.h b/include/camera/CameraMetadata.h index 8eeb2e7..fe2bd19 100644 --- a/include/camera/CameraMetadata.h +++ b/include/camera/CameraMetadata.h @@ -22,6 +22,7 @@ #include <utils/Vector.h> namespace android { +class Parcel; /** * A convenience wrapper around the C-based camera_metadata_t library. @@ -159,6 +160,12 @@ class CameraMetadata { status_t erase(uint32_t tag); /** + * Swap the underlying camera metadata between this and the other + * metadata object. + */ + void swap(CameraMetadata &other); + + /** * Dump contents into FD for debugging. The verbosity levels are * 0: Tag entry information only, no data values * 1: Level 0 plus at most 16 data values per entry @@ -169,6 +176,31 @@ class CameraMetadata { */ void dump(int fd, int verbosity = 1, int indentation = 0) const; + /** + * Serialization over Binder + */ + + // Metadata object is unchanged when reading from parcel fails. + status_t readFromParcel(Parcel *parcel); + status_t writeToParcel(Parcel *parcel) const; + + /** + * Caller becomes the owner of the new metadata + * 'const Parcel' doesnt prevent us from calling the read functions. + * which is interesting since it changes the internal state + * + * NULL can be returned when no metadata was sent, OR if there was an issue + * unpacking the serialized data (i.e. bad parcel or invalid structure). + */ + static status_t readFromParcel(const Parcel &parcel, + camera_metadata_t** out); + /** + * Caller retains ownership of metadata + * - Write 2 (int32 + blob) args in the current position + */ + static status_t writeToParcel(Parcel &parcel, + const camera_metadata_t* metadata); + private: camera_metadata_t *mBuffer; bool mLocked; diff --git a/include/camera/ICamera.h b/include/camera/ICamera.h index 2236c1f..b025735 100644 --- a/include/camera/ICamera.h +++ b/include/camera/ICamera.h @@ -32,6 +32,9 @@ class Surface; class ICamera: public IInterface { + /** + * Keep up-to-date with ICamera.aidl in frameworks/base + */ public: DECLARE_META_INTERFACE(Camera); @@ -47,14 +50,21 @@ public: virtual status_t unlock() = 0; // pass the buffered IGraphicBufferProducer to the camera service - virtual status_t setPreviewTexture( + virtual status_t setPreviewTarget( const sp<IGraphicBufferProducer>& bufferProducer) = 0; // set the preview callback flag to affect how the received frames from - // preview are handled. + // preview are handled. Enabling preview callback flags disables any active + // preview callback surface set by setPreviewCallbackTarget(). virtual void setPreviewCallbackFlag(int flag) = 0; - - // start preview mode, must call setPreviewDisplay first + // set a buffer interface to use for client-received preview frames instead + // of preview callback buffers. Passing a valid interface here disables any + // active preview callbacks set by setPreviewCallbackFlag(). Passing NULL + // disables the use of the callback target. + virtual status_t setPreviewCallbackTarget( + const sp<IGraphicBufferProducer>& callbackProducer) = 0; + + // start preview mode, must call setPreviewTarget first virtual status_t startPreview() = 0; // stop preview mode diff --git a/include/camera/ICameraClient.h b/include/camera/ICameraClient.h index b30aa7a..1584dba 100644 --- a/include/camera/ICameraClient.h +++ b/include/camera/ICameraClient.h @@ -28,6 +28,9 @@ namespace android { class ICameraClient: public IInterface { + /** + * Keep up-to-date with ICameraClient.aidl in frameworks/base + */ public: DECLARE_META_INTERFACE(CameraClient); diff --git a/include/camera/ICameraService.h b/include/camera/ICameraService.h index aaf6eb3..f342122 100644 --- a/include/camera/ICameraService.h +++ b/include/camera/ICameraService.h @@ -28,17 +28,25 @@ class ICameraClient; class IProCameraUser; class IProCameraCallbacks; class ICameraServiceListener; +class ICameraDeviceUser; +class ICameraDeviceCallbacks; +class CameraMetadata; class ICameraService : public IInterface { public: + /** + * Keep up-to-date with ICameraService.aidl in frameworks/base + */ enum { GET_NUMBER_OF_CAMERAS = IBinder::FIRST_CALL_TRANSACTION, GET_CAMERA_INFO, CONNECT, CONNECT_PRO, + CONNECT_DEVICE, ADD_LISTENER, REMOVE_LISTENER, + GET_CAMERA_CHARACTERISTICS, }; enum { @@ -52,6 +60,9 @@ public: virtual status_t getCameraInfo(int cameraId, struct CameraInfo* cameraInfo) = 0; + virtual status_t getCameraCharacteristics(int cameraId, + CameraMetadata* cameraInfo) = 0; + // Returns 'OK' if operation succeeded // - Errors: ALREADY_EXISTS if the listener was already added virtual status_t addListener(const sp<ICameraServiceListener>& listener) @@ -65,15 +76,27 @@ public: * clientUid == USE_CALLING_UID, then the calling UID is used instead. Only * trusted callers can set a clientUid other than USE_CALLING_UID. */ - virtual sp<ICamera> connect(const sp<ICameraClient>& cameraClient, + virtual status_t connect(const sp<ICameraClient>& cameraClient, + int cameraId, + const String16& clientPackageName, + int clientUid, + /*out*/ + sp<ICamera>& device) = 0; + + virtual status_t connectPro(const sp<IProCameraCallbacks>& cameraCb, int cameraId, const String16& clientPackageName, - int clientUid) = 0; + int clientUid, + /*out*/ + sp<IProCameraUser>& device) = 0; - virtual sp<IProCameraUser> connect(const sp<IProCameraCallbacks>& cameraCb, + virtual status_t connectDevice( + const sp<ICameraDeviceCallbacks>& cameraCb, int cameraId, const String16& clientPackageName, - int clientUid) = 0; + int clientUid, + /*out*/ + sp<ICameraDeviceUser>& device) = 0; }; // ---------------------------------------------------------------------------- diff --git a/include/camera/ICameraServiceListener.h b/include/camera/ICameraServiceListener.h index f2a11c2..0a0e43a 100644 --- a/include/camera/ICameraServiceListener.h +++ b/include/camera/ICameraServiceListener.h @@ -26,6 +26,9 @@ namespace android { class ICameraServiceListener : public IInterface { + /** + * Keep up-to-date with ICameraServiceListener.aidl in frameworks/base + */ public: /** diff --git a/include/camera/IProCameraCallbacks.h b/include/camera/IProCameraCallbacks.h index 563ec17..e8abb89 100644 --- a/include/camera/IProCameraCallbacks.h +++ b/include/camera/IProCameraCallbacks.h @@ -30,6 +30,9 @@ namespace android { class IProCameraCallbacks : public IInterface { + /** + * Keep up-to-date with IProCameraCallbacks.aidl in frameworks/base + */ public: DECLARE_META_INTERFACE(ProCameraCallbacks); @@ -48,7 +51,7 @@ public: /** Missing by design: implementation is client-side in ProCamera.cpp **/ // virtual void onBufferReceived(int streamId, // const CpuConsumer::LockedBufer& buf); - virtual void onResultReceived(int32_t frameId, + virtual void onResultReceived(int32_t requestId, camera_metadata* result) = 0; }; diff --git a/include/camera/IProCameraUser.h b/include/camera/IProCameraUser.h index 45b818c..2ccc4d2 100644 --- a/include/camera/IProCameraUser.h +++ b/include/camera/IProCameraUser.h @@ -34,6 +34,9 @@ class Surface; class IProCameraUser: public IInterface { + /** + * Keep up-to-date with IProCameraUser.aidl in frameworks/base + */ public: DECLARE_META_INTERFACE(ProCameraUser); diff --git a/include/camera/ProCamera.h b/include/camera/ProCamera.h index 3d1652f..83a3028 100644 --- a/include/camera/ProCamera.h +++ b/include/camera/ProCamera.h @@ -25,6 +25,7 @@ #include <camera/IProCameraUser.h> #include <camera/Camera.h> #include <camera/CameraMetadata.h> +#include <camera/ICameraService.h> #include <gui/CpuConsumer.h> #include <gui/Surface.h> @@ -87,8 +88,14 @@ struct CameraTraits<ProCamera> typedef ProCameraListener TCamListener; typedef IProCameraUser TCamUser; typedef IProCameraCallbacks TCamCallbacks; + typedef status_t (ICameraService::*TCamConnectService)(const sp<IProCameraCallbacks>&, + int, const String16&, int, + /*out*/ + sp<IProCameraUser>&); + static TCamConnectService fnConnectService; }; + class ProCamera : public CameraBase<ProCamera>, public BnProCameraCallbacks @@ -245,7 +252,7 @@ protected: virtual void onLockStatusChanged( IProCameraCallbacks::LockStatus newLockStatus); - virtual void onResultReceived(int32_t frameId, + virtual void onResultReceived(int32_t requestId, camera_metadata* result); private: ProCamera(int cameraId); diff --git a/include/camera/camera2/CaptureRequest.h b/include/camera/camera2/CaptureRequest.h new file mode 100644 index 0000000..e56d61f --- /dev/null +++ b/include/camera/camera2/CaptureRequest.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_HARDWARE_PHOTOGRAPHY_CAPTUREREQUEST_H +#define ANDROID_HARDWARE_PHOTOGRAPHY_CAPTUREREQUEST_H + +#include <utils/RefBase.h> +#include <utils/Vector.h> +#include <camera/CameraMetadata.h> + +namespace android { + +class Surface; + +struct CaptureRequest : public virtual RefBase { +public: + + CameraMetadata mMetadata; + Vector<sp<Surface> > mSurfaceList; + + /** + * Keep impl up-to-date with CaptureRequest.java in frameworks/base + */ + status_t readFromParcel(Parcel* parcel); + status_t writeToParcel(Parcel* parcel) const; +}; +}; // namespace android + +#endif diff --git a/include/camera/camera2/ICameraDeviceCallbacks.h b/include/camera/camera2/ICameraDeviceCallbacks.h new file mode 100644 index 0000000..8dac4f2 --- /dev/null +++ b/include/camera/camera2/ICameraDeviceCallbacks.h @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_HARDWARE_PHOTOGRAPHY_CALLBACKS_H +#define ANDROID_HARDWARE_PHOTOGRAPHY_CALLBACKS_H + +#include <utils/RefBase.h> +#include <binder/IInterface.h> +#include <binder/Parcel.h> +#include <binder/IMemory.h> +#include <utils/Timers.h> +#include <system/camera.h> + +namespace android { +class CameraMetadata; + +class ICameraDeviceCallbacks : public IInterface +{ + /** + * Keep up-to-date with ICameraDeviceCallbacks.aidl in frameworks/base + */ +public: + DECLARE_META_INTERFACE(CameraDeviceCallbacks); + + /** + * Error codes for CAMERA_MSG_ERROR + */ + enum CameraErrorCode { + ERROR_CAMERA_DISCONNECTED = 0, + ERROR_CAMERA_DEVICE = 1, + ERROR_CAMERA_SERVICE = 2 + }; + + // One way + virtual void onDeviceError(CameraErrorCode errorCode) = 0; + + // One way + virtual void onDeviceIdle() = 0; + + // One way + virtual void onCaptureStarted(int32_t requestId, + int64_t timestamp) = 0; + + // One way + virtual void onResultReceived(int32_t requestId, + const CameraMetadata& result) = 0; +}; + +// ---------------------------------------------------------------------------- + +class BnCameraDeviceCallbacks : public BnInterface<ICameraDeviceCallbacks> +{ +public: + virtual status_t onTransact( uint32_t code, + const Parcel& data, + Parcel* reply, + uint32_t flags = 0); +}; + +}; // namespace android + +#endif diff --git a/include/camera/camera2/ICameraDeviceUser.h b/include/camera/camera2/ICameraDeviceUser.h new file mode 100644 index 0000000..f71f302 --- /dev/null +++ b/include/camera/camera2/ICameraDeviceUser.h @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_HARDWARE_PHOTOGRAPHY_ICAMERADEVICEUSER_H +#define ANDROID_HARDWARE_PHOTOGRAPHY_ICAMERADEVICEUSER_H + +#include <binder/IInterface.h> +#include <binder/Parcel.h> + +struct camera_metadata; + +namespace android { + +class ICameraDeviceUserClient; +class IGraphicBufferProducer; +class Surface; +class CaptureRequest; +class CameraMetadata; + +class ICameraDeviceUser : public IInterface +{ + /** + * Keep up-to-date with ICameraDeviceUser.aidl in frameworks/base + */ +public: + DECLARE_META_INTERFACE(CameraDeviceUser); + + virtual void disconnect() = 0; + + /** + * Request Handling + **/ + + virtual int submitRequest(sp<CaptureRequest> request, + bool streaming = false) = 0; + virtual status_t cancelRequest(int requestId) = 0; + + virtual status_t deleteStream(int streamId) = 0; + virtual status_t createStream( + int width, int height, int format, + const sp<IGraphicBufferProducer>& bufferProducer) = 0; + + // Create a request object from a template. + virtual status_t createDefaultRequest(int templateId, + /*out*/ + CameraMetadata* request) = 0; + // Get static camera metadata + virtual status_t getCameraInfo(/*out*/ + CameraMetadata* info) = 0; + + // Wait until all the submitted requests have finished processing + virtual status_t waitUntilIdle() = 0; + + // Flush all pending and in-progress work as quickly as possible. + virtual status_t flush() = 0; +}; + +// ---------------------------------------------------------------------------- + +class BnCameraDeviceUser: public BnInterface<ICameraDeviceUser> +{ +public: + virtual status_t onTransact( uint32_t code, + const Parcel& data, + Parcel* reply, + uint32_t flags = 0); +}; + +}; // namespace android + +#endif diff --git a/include/cpustats/CentralTendencyStatistics.h b/include/cpustats/CentralTendencyStatistics.h new file mode 100644 index 0000000..21b6981 --- /dev/null +++ b/include/cpustats/CentralTendencyStatistics.h @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _CENTRAL_TENDENCY_STATISTICS_H +#define _CENTRAL_TENDENCY_STATISTICS_H + +#include <math.h> + +// Not multithread safe +class CentralTendencyStatistics { + +public: + + CentralTendencyStatistics() : + mMean(NAN), mMedian(NAN), mMinimum(INFINITY), mMaximum(-INFINITY), mN(0), mM2(0), + mVariance(NAN), mVarianceKnownForN(0), mStddev(NAN), mStddevKnownForN(0) { } + + ~CentralTendencyStatistics() { } + + // add x to the set of samples + void sample(double x); + + // return the arithmetic mean of all samples so far + double mean() const { return mMean; } + + // return the minimum of all samples so far + double minimum() const { return mMinimum; } + + // return the maximum of all samples so far + double maximum() const { return mMaximum; } + + // return the variance of all samples so far + double variance() const; + + // return the standard deviation of all samples so far + double stddev() const; + + // return the number of samples added so far + unsigned n() const { return mN; } + + // reset the set of samples to be empty + void reset(); + +private: + double mMean; + double mMedian; + double mMinimum; + double mMaximum; + unsigned mN; // number of samples so far + double mM2; + + // cached variance, and n at time of caching + mutable double mVariance; + mutable unsigned mVarianceKnownForN; + + // cached standard deviation, and n at time of caching + mutable double mStddev; + mutable unsigned mStddevKnownForN; + +}; + +#endif // _CENTRAL_TENDENCY_STATISTICS_H diff --git a/include/cpustats/README.txt b/include/cpustats/README.txt new file mode 100644 index 0000000..14439f0 --- /dev/null +++ b/include/cpustats/README.txt @@ -0,0 +1,6 @@ +This is a static library of CPU usage statistics, originally written +for audio but most are not actually specific to audio. + +Requirements to be here: + * should be related to CPU usage statistics + * should be portable to host; avoid Android OS dependencies without a conditional diff --git a/include/cpustats/ThreadCpuUsage.h b/include/cpustats/ThreadCpuUsage.h new file mode 100644 index 0000000..9756844 --- /dev/null +++ b/include/cpustats/ThreadCpuUsage.h @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _THREAD_CPU_USAGE_H +#define _THREAD_CPU_USAGE_H + +#include <fcntl.h> +#include <pthread.h> + +namespace android { + +// Track CPU usage for the current thread. +// Units are in per-thread CPU ns, as reported by +// clock_gettime(CLOCK_THREAD_CPUTIME_ID). Simple usage: for cyclic +// threads where you want to measure the execution time of the whole +// cycle, just call sampleAndEnable() at the start of each cycle. +// For acyclic threads, or for cyclic threads where you want to measure/track +// only part of each cycle, call enable(), disable(), and/or setEnabled() +// to demarcate the region(s) of interest, and then call sample() periodically. +// This class is not thread-safe for concurrent calls from multiple threads; +// the methods of this class may only be called by the current thread +// which constructed the object. + +class ThreadCpuUsage +{ + +public: + ThreadCpuUsage() : + mIsEnabled(false), + mWasEverEnabled(false), + mAccumulator(0), + // mPreviousTs + // mMonotonicTs + mMonotonicKnown(false) + { + (void) pthread_once(&sOnceControl, &init); + for (int i = 0; i < sKernelMax; ++i) { + mCurrentkHz[i] = (uint32_t) ~0; // unknown + } + } + + ~ThreadCpuUsage() { } + + // Return whether currently tracking CPU usage by current thread + bool isEnabled() const { return mIsEnabled; } + + // Enable tracking of CPU usage by current thread; + // any CPU used from this point forward will be tracked. + // Returns the previous enabled status. + bool enable() { return setEnabled(true); } + + // Disable tracking of CPU usage by current thread; + // any CPU used from this point forward will be ignored. + // Returns the previous enabled status. + bool disable() { return setEnabled(false); } + + // Set the enabled status and return the previous enabled status. + // This method is intended to be used for safe nested enable/disabling. + bool setEnabled(bool isEnabled); + + // Add a sample point, and also enable tracking if needed. + // If tracking has never been enabled, then this call enables tracking but + // does _not_ add a sample -- it is not possible to add a sample the + // first time because there is no previous point to subtract from. + // Otherwise, if tracking is enabled, + // then adds a sample for tracked CPU ns since the previous + // sample, or since the first call to sampleAndEnable(), enable(), or + // setEnabled(true). If there was a previous sample but tracking is + // now disabled, then adds a sample for the tracked CPU ns accumulated + // up until the most recent disable(), resets this accumulator, and then + // enables tracking. Calling this method rather than enable() followed + // by sample() avoids a race condition for the first sample. + // Returns true if the sample 'ns' is valid, or false if invalid. + // Note that 'ns' is an output parameter passed by reference. + // The caller does not need to initialize this variable. + // The units are CPU nanoseconds consumed by current thread. + bool sampleAndEnable(double& ns); + + // Add a sample point, but do not + // change the tracking enabled status. If tracking has either never been + // enabled, or has never been enabled since the last sample, then log a warning + // and don't add sample. Otherwise, adds a sample for tracked CPU ns since + // the previous sample or since the first call to sampleAndEnable(), + // enable(), or setEnabled(true) if no previous sample. + // Returns true if the sample is valid, or false if invalid. + // Note that 'ns' is an output parameter passed by reference. + // The caller does not need to initialize this variable. + // The units are CPU nanoseconds consumed by current thread. + bool sample(double& ns); + + // Return the elapsed delta wall clock ns since initial enable or reset, + // as reported by clock_gettime(CLOCK_MONOTONIC). + long long elapsed() const; + + // Reset elapsed wall clock. Has no effect on tracking or accumulator. + void resetElapsed(); + + // Return current clock frequency for specified CPU, in kHz. + // You can get your CPU number using sched_getcpu(2). Note that, unless CPU affinity + // has been configured appropriately, the CPU number can change. + // Also note that, unless the CPU governor has been configured appropriately, + // the CPU frequency can change. And even if the CPU frequency is locked down + // to a particular value, that the frequency might still be adjusted + // to prevent thermal overload. Therefore you should poll for your thread's + // current CPU number and clock frequency periodically. + uint32_t getCpukHz(int cpuNum); + +private: + bool mIsEnabled; // whether tracking is currently enabled + bool mWasEverEnabled; // whether tracking was ever enabled + long long mAccumulator; // accumulated thread CPU time since last sample, in ns + struct timespec mPreviousTs; // most recent thread CPU time, valid only if mIsEnabled is true + struct timespec mMonotonicTs; // most recent monotonic time + bool mMonotonicKnown; // whether mMonotonicTs has been set + + static const int MAX_CPU = 8; + static int sScalingFds[MAX_CPU];// file descriptor per CPU for reading scaling_cur_freq + uint32_t mCurrentkHz[MAX_CPU]; // current CPU frequency in kHz, not static to avoid a race + static pthread_once_t sOnceControl; + static int sKernelMax; // like MAX_CPU, but determined at runtime == cpu/kernel_max + 1 + static void init(); // called once at first ThreadCpuUsage construction + static pthread_mutex_t sMutex; // protects sScalingFds[] after initialization +}; + +} // namespace android + +#endif // _THREAD_CPU_USAGE_H diff --git a/include/media/AudioBufferProvider.h b/include/media/AudioBufferProvider.h index 43e4de7..ef392f0 100644 --- a/include/media/AudioBufferProvider.h +++ b/include/media/AudioBufferProvider.h @@ -26,6 +26,8 @@ class AudioBufferProvider { public: + // FIXME merge with AudioTrackShared::Buffer, AudioTrack::Buffer, and AudioRecord::Buffer + // and rename getNextBuffer() to obtainBuffer() struct Buffer { Buffer() : raw(NULL), frameCount(0) { } union { @@ -44,6 +46,19 @@ public: // pts is the local time when the next sample yielded by getNextBuffer // will be rendered. // Pass kInvalidPTS if the PTS is unknown or not applicable. + // On entry: + // buffer != NULL + // buffer->raw unused + // buffer->frameCount maximum number of desired frames + // On successful return: + // status NO_ERROR + // buffer->raw non-NULL pointer to buffer->frameCount contiguous available frames + // buffer->frameCount number of contiguous available frames at buffer->raw, + // 0 < buffer->frameCount <= entry value + // On error return: + // status != NO_ERROR + // buffer->raw NULL + // buffer->frameCount 0 virtual status_t getNextBuffer(Buffer* buffer, int64_t pts = kInvalidPTS) = 0; virtual void releaseBuffer(Buffer* buffer) = 0; diff --git a/include/media/AudioRecord.h b/include/media/AudioRecord.h index 38c6548..052064d 100644 --- a/include/media/AudioRecord.h +++ b/include/media/AudioRecord.h @@ -14,31 +14,27 @@ * limitations under the License. */ -#ifndef AUDIORECORD_H_ -#define AUDIORECORD_H_ +#ifndef ANDROID_AUDIORECORD_H +#define ANDROID_AUDIORECORD_H -#include <binder/IMemory.h> #include <cutils/sched_policy.h> #include <media/AudioSystem.h> #include <media/IAudioRecord.h> -#include <system/audio.h> -#include <utils/RefBase.h> -#include <utils/Errors.h> #include <utils/threads.h> namespace android { +// ---------------------------------------------------------------------------- + class audio_track_cblk_t; class AudioRecordClientProxy; // ---------------------------------------------------------------------------- -class AudioRecord : virtual public RefBase +class AudioRecord : public RefBase { public: - static const int DEFAULT_SAMPLE_RATE = 8000; - /* Events used by AudioRecord callback function (callback_t). * Keep in sync with frameworks/base/media/java/android/media/AudioRecord.java NATIVE_EVENT_*. */ @@ -49,6 +45,8 @@ public: // (See setMarkerPosition()). EVENT_NEW_POS = 3, // Record head is at a new position // (See setPositionUpdatePeriod()). + EVENT_NEW_IAUDIORECORD = 4, // IAudioRecord was re-created, either due to re-routing and + // voluntary invalidation by mediaserver, or mediaserver crash. }; /* Client should declare Buffer on the stack and pass address to obtainBuffer() @@ -58,11 +56,17 @@ public: class Buffer { public: + // FIXME use m prefix size_t frameCount; // number of sample frames corresponding to size; // on input it is the number of frames available, // on output is the number of frames actually drained + // (currently ignored, but will make the primary field in future) + + size_t size; // input/output in bytes == frameCount * frameSize + // FIXME this is redundant with respect to frameCount, + // and TRANSFER_OBTAIN mode is broken for 8-bit data + // since we don't define the frame format - size_t size; // total size in bytes == frameCount * frameSize union { void* raw; short* i16; // signed 16-bit @@ -84,6 +88,7 @@ public: * - EVENT_OVERRUN: unused. * - EVENT_MARKER: pointer to const uint32_t containing the marker position in frames. * - EVENT_NEW_POS: pointer to const uint32_t containing the new position in frames. + * - EVENT_NEW_IAUDIORECORD: unused. */ typedef void (*callback_t)(int event, void* user, void *info); @@ -101,94 +106,112 @@ public: audio_format_t format, audio_channel_mask_t channelMask); + /* How data is transferred from AudioRecord + */ + enum transfer_type { + TRANSFER_DEFAULT, // not specified explicitly; determine from other parameters + TRANSFER_CALLBACK, // callback EVENT_MORE_DATA + TRANSFER_OBTAIN, // FIXME deprecated: call obtainBuffer() and releaseBuffer() + TRANSFER_SYNC, // synchronous read() + }; + /* Constructs an uninitialized AudioRecord. No connection with - * AudioFlinger takes place. + * AudioFlinger takes place. Use set() after this. */ AudioRecord(); /* Creates an AudioRecord object and registers it with AudioFlinger. * Once created, the track needs to be started before it can be used. - * Unspecified values are set to the audio hardware's current - * values. + * Unspecified values are set to appropriate default values. * * Parameters: * - * inputSource: Select the audio input to record to (e.g. AUDIO_SOURCE_DEFAULT). - * sampleRate: Track sampling rate in Hz. + * inputSource: Select the audio input to record from (e.g. AUDIO_SOURCE_DEFAULT). + * sampleRate: Data sink sampling rate in Hz. * format: Audio format (e.g AUDIO_FORMAT_PCM_16_BIT for signed * 16 bits per sample). - * channelMask: Channel mask. + * channelMask: Channel mask, such that audio_is_input_channel(channelMask) is true. * frameCount: Minimum size of track PCM buffer in frames. This defines the * application's contribution to the * latency of the track. The actual size selected by the AudioRecord could * be larger if the requested size is not compatible with current audio HAL * latency. Zero means to use a default value. * cbf: Callback function. If not null, this function is called periodically - * to consume new PCM data. + * to consume new PCM data and inform of marker, position updates, etc. * user: Context for use by the callback receiver. * notificationFrames: The callback function is called each time notificationFrames PCM * frames are ready in record track output buffer. * sessionId: Not yet supported. + * transferType: How data is transferred from AudioRecord. + * flags: See comments on audio_input_flags_t in <system/audio.h> + * threadCanCallJava: Not present in parameter list, and so is fixed at false. */ AudioRecord(audio_source_t inputSource, - uint32_t sampleRate = 0, - audio_format_t format = AUDIO_FORMAT_DEFAULT, - audio_channel_mask_t channelMask = AUDIO_CHANNEL_IN_MONO, + uint32_t sampleRate, + audio_format_t format, + audio_channel_mask_t channelMask, int frameCount = 0, callback_t cbf = NULL, void* user = NULL, int notificationFrames = 0, - int sessionId = 0); - + int sessionId = 0, + transfer_type transferType = TRANSFER_DEFAULT, + audio_input_flags_t flags = AUDIO_INPUT_FLAG_NONE); /* Terminates the AudioRecord and unregisters it from AudioFlinger. * Also destroys all resources associated with the AudioRecord. */ - ~AudioRecord(); - +protected: + virtual ~AudioRecord(); +public: - /* Initialize an uninitialized AudioRecord. + /* Initialize an AudioRecord that was created using the AudioRecord() constructor. + * Don't call set() more than once, or after an AudioRecord() constructor that takes parameters. * Returned status (from utils/Errors.h) can be: * - NO_ERROR: successful intialization - * - INVALID_OPERATION: AudioRecord is already intitialized or record device is already in use + * - INVALID_OPERATION: AudioRecord is already initialized or record device is already in use * - BAD_VALUE: invalid parameter (channels, format, sampleRate...) * - NO_INIT: audio server or audio hardware not initialized * - PERMISSION_DENIED: recording is not allowed for the requesting process + * + * Parameters not listed in the AudioRecord constructors above: + * + * threadCanCallJava: Whether callbacks are made from an attached thread and thus can call JNI. */ - status_t set(audio_source_t inputSource = AUDIO_SOURCE_DEFAULT, - uint32_t sampleRate = 0, - audio_format_t format = AUDIO_FORMAT_DEFAULT, - audio_channel_mask_t channelMask = AUDIO_CHANNEL_IN_MONO, + status_t set(audio_source_t inputSource, + uint32_t sampleRate, + audio_format_t format, + audio_channel_mask_t channelMask, int frameCount = 0, callback_t cbf = NULL, void* user = NULL, int notificationFrames = 0, bool threadCanCallJava = false, - int sessionId = 0); - + int sessionId = 0, + transfer_type transferType = TRANSFER_DEFAULT, + audio_input_flags_t flags = AUDIO_INPUT_FLAG_NONE); /* Result of constructing the AudioRecord. This must be checked * before using any AudioRecord API (except for set()), because using * an uninitialized AudioRecord produces undefined results. * See set() method above for possible return codes. */ - status_t initCheck() const; + status_t initCheck() const { return mStatus; } /* Returns this track's estimated latency in milliseconds. * This includes the latency due to AudioRecord buffer size, * and audio hardware driver. */ - uint32_t latency() const; + uint32_t latency() const { return mLatency; } /* getters, see constructor and set() */ - audio_format_t format() const; - uint32_t channelCount() const; - size_t frameCount() const; - size_t frameSize() const { return mFrameSize; } - audio_source_t inputSource() const; - + audio_format_t format() const { return mFormat; } + uint32_t channelCount() const { return mChannelCount; } + size_t frameCount() const { return mFrameCount; } + size_t frameSize() const { return mFrameSize; } + audio_source_t inputSource() const { return mInputSource; } /* After it's created the track is not active. Call start() to * make it active. If set, the callback will start being called. @@ -198,26 +221,29 @@ public: status_t start(AudioSystem::sync_event_t event = AudioSystem::SYNC_EVENT_NONE, int triggerSession = 0); - /* Stop a track. If set, the callback will cease being called and - * obtainBuffer returns STOPPED. Note that obtainBuffer() still works - * and will drain buffers until the pool is exhausted. + /* Stop a track. If set, the callback will cease being called. Note that obtainBuffer() still + * works and will drain buffers until the pool is exhausted, and then will return WOULD_BLOCK. */ void stop(); bool stopped() const; - /* Get sample rate for this record track in Hz. + /* Return the sink sample rate for this record track in Hz. + * Unlike AudioTrack, the sample rate is const after initialization, so doesn't need a lock. */ - uint32_t getSampleRate() const; + uint32_t getSampleRate() const { return mSampleRate; } /* Sets marker position. When record reaches the number of frames specified, * a callback with event type EVENT_MARKER is called. Calling setMarkerPosition * with marker == 0 cancels marker notification callback. + * To set a marker at a position which would compute as 0, + * a workaround is to the set the marker at a nearby position such as ~0 or 1. * If the AudioRecord has been opened with no callback function associated, * the operation will fail. * * Parameters: * - * marker: marker position expressed in frames. + * marker: marker position expressed in wrapping (overflow) frame units, + * like the return value of getPosition(). * * Returned status (from utils/Errors.h) can be: * - NO_ERROR: successful operation @@ -226,13 +252,13 @@ public: status_t setMarkerPosition(uint32_t marker); status_t getMarkerPosition(uint32_t *marker) const; - /* Sets position update period. Every time the number of frames specified has been recorded, * a callback with event type EVENT_NEW_POS is called. * Calling setPositionUpdatePeriod with updatePeriod == 0 cancels new position notification * callback. * If the AudioRecord has been opened with no callback function associated, * the operation will fail. + * Extremely small values may be rounded up to a value the implementation can support. * * Parameters: * @@ -245,13 +271,13 @@ public: status_t setPositionUpdatePeriod(uint32_t updatePeriod); status_t getPositionUpdatePeriod(uint32_t *updatePeriod) const; - - /* Gets record head position. The position is the total number of frames - * recorded since record start. + /* Return the total number of frames recorded since recording started. + * The counter will wrap (overflow) periodically, e.g. every ~27 hours at 44.1 kHz. + * It is reset to zero by stop(). * * Parameters: * - * position: Address where to return record head position within AudioRecord buffer. + * position: Address where to return record head position. * * Returned status (from utils/Errors.h) can be: * - NO_ERROR: successful operation @@ -276,38 +302,74 @@ public: * * Returned value: * AudioRecord session ID. + * + * No lock needed because session ID doesn't change after first set(). */ - int getSessionId() const; - - /* Obtains a buffer of "frameCount" frames. The buffer must be - * drained entirely, and then released with releaseBuffer(). - * If the track is stopped, obtainBuffer() returns - * STOPPED instead of NO_ERROR as long as there are buffers available, - * at which point NO_MORE_BUFFERS is returned. + int getSessionId() const { return mSessionId; } + + /* Obtains a buffer of up to "audioBuffer->frameCount" full frames. + * After draining these frames of data, the caller should release them with releaseBuffer(). + * If the track buffer is not empty, obtainBuffer() returns as many contiguous + * full frames as are available immediately. + * If the track buffer is empty and track is stopped, obtainBuffer() returns WOULD_BLOCK + * regardless of the value of waitCount. + * If the track buffer is empty and track is not stopped, obtainBuffer() blocks with a + * maximum timeout based on waitCount; see chart below. * Buffers will be returned until the pool * is exhausted, at which point obtainBuffer() will either block - * or return WOULD_BLOCK depending on the value of the "blocking" + * or return WOULD_BLOCK depending on the value of the "waitCount" * parameter. * + * obtainBuffer() and releaseBuffer() are deprecated for direct use by applications, + * which should use read() or callback EVENT_MORE_DATA instead. + * * Interpretation of waitCount: * +n limits wait time to n * WAIT_PERIOD_MS, * -1 causes an (almost) infinite wait time, * 0 non-blocking. + * + * Buffer fields + * On entry: + * frameCount number of frames requested + * After error return: + * frameCount 0 + * size 0 + * raw undefined + * After successful return: + * frameCount actual number of frames available, <= number requested + * size actual number of bytes available + * raw pointer to the buffer */ - enum { - NO_MORE_BUFFERS = 0x80000001, // same name in AudioFlinger.h, ok to be different value - STOPPED = 1 - }; + /* FIXME Deprecated public API for TRANSFER_OBTAIN mode */ + status_t obtainBuffer(Buffer* audioBuffer, int32_t waitCount) + __attribute__((__deprecated__)); - status_t obtainBuffer(Buffer* audioBuffer, int32_t waitCount); +private: + /* If nonContig is non-NULL, it is an output parameter that will be set to the number of + * additional non-contiguous frames that are available immediately. + * FIXME We could pass an array of Buffers instead of only one Buffer to obtainBuffer(), + * in case the requested amount of frames is in two or more non-contiguous regions. + * FIXME requested and elapsed are both relative times. Consider changing to absolute time. + */ + status_t obtainBuffer(Buffer* audioBuffer, const struct timespec *requested, + struct timespec *elapsed = NULL, size_t *nonContig = NULL); +public: - /* Release an emptied buffer of "frameCount" frames for AudioFlinger to re-fill. */ + /* Release an emptied buffer of "audioBuffer->frameCount" frames for AudioFlinger to re-fill. */ + // FIXME make private when obtainBuffer() for TRANSFER_OBTAIN is removed void releaseBuffer(Buffer* audioBuffer); - /* As a convenience we provide a read() interface to the audio buffer. - * This is implemented on top of obtainBuffer/releaseBuffer. + * Input parameter 'size' is in byte units. + * This is implemented on top of obtainBuffer/releaseBuffer. For best + * performance use callbacks. Returns actual number of bytes read >= 0, + * or one of the following negative status codes: + * INVALID_OPERATION AudioRecord is configured for streaming mode + * BAD_VALUE size is invalid + * WOULD_BLOCK when obtainBuffer() returns same, or + * AudioRecord was stopped during the read + * or any other error code returned by IAudioRecord::start() or restoreRecord_l(). */ ssize_t read(void* buffer, size_t size); @@ -338,66 +400,113 @@ private: void resume(); // allow thread to execute, if not requested to exit private: + void pauseInternal(nsecs_t ns = 0LL); + // like pause(), but only used internally within thread + friend class AudioRecord; virtual bool threadLoop(); - AudioRecord& mReceiver; + AudioRecord& mReceiver; virtual ~AudioRecordThread(); Mutex mMyLock; // Thread::mLock is private Condition mMyCond; // Thread::mThreadExitedCondition is private - bool mPaused; // whether thread is currently paused + bool mPaused; // whether thread is requested to pause at next loop entry + bool mPausedInt; // whether thread internally requests pause + nsecs_t mPausedNs; // if mPausedInt then associated timeout, otherwise ignored }; // body of AudioRecordThread::threadLoop() - bool processAudioBuffer(const sp<AudioRecordThread>& thread); + // returns the maximum amount of time before we would like to run again, where: + // 0 immediately + // > 0 no later than this many nanoseconds from now + // NS_WHENEVER still active but no particular deadline + // NS_INACTIVE inactive so don't run again until re-started + // NS_NEVER never again + static const nsecs_t NS_WHENEVER = -1, NS_INACTIVE = -2, NS_NEVER = -3; + nsecs_t processAudioBuffer(const sp<AudioRecordThread>& thread); + + // caller must hold lock on mLock for all _l methods + status_t openRecord_l(size_t epoch); - status_t openRecord_l(uint32_t sampleRate, - audio_format_t format, - size_t frameCount, - audio_io_handle_t input); - audio_io_handle_t getInput_l(); - status_t restoreRecord_l(audio_track_cblk_t*& cblk); + // FIXME enum is faster than strcmp() for parameter 'from' + status_t restoreRecord_l(const char *from); sp<AudioRecordThread> mAudioRecordThread; mutable Mutex mLock; - bool mActive; // protected by mLock + // Current client state: false = stopped, true = active. Protected by mLock. If more states + // are added, consider changing this to enum State { ... } mState as in AudioTrack. + bool mActive; // for client callback handler callback_t mCbf; // callback handler for events, or NULL void* mUserData; // for notification APIs - uint32_t mNotificationFrames; - uint32_t mRemainingFrames; - uint32_t mMarkerPosition; // in frames + uint32_t mNotificationFramesReq; // requested number of frames between each + // notification callback + uint32_t mNotificationFramesAct; // actual number of frames between each + // notification callback + bool mRefreshRemaining; // processAudioBuffer() should refresh next 2 + + // These are private to processAudioBuffer(), and are not protected by a lock + uint32_t mRemainingFrames; // number of frames to request in obtainBuffer() + bool mRetryOnPartialBuffer; // sleep and retry after partial obtainBuffer() + int mObservedSequence; // last observed value of mSequence + + uint32_t mMarkerPosition; // in wrapping (overflow) frame units bool mMarkerReached; uint32_t mNewPosition; // in frames - uint32_t mUpdatePeriod; // in ms + uint32_t mUpdatePeriod; // in frames, zero means no EVENT_NEW_POS + + status_t mStatus; // constant after constructor or set() uint32_t mSampleRate; size_t mFrameCount; audio_format_t mFormat; - uint8_t mChannelCount; + uint32_t mChannelCount; size_t mFrameSize; // app-level frame size == AudioFlinger frame size audio_source_t mInputSource; - status_t mStatus; - uint32_t mLatency; + uint32_t mLatency; // in ms audio_channel_mask_t mChannelMask; - audio_io_handle_t mInput; // returned by AudioSystem::getInput() + audio_input_flags_t mFlags; int mSessionId; + transfer_type mTransfer; + + audio_io_handle_t mInput; // returned by AudioSystem::getInput() // may be changed if IAudioRecord object is re-created sp<IAudioRecord> mAudioRecord; sp<IMemory> mCblkMemory; - audio_track_cblk_t* mCblk; - void* mBuffers; // starting address of buffers in shared memory + audio_track_cblk_t* mCblk; // re-load after mLock.unlock() - int mPreviousPriority; // before start() + int mPreviousPriority; // before start() SchedPolicy mPreviousSchedulingGroup; - AudioRecordClientProxy* mProxy; + bool mAwaitBoost; // thread should wait for priority boost before running + + // The proxy should only be referenced while a lock is held because the proxy isn't + // multi-thread safe. + // An exception is that a blocking ClientProxy::obtainBuffer() may be called without a lock, + // provided that the caller also holds an extra reference to the proxy and shared memory to keep + // them around in case they are replaced during the obtainBuffer(). + sp<AudioRecordClientProxy> mProxy; + + bool mInOverrun; // whether recorder is currently in overrun state + +private: + class DeathNotifier : public IBinder::DeathRecipient { + public: + DeathNotifier(AudioRecord* audioRecord) : mAudioRecord(audioRecord) { } + protected: + virtual void binderDied(const wp<IBinder>& who); + private: + const wp<AudioRecord> mAudioRecord; + }; + + sp<DeathNotifier> mDeathNotifier; + uint32_t mSequence; // incremented for each new IAudioRecord attempt }; }; // namespace android -#endif /*AUDIORECORD_H_*/ +#endif // ANDROID_AUDIORECORD_H diff --git a/include/media/AudioSystem.h b/include/media/AudioSystem.h index b11c812..225ef76 100644 --- a/include/media/AudioSystem.h +++ b/include/media/AudioSystem.h @@ -17,20 +17,18 @@ #ifndef ANDROID_AUDIOSYSTEM_H_ #define ANDROID_AUDIOSYSTEM_H_ -#include <utils/RefBase.h> -#include <utils/threads.h> -#include <media/IAudioFlinger.h> - +#include <hardware/audio_effect.h> +#include <media/IAudioFlingerClient.h> #include <system/audio.h> #include <system/audio_policy.h> - -/* XXX: Should be include by all the users instead */ -#include <media/AudioParameter.h> +#include <utils/Errors.h> +#include <utils/Mutex.h> namespace android { typedef void (*audio_error_callback)(status_t err); +class IAudioFlinger; class IAudioPolicyService; class String8; @@ -128,8 +126,10 @@ public: // - BAD_VALUE: invalid parameter // NOTE: this feature is not supported on all hardware platforms and it is // necessary to check returned status before using the returned values. - static status_t getRenderPosition(size_t *halFrames, size_t *dspFrames, - audio_stream_type_t stream = AUDIO_STREAM_DEFAULT); + static status_t getRenderPosition(audio_io_handle_t output, + size_t *halFrames, + size_t *dspFrames, + audio_stream_type_t stream = AUDIO_STREAM_DEFAULT); // return the number of input frames lost by HAL implementation, or 0 if the handle is invalid static size_t getInputFramesLost(audio_io_handle_t ioHandle); @@ -155,11 +155,11 @@ public: class OutputDescriptor { public: OutputDescriptor() - : samplingRate(0), format(AUDIO_FORMAT_DEFAULT), channels(0), frameCount(0), latency(0) {} + : samplingRate(0), format(AUDIO_FORMAT_DEFAULT), channelMask(0), frameCount(0), latency(0) {} uint32_t samplingRate; - int32_t format; - int32_t channels; + audio_format_t format; + audio_channel_mask_t channelMask; size_t frameCount; uint32_t latency; }; @@ -197,7 +197,8 @@ public: uint32_t samplingRate = 0, audio_format_t format = AUDIO_FORMAT_DEFAULT, audio_channel_mask_t channelMask = AUDIO_CHANNEL_OUT_STEREO, - audio_output_flags_t flags = AUDIO_OUTPUT_FLAG_NONE); + audio_output_flags_t flags = AUDIO_OUTPUT_FLAG_NONE, + const audio_offload_info_t *offloadInfo = NULL); static status_t startOutput(audio_io_handle_t output, audio_stream_type_t stream, int session = 0); @@ -245,6 +246,15 @@ public: static uint32_t getPrimaryOutputSamplingRate(); static size_t getPrimaryOutputFrameCount(); + static status_t setLowRamDevice(bool isLowRamDevice); + + // Check if hw offload is possible for given format, stream type, sample rate, + // bit rate, duration, video and streaming or offload property is enabled + static bool isOffloadSupported(const audio_offload_info_t& info); + + // check presence of audio flinger service. + // returns NO_ERROR if binding to service succeeds, DEAD_OBJECT otherwise + static status_t checkAudioFlinger(); // ---------------------------------------------------------------------------- private: diff --git a/include/media/AudioTimestamp.h b/include/media/AudioTimestamp.h new file mode 100644 index 0000000..c29c7e5 --- /dev/null +++ b/include/media/AudioTimestamp.h @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_AUDIO_TIMESTAMP_H +#define ANDROID_AUDIO_TIMESTAMP_H + +#include <time.h> + +class AudioTimestamp { +public: + AudioTimestamp() : mPosition(0) { + mTime.tv_sec = 0; + mTime.tv_nsec = 0; + } + // FIXME change type to match android.media.AudioTrack + uint32_t mPosition; // a frame position in AudioTrack::getPosition() units + struct timespec mTime; // corresponding CLOCK_MONOTONIC when frame is expected to present +}; + +#endif // ANDROID_AUDIO_TIMESTAMP_H diff --git a/include/media/AudioTrack.h b/include/media/AudioTrack.h index 64f82bb..f2f9c22 100644 --- a/include/media/AudioTrack.h +++ b/include/media/AudioTrack.h @@ -17,18 +17,10 @@ #ifndef ANDROID_AUDIOTRACK_H #define ANDROID_AUDIOTRACK_H -#include <stdint.h> -#include <sys/types.h> - -#include <media/IAudioFlinger.h> -#include <media/IAudioTrack.h> -#include <media/AudioSystem.h> - -#include <utils/RefBase.h> -#include <utils/Errors.h> -#include <binder/IInterface.h> -#include <binder/IMemory.h> #include <cutils/sched_policy.h> +#include <media/AudioSystem.h> +#include <media/AudioTimestamp.h> +#include <media/IAudioTrack.h> #include <utils/threads.h> namespace android { @@ -37,10 +29,11 @@ namespace android { class audio_track_cblk_t; class AudioTrackClientProxy; +class StaticAudioTrackClientProxy; // ---------------------------------------------------------------------------- -class AudioTrack : virtual public RefBase +class AudioTrack : public RefBase { public: enum channel_index { @@ -49,7 +42,7 @@ public: RIGHT = 1 }; - /* Events used by AudioTrack callback function (audio_track_cblk_t). + /* Events used by AudioTrack callback function (callback_t). * Keep in sync with frameworks/base/media/java/android/media/AudioTrack.java NATIVE_EVENT_*. */ enum event_type { @@ -64,7 +57,15 @@ public: // (See setMarkerPosition()). EVENT_NEW_POS = 4, // Playback head is at a new position // (See setPositionUpdatePeriod()). - EVENT_BUFFER_END = 5 // Playback head is at the end of the buffer. + EVENT_BUFFER_END = 5, // Playback head is at the end of the buffer. + // Not currently used by android.media.AudioTrack. + EVENT_NEW_IAUDIOTRACK = 6, // IAudioTrack was re-created, either due to re-routing and + // voluntary invalidation by mediaserver, or mediaserver crash. + EVENT_STREAM_END = 7, // Sent after all the buffers queued in AF and HW are played + // back (after stop is called) + EVENT_NEW_TIMESTAMP = 8, // Delivered periodically and when there's a significant change + // in the mapping from frame position to presentation time. + // See AudioTimestamp for the information included with event. }; /* Client should declare Buffer on the stack and pass address to obtainBuffer() @@ -74,19 +75,25 @@ public: class Buffer { public: + // FIXME use m prefix size_t frameCount; // number of sample frames corresponding to size; // on input it is the number of frames desired, // on output is the number of frames actually filled + // (currently ignored, but will make the primary field in future) + + size_t size; // input/output in bytes == frameCount * frameSize + // on output is the number of bytes actually filled + // FIXME this is redundant with respect to frameCount, + // and TRANSFER_OBTAIN mode is broken for 8-bit data + // since we don't define the frame format - size_t size; // input/output in byte units union { void* raw; - short* i16; // signed 16-bit - int8_t* i8; // unsigned 8-bit, offset by 0x80 + short* i16; // signed 16-bit + int8_t* i8; // unsigned 8-bit, offset by 0x80 }; }; - /* As a convenience, if a callback is supplied, a handler thread * is automatically created with the appropriate priority. This thread * invokes the callback when a new buffer becomes available or various conditions occur. @@ -100,9 +107,12 @@ public: * written. * - EVENT_UNDERRUN: unused. * - EVENT_LOOP_END: pointer to an int indicating the number of loops remaining. - * - EVENT_MARKER: pointer to an uint32_t containing the marker position in frames. - * - EVENT_NEW_POS: pointer to an uint32_t containing the new position in frames. + * - EVENT_MARKER: pointer to const uint32_t containing the marker position in frames. + * - EVENT_NEW_POS: pointer to const uint32_t containing the new position in frames. * - EVENT_BUFFER_END: unused. + * - EVENT_NEW_IAUDIOTRACK: unused. + * - EVENT_STREAM_END: unused. + * - EVENT_NEW_TIMESTAMP: pointer to const AudioTimestamp. */ typedef void (*callback_t)(int event, void* user, void *info); @@ -112,11 +122,22 @@ public: * Returned status (from utils/Errors.h) can be: * - NO_ERROR: successful operation * - NO_INIT: audio server or audio hardware not initialized + * - BAD_VALUE: unsupported configuration */ - static status_t getMinFrameCount(size_t* frameCount, - audio_stream_type_t streamType = AUDIO_STREAM_DEFAULT, - uint32_t sampleRate = 0); + static status_t getMinFrameCount(size_t* frameCount, + audio_stream_type_t streamType, + uint32_t sampleRate); + + /* How data is transferred to AudioTrack + */ + enum transfer_type { + TRANSFER_DEFAULT, // not specified explicitly; determine from the other parameters + TRANSFER_CALLBACK, // callback EVENT_MORE_DATA + TRANSFER_OBTAIN, // FIXME deprecated: call obtainBuffer() and releaseBuffer() + TRANSFER_SYNC, // synchronous write() + TRANSFER_SHARED, // shared memory + }; /* Constructs an uninitialized AudioTrack. No connection with * AudioFlinger takes place. Use set() after this. @@ -128,13 +149,13 @@ public: * Unspecified values are set to appropriate default values. * With this constructor, the track is configured for streaming mode. * Data to be rendered is supplied by write() or by the callback EVENT_MORE_DATA. - * Intermixing a combination of write() and non-ignored EVENT_MORE_DATA is deprecated. + * Intermixing a combination of write() and non-ignored EVENT_MORE_DATA is not allowed. * * Parameters: * * streamType: Select the type of audio stream this track is attached to * (e.g. AUDIO_STREAM_MUSIC). - * sampleRate: Track sampling rate in Hz. + * sampleRate: Data source sampling rate in Hz. * format: Audio format (e.g AUDIO_FORMAT_PCM_16_BIT for signed * 16 bits per sample). * channelMask: Channel mask. @@ -149,21 +170,24 @@ public: * user: Context for use by the callback receiver. * notificationFrames: The callback function is called each time notificationFrames PCM * frames have been consumed from track input buffer. + * This is expressed in units of frames at the initial source sample rate. * sessionId: Specific session ID, or zero to use default. - * threadCanCallJava: Whether callbacks are made from an attached thread and thus can call JNI. - * If not present in parameter list, then fixed at false. + * transferType: How data is transferred to AudioTrack. + * threadCanCallJava: Not present in parameter list, and so is fixed at false. */ AudioTrack( audio_stream_type_t streamType, - uint32_t sampleRate = 0, - audio_format_t format = AUDIO_FORMAT_DEFAULT, - audio_channel_mask_t channelMask = 0, + uint32_t sampleRate, + audio_format_t format, + audio_channel_mask_t, int frameCount = 0, audio_output_flags_t flags = AUDIO_OUTPUT_FLAG_NONE, callback_t cbf = NULL, void* user = NULL, int notificationFrames = 0, - int sessionId = 0); + int sessionId = 0, + transfer_type transferType = TRANSFER_DEFAULT, + const audio_offload_info_t *offloadInfo = NULL); /* Creates an audio track and registers it with AudioFlinger. * With this constructor, the track is configured for static buffer mode. @@ -174,38 +198,47 @@ public: * The write() method is not supported in this case. * It is recommended to pass a callback function to be notified of playback end by an * EVENT_UNDERRUN event. - * FIXME EVENT_MORE_DATA still occurs; it must be ignored. */ AudioTrack( audio_stream_type_t streamType, - uint32_t sampleRate = 0, - audio_format_t format = AUDIO_FORMAT_DEFAULT, - audio_channel_mask_t channelMask = 0, - const sp<IMemory>& sharedBuffer = 0, + uint32_t sampleRate, + audio_format_t format, + audio_channel_mask_t channelMask, + const sp<IMemory>& sharedBuffer, audio_output_flags_t flags = AUDIO_OUTPUT_FLAG_NONE, callback_t cbf = NULL, void* user = NULL, int notificationFrames = 0, - int sessionId = 0); + int sessionId = 0, + transfer_type transferType = TRANSFER_DEFAULT, + const audio_offload_info_t *offloadInfo = NULL); /* Terminates the AudioTrack and unregisters it from AudioFlinger. * Also destroys all resources associated with the AudioTrack. */ - ~AudioTrack(); +protected: + virtual ~AudioTrack(); +public: - /* Initialize an uninitialized AudioTrack. + /* Initialize an AudioTrack that was created using the AudioTrack() constructor. + * Don't call set() more than once, or after the AudioTrack() constructors that take parameters. * Returned status (from utils/Errors.h) can be: * - NO_ERROR: successful initialization * - INVALID_OPERATION: AudioTrack is already initialized * - BAD_VALUE: invalid parameter (channelMask, format, sampleRate...) * - NO_INIT: audio server or audio hardware not initialized + * If status is not equal to NO_ERROR, don't call any other APIs on this AudioTrack. * If sharedBuffer is non-0, the frameCount parameter is ignored and * replaced by the shared buffer's total allocated size in frame units. + * + * Parameters not listed in the AudioTrack constructors above: + * + * threadCanCallJava: Whether callbacks are made from an attached thread and thus can call JNI. */ - status_t set(audio_stream_type_t streamType = AUDIO_STREAM_DEFAULT, - uint32_t sampleRate = 0, - audio_format_t format = AUDIO_FORMAT_DEFAULT, - audio_channel_mask_t channelMask = 0, + status_t set(audio_stream_type_t streamType, + uint32_t sampleRate, + audio_format_t format, + audio_channel_mask_t channelMask, int frameCount = 0, audio_output_flags_t flags = AUDIO_OUTPUT_FLAG_NONE, callback_t cbf = NULL, @@ -213,9 +246,11 @@ public: int notificationFrames = 0, const sp<IMemory>& sharedBuffer = 0, bool threadCanCallJava = false, - int sessionId = 0); + int sessionId = 0, + transfer_type transferType = TRANSFER_DEFAULT, + const audio_offload_info_t *offloadInfo = NULL); - /* Result of constructing the AudioTrack. This must be checked + /* Result of constructing the AudioTrack. This must be checked for successful initialization * before using any AudioTrack API (except for set()), because using * an uninitialized AudioTrack produces undefined results. * See set() method above for possible return codes. @@ -233,14 +268,15 @@ public: audio_stream_type_t streamType() const { return mStreamType; } audio_format_t format() const { return mFormat; } - /* Return frame size in bytes, which for linear PCM is channelCount * (bit depth per channel / 8). + /* Return frame size in bytes, which for linear PCM is + * channelCount * (bit depth per channel / 8). * channelCount is determined from channelMask, and bit depth comes from format. * For non-linear formats, the frame size is typically 1 byte. */ - uint32_t channelCount() const { return mChannelCount; } + size_t frameSize() const { return mFrameSize; } + uint32_t channelCount() const { return mChannelCount; } uint32_t frameCount() const { return mFrameCount; } - size_t frameSize() const { return mFrameSize; } /* Return the static buffer specified in constructor or set(), or 0 for streaming mode */ sp<IMemory> sharedBuffer() const { return mSharedBuffer; } @@ -249,14 +285,13 @@ public: * make it active. If set, the callback will start being called. * If the track was previously paused, volume is ramped up over the first mix buffer. */ - void start(); + status_t start(); /* Stop a track. * In static buffer mode, the track is stopped immediately. - * In streaming mode, the callback will cease being called and - * obtainBuffer returns STOPPED. Note that obtainBuffer() still works - * and will fill up buffers until the pool is exhausted. - * The stop does not occur immediately: any data remaining in the buffer + * In streaming mode, the callback will cease being called. Note that obtainBuffer() still + * works and will fill up buffers until the pool is exhausted, and then will return WOULD_BLOCK. + * In streaming mode the stop does not occur immediately: any data remaining in the buffer * is first drained, mixed, and output, and only then is the track marked as stopped. */ void stop(); @@ -270,7 +305,7 @@ public: void flush(); /* Pause a track. After pause, the callback will cease being called and - * obtainBuffer returns STOPPED. Note that obtainBuffer() still works + * obtainBuffer returns WOULD_BLOCK. Note that obtainBuffer() still works * and will fill up buffers until the pool is exhausted. * Volume is ramped down over the next mix buffer following the pause request, * and then the track is marked as paused. It can be resumed with ramp up by start(). @@ -294,11 +329,11 @@ public: status_t setAuxEffectSendLevel(float level); void getAuxEffectSendLevel(float* level) const; - /* Set sample rate for this track in Hz, mostly used for games' sound effects + /* Set source sample rate for this track in Hz, mostly used for games' sound effects */ status_t setSampleRate(uint32_t sampleRate); - /* Return current sample rate in Hz, or 0 if unknown */ + /* Return current source sample rate in Hz, or 0 if unknown */ uint32_t getSampleRate() const; /* Enables looping and sets the start and end points of looping. @@ -306,20 +341,24 @@ public: * * Parameters: * - * loopStart: loop start expressed as the number of PCM frames played since AudioTrack start. - * loopEnd: loop end expressed as the number of PCM frames played since AudioTrack start. + * loopStart: loop start in frames relative to start of buffer. + * loopEnd: loop end in frames relative to start of buffer. * loopCount: number of loops to execute. Calling setLoop() with loopCount == 0 cancels any - * pending or active loop. loopCount = -1 means infinite looping. + * pending or active loop. loopCount == -1 means infinite looping. * * For proper operation the following condition must be respected: - * (loopEnd-loopStart) <= framecount() + * loopCount != 0 implies 0 <= loopStart < loopEnd <= frameCount(). + * + * If the loop period (loopEnd - loopStart) is too small for the implementation to support, + * setLoop() will return BAD_VALUE. loopCount must be >= -1. + * */ status_t setLoop(uint32_t loopStart, uint32_t loopEnd, int loopCount); /* Sets marker position. When playback reaches the number of frames specified, a callback with * event type EVENT_MARKER is called. Calling setMarkerPosition with marker == 0 cancels marker * notification callback. To set a marker at a position which would compute as 0, - * a workaround is to the set the marker at a nearby position such as -1 or 1. + * a workaround is to the set the marker at a nearby position such as ~0 or 1. * If the AudioTrack has been opened with no callback function associated, the operation will * fail. * @@ -354,18 +393,14 @@ public: status_t setPositionUpdatePeriod(uint32_t updatePeriod); status_t getPositionUpdatePeriod(uint32_t *updatePeriod) const; - /* Sets playback head position within AudioTrack buffer. The new position is specified - * in number of frames. - * This method must be called with the AudioTrack in paused or stopped state. - * Note that the actual position set is <position> modulo the AudioTrack buffer size in frames. - * Therefore using this method makes sense only when playing a "static" audio buffer - * as opposed to streaming. - * The getPosition() method on the other hand returns the total number of frames played since - * playback start. + /* Sets playback head position. + * Only supported for static buffer mode. * * Parameters: * - * position: New playback head position within AudioTrack buffer. + * position: New playback head position in frames relative to start of buffer. + * 0 <= position <= frameCount(). Note that end of buffer is permitted, + * but will result in an immediate underrun if started. * * Returned status (from utils/Errors.h) can be: * - NO_ERROR: successful operation @@ -378,8 +413,22 @@ public: /* Return the total number of frames played since playback start. * The counter will wrap (overflow) periodically, e.g. every ~27 hours at 44.1 kHz. * It is reset to zero by flush(), reload(), and stop(). + * + * Parameters: + * + * position: Address where to return play head position. + * + * Returned status (from utils/Errors.h) can be: + * - NO_ERROR: successful operation + * - BAD_VALUE: position is NULL */ - status_t getPosition(uint32_t *position); + status_t getPosition(uint32_t *position) const; + + /* For static buffer mode only, this returns the current playback position in frames + * relative to start of buffer. It is analogous to the position units used by + * setLoop() and setPosition(). After underrun, the position will be at end of buffer. + */ + status_t getBufferPosition(uint32_t *position); /* Forces AudioTrack buffer full condition. When playing a static buffer, this method avoids * rewriting the buffer before restarting playback after a stop. @@ -426,15 +475,19 @@ public: */ status_t attachAuxEffect(int effectId); - /* Obtains a buffer of "frameCount" frames. The buffer must be - * filled entirely, and then released with releaseBuffer(). - * If the track is stopped, obtainBuffer() returns - * STOPPED instead of NO_ERROR as long as there are buffers available, - * at which point NO_MORE_BUFFERS is returned. + /* Obtains a buffer of up to "audioBuffer->frameCount" empty slots for frames. + * After filling these slots with data, the caller should release them with releaseBuffer(). + * If the track buffer is not full, obtainBuffer() returns as many contiguous + * [empty slots for] frames as are available immediately. + * If the track buffer is full and track is stopped, obtainBuffer() returns WOULD_BLOCK + * regardless of the value of waitCount. + * If the track buffer is full and track is not stopped, obtainBuffer() blocks with a + * maximum timeout based on waitCount; see chart below. * Buffers will be returned until the pool * is exhausted, at which point obtainBuffer() will either block - * or return WOULD_BLOCK depending on the value of the "blocking" + * or return WOULD_BLOCK depending on the value of the "waitCount" * parameter. + * Each sample is 16-bit signed PCM. * * obtainBuffer() and releaseBuffer() are deprecated for direct use by applications, * which should use write() or callback EVENT_MORE_DATA instead. @@ -457,33 +510,76 @@ public: * raw pointer to the buffer */ - enum { - NO_MORE_BUFFERS = 0x80000001, // same name in AudioFlinger.h, ok to be different value - STOPPED = 1 - }; + /* FIXME Deprecated public API for TRANSFER_OBTAIN mode */ + status_t obtainBuffer(Buffer* audioBuffer, int32_t waitCount) + __attribute__((__deprecated__)); - status_t obtainBuffer(Buffer* audioBuffer, int32_t waitCount); +private: + /* If nonContig is non-NULL, it is an output parameter that will be set to the number of + * additional non-contiguous frames that are available immediately. + * FIXME We could pass an array of Buffers instead of only one Buffer to obtainBuffer(), + * in case the requested amount of frames is in two or more non-contiguous regions. + * FIXME requested and elapsed are both relative times. Consider changing to absolute time. + */ + status_t obtainBuffer(Buffer* audioBuffer, const struct timespec *requested, + struct timespec *elapsed = NULL, size_t *nonContig = NULL); +public: - /* Release a filled buffer of "frameCount" frames for AudioFlinger to process. */ +//EL_FIXME to be reconciled with new obtainBuffer() return codes and control block proxy +// enum { +// NO_MORE_BUFFERS = 0x80000001, // same name in AudioFlinger.h, ok to be different value +// TEAR_DOWN = 0x80000002, +// STOPPED = 1, +// STREAM_END_WAIT, +// STREAM_END +// }; + + /* Release a filled buffer of "audioBuffer->frameCount" frames for AudioFlinger to process. */ + // FIXME make private when obtainBuffer() for TRANSFER_OBTAIN is removed void releaseBuffer(Buffer* audioBuffer); /* As a convenience we provide a write() interface to the audio buffer. + * Input parameter 'size' is in byte units. * This is implemented on top of obtainBuffer/releaseBuffer. For best * performance use callbacks. Returns actual number of bytes written >= 0, * or one of the following negative status codes: - * INVALID_OPERATION AudioTrack is configured for shared buffer mode + * INVALID_OPERATION AudioTrack is configured for static buffer or streaming mode * BAD_VALUE size is invalid - * STOPPED AudioTrack was stopped during the write - * NO_MORE_BUFFERS when obtainBuffer() returns same + * WOULD_BLOCK when obtainBuffer() returns same, or + * AudioTrack was stopped during the write * or any other error code returned by IAudioTrack::start() or restoreTrack_l(). - * Not supported for static buffer mode. */ ssize_t write(const void* buffer, size_t size); /* * Dumps the state of an audio track. */ - status_t dump(int fd, const Vector<String16>& args) const; + status_t dump(int fd, const Vector<String16>& args) const; + + /* + * Return the total number of frames which AudioFlinger desired but were unavailable, + * and thus which resulted in an underrun. Reset to zero by stop(). + */ + uint32_t getUnderrunFrames() const; + + /* Get the flags */ + audio_output_flags_t getFlags() const { return mFlags; } + + /* Set parameters - only possible when using direct output */ + status_t setParameters(const String8& keyValuePairs); + + /* Get parameters */ + String8 getParameters(const String8& keys); + + /* Poll for a timestamp on demand. + * Use if EVENT_NEW_TIMESTAMP is not delivered often enough for your needs, + * or if you need to get the most recent timestamp outside of the event callback handler. + * Caution: calling this method too often may be inefficient; + * if you need a high resolution mapping between frame position and presentation time, + * consider implementing that at application level, based on the low resolution timestamps. + * Returns NO_ERROR if timestamp is valid. + */ + status_t getTimestamp(AudioTimestamp& timestamp); protected: /* copying audio tracks is not allowed */ @@ -504,39 +600,62 @@ protected: void resume(); // allow thread to execute, if not requested to exit private: + void pauseInternal(nsecs_t ns = 0LL); + // like pause(), but only used internally within thread + friend class AudioTrack; virtual bool threadLoop(); - AudioTrack& mReceiver; - ~AudioTrackThread(); + AudioTrack& mReceiver; + virtual ~AudioTrackThread(); Mutex mMyLock; // Thread::mLock is private Condition mMyCond; // Thread::mThreadExitedCondition is private - bool mPaused; // whether thread is currently paused + bool mPaused; // whether thread is requested to pause at next loop entry + bool mPausedInt; // whether thread internally requests pause + nsecs_t mPausedNs; // if mPausedInt then associated timeout, otherwise ignored + bool mIgnoreNextPausedInt; // whether to ignore next mPausedInt request }; // body of AudioTrackThread::threadLoop() - bool processAudioBuffer(const sp<AudioTrackThread>& thread); + // returns the maximum amount of time before we would like to run again, where: + // 0 immediately + // > 0 no later than this many nanoseconds from now + // NS_WHENEVER still active but no particular deadline + // NS_INACTIVE inactive so don't run again until re-started + // NS_NEVER never again + static const nsecs_t NS_WHENEVER = -1, NS_INACTIVE = -2, NS_NEVER = -3; + nsecs_t processAudioBuffer(const sp<AudioTrackThread>& thread); + status_t processStreamEnd(int32_t waitCount); + // caller must hold lock on mLock for all _l methods + status_t createTrack_l(audio_stream_type_t streamType, uint32_t sampleRate, audio_format_t format, size_t frameCount, audio_output_flags_t flags, const sp<IMemory>& sharedBuffer, - audio_io_handle_t output); + audio_io_handle_t output, + size_t epoch); - // can only be called when !mActive + // can only be called when mState != STATE_ACTIVE void flush_l(); - status_t setLoop_l(uint32_t loopStart, uint32_t loopEnd, int loopCount); + void setLoop_l(uint32_t loopStart, uint32_t loopEnd, int loopCount); audio_io_handle_t getOutput_l(); - status_t restoreTrack_l(audio_track_cblk_t*& cblk, bool fromStart); - bool stopped_l() const { return !mActive; } + // FIXME enum is faster than strcmp() for parameter 'from' + status_t restoreTrack_l(const char *from); + + bool isOffloaded() const + { return (mFlags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) != 0; } + + // Next 3 fields may be changed if IAudioTrack is re-created, but always != 0 sp<IAudioTrack> mAudioTrack; sp<IMemory> mCblkMemory; - sp<AudioTrackThread> mAudioTrackThread; + audio_track_cblk_t* mCblk; // re-load after mLock.unlock() + sp<AudioTrackThread> mAudioTrackThread; float mVolume[2]; float mSendLevel; uint32_t mSampleRate; @@ -544,62 +663,94 @@ protected: size_t mReqFrameCount; // frame count to request the next time a new // IAudioTrack is needed - audio_track_cblk_t* mCblk; // re-load after mLock.unlock() - - // Starting address of buffers in shared memory. If there is a shared buffer, mBuffers - // is the value of pointer() for the shared buffer, otherwise mBuffers points - // immediately after the control block. This address is for the mapping within client - // address space. AudioFlinger::TrackBase::mBuffer is for the server address space. - void* mBuffers; + // constant after constructor or set() audio_format_t mFormat; // as requested by client, not forced to 16-bit audio_stream_type_t mStreamType; uint32_t mChannelCount; audio_channel_mask_t mChannelMask; + transfer_type mTransfer; - // mFrameSize is equal to mFrameSizeAF for non-PCM or 16-bit PCM data. - // For 8-bit PCM data, mFrameSizeAF is - // twice as large because data is expanded to 16-bit before being stored in buffer. + // mFrameSize is equal to mFrameSizeAF for non-PCM or 16-bit PCM data. For 8-bit PCM data, it's + // twice as large as mFrameSize because data is expanded to 16-bit before it's stored in buffer. size_t mFrameSize; // app-level frame size size_t mFrameSizeAF; // AudioFlinger frame size status_t mStatus; - uint32_t mLatency; - bool mActive; // protected by mLock + // can change dynamically when IAudioTrack invalidated + uint32_t mLatency; // in ms + // Indicates the current track state. Protected by mLock. + enum State { + STATE_ACTIVE, + STATE_STOPPED, + STATE_PAUSED, + STATE_PAUSED_STOPPING, + STATE_FLUSHED, + STATE_STOPPING, + } mState; + + // for client callback handler callback_t mCbf; // callback handler for events, or NULL - void* mUserData; // for client callback handler + void* mUserData; // for notification APIs uint32_t mNotificationFramesReq; // requested number of frames between each - // notification callback + // notification callback, + // at initial source sample rate uint32_t mNotificationFramesAct; // actual number of frames between each - // notification callback + // notification callback, + // at initial source sample rate + bool mRefreshRemaining; // processAudioBuffer() should refresh next 2 + + // These are private to processAudioBuffer(), and are not protected by a lock + uint32_t mRemainingFrames; // number of frames to request in obtainBuffer() + bool mRetryOnPartialBuffer; // sleep and retry after partial obtainBuffer() + uint32_t mObservedSequence; // last observed value of mSequence + sp<IMemory> mSharedBuffer; - int mLoopCount; - uint32_t mRemainingFrames; + uint32_t mLoopPeriod; // in frames, zero means looping is disabled uint32_t mMarkerPosition; // in wrapping (overflow) frame units bool mMarkerReached; uint32_t mNewPosition; // in frames - uint32_t mUpdatePeriod; // in frames + uint32_t mUpdatePeriod; // in frames, zero means no EVENT_NEW_POS - bool mFlushed; // FIXME will be made obsolete by making flush() synchronous audio_output_flags_t mFlags; int mSessionId; int mAuxEffectId; - // When locking both mLock and mCblk->lock, must lock in this order to avoid deadlock: - // 1. mLock - // 2. mCblk->lock - // It is OK to lock only mCblk->lock. mutable Mutex mLock; bool mIsTimed; int mPreviousPriority; // before start() SchedPolicy mPreviousSchedulingGroup; - AudioTrackClientProxy* mProxy; bool mAwaitBoost; // thread should wait for priority boost before running + + // The proxy should only be referenced while a lock is held because the proxy isn't + // multi-thread safe, especially the SingleStateQueue part of the proxy. + // An exception is that a blocking ClientProxy::obtainBuffer() may be called without a lock, + // provided that the caller also holds an extra reference to the proxy and shared memory to keep + // them around in case they are replaced during the obtainBuffer(). + sp<StaticAudioTrackClientProxy> mStaticProxy; // for type safety only + sp<AudioTrackClientProxy> mProxy; // primary owner of the memory + + bool mInUnderrun; // whether track is currently in underrun state + String8 mName; // server's name for this IAudioTrack + +private: + class DeathNotifier : public IBinder::DeathRecipient { + public: + DeathNotifier(AudioTrack* audioTrack) : mAudioTrack(audioTrack) { } + protected: + virtual void binderDied(const wp<IBinder>& who); + private: + const wp<AudioTrack> mAudioTrack; + }; + + sp<DeathNotifier> mDeathNotifier; + uint32_t mSequence; // incremented for each new IAudioTrack attempt + audio_io_handle_t mOutput; // cached output io handle }; class TimedAudioTrack : public AudioTrack diff --git a/include/media/EffectsFactoryApi.h b/include/media/EffectsFactoryApi.h index b1ed7b0..b1143b9 100644 --- a/include/media/EffectsFactoryApi.h +++ b/include/media/EffectsFactoryApi.h @@ -171,6 +171,30 @@ int EffectGetDescriptor(const effect_uuid_t *pEffectUuid, effect_descriptor_t *p //////////////////////////////////////////////////////////////////////////////// int EffectIsNullUuid(const effect_uuid_t *pEffectUuid); +//////////////////////////////////////////////////////////////////////////////// +// +// Function: EffectGetSubEffects +// +// Description: Returns the descriptors of the sub effects of the effect +// whose uuid is pointed to by first argument. +// +// Input: +// pEffectUuid: pointer to the effect uuid. +// size: size of the buffer pointed by pDescriptor. +// +// Input/Output: +// pDescriptor: address where to return the sub effect descriptors. +// +// Output: +// returned value: 0 successful operation. +// -ENODEV factory failed to initialize +// -EINVAL invalid pEffectUuid or pDescriptor +// -ENOENT no effect with this uuid found +// *pDescriptor: updated with the sub effect descriptors. +// +//////////////////////////////////////////////////////////////////////////////// +int EffectGetSubEffects(const effect_uuid_t *pEffectUuid, effect_descriptor_t *pDescriptors, size_t size); + #if __cplusplus } // extern "C" #endif diff --git a/include/media/ExtendedAudioBufferProvider.h b/include/media/ExtendedAudioBufferProvider.h index 00c4444..2539ed3 100644 --- a/include/media/ExtendedAudioBufferProvider.h +++ b/include/media/ExtendedAudioBufferProvider.h @@ -18,12 +18,20 @@ #define ANDROID_EXTENDED_AUDIO_BUFFER_PROVIDER_H #include <media/AudioBufferProvider.h> +#include <media/AudioTimestamp.h> namespace android { class ExtendedAudioBufferProvider : public AudioBufferProvider { public: virtual size_t framesReady() const = 0; // see description at AudioFlinger.h + + // Return the total number of frames that have been obtained and released + virtual size_t framesReleased() const { return 0; } + + // Invoked by buffer consumer when a new timestamp is available. + // Default implementation ignores the timestamp. + virtual void onTimestamp(const AudioTimestamp& timestamp) { } }; } // namespace android diff --git a/include/media/IAudioFlinger.h b/include/media/IAudioFlinger.h index 9c3067e..eaf7780 100644 --- a/include/media/IAudioFlinger.h +++ b/include/media/IAudioFlinger.h @@ -49,9 +49,13 @@ public: TRACK_DEFAULT = 0, // client requests a default AudioTrack TRACK_TIMED = 1, // client requests a TimedAudioTrack TRACK_FAST = 2, // client requests a fast AudioTrack or AudioRecord + TRACK_OFFLOAD = 4, // client requests offload to hw codec }; typedef uint32_t track_flags_t; + // invariant on exit for all APIs that return an sp<>: + // (return value != 0) == (*status == NO_ERROR) + /* create an audio track and registers it with AudioFlinger. * return null if the track cannot be created. */ @@ -66,6 +70,10 @@ public: audio_io_handle_t output, pid_t tid, // -1 means unused, otherwise must be valid non-0 int *sessionId, + // input: ignored + // output: server's description of IAudioTrack for display in logs. + // Don't attempt to parse, as the format could change. + String8& name, status_t *status) = 0; virtual sp<IAudioRecord> openRecord( @@ -74,7 +82,7 @@ public: audio_format_t format, audio_channel_mask_t channelMask, size_t frameCount, - track_flags_t flags, + track_flags_t *flags, pid_t tid, // -1 means unused, otherwise must be valid non-0 int *sessionId, status_t *status) = 0; @@ -124,7 +132,9 @@ public: virtual String8 getParameters(audio_io_handle_t ioHandle, const String8& keys) const = 0; - // register a current process for audio output change notifications + // Register an object to receive audio input/output change and track notifications. + // For a given calling pid, AudioFlinger disregards any registrations after the first. + // Thus the IAudioFlingerClient must be a singleton per process. virtual void registerClient(const sp<IAudioFlingerClient>& client) = 0; // retrieve the audio recording buffer size @@ -137,7 +147,8 @@ public: audio_format_t *pFormat, audio_channel_mask_t *pChannelMask, uint32_t *pLatencyMs, - audio_output_flags_t flags) = 0; + audio_output_flags_t flags, + const audio_offload_info_t *offloadInfo = NULL) = 0; virtual audio_io_handle_t openDuplicateOutput(audio_io_handle_t output1, audio_io_handle_t output2) = 0; virtual status_t closeOutput(audio_io_handle_t output) = 0; @@ -193,6 +204,10 @@ public: virtual uint32_t getPrimaryOutputSamplingRate() = 0; virtual size_t getPrimaryOutputFrameCount() = 0; + // Intended for AudioService to inform AudioFlinger of device's low RAM attribute, + // and should be called at most once. For a definition of what "low RAM" means, see + // android.app.ActivityManager.isLowRamDevice(). + virtual status_t setLowRamDevice(bool isLowRamDevice) = 0; }; diff --git a/include/media/IAudioPolicyService.h b/include/media/IAudioPolicyService.h index b5ad4ef..09b9ea6 100644 --- a/include/media/IAudioPolicyService.h +++ b/include/media/IAudioPolicyService.h @@ -53,7 +53,8 @@ public: uint32_t samplingRate = 0, audio_format_t format = AUDIO_FORMAT_DEFAULT, audio_channel_mask_t channelMask = 0, - audio_output_flags_t flags = AUDIO_OUTPUT_FLAG_NONE) = 0; + audio_output_flags_t flags = AUDIO_OUTPUT_FLAG_NONE, + const audio_offload_info_t *offloadInfo = NULL) = 0; virtual status_t startOutput(audio_io_handle_t output, audio_stream_type_t stream, int session = 0) = 0; @@ -95,6 +96,9 @@ public: virtual status_t queryDefaultPreProcessing(int audioSession, effect_descriptor_t *descriptors, uint32_t *count) = 0; + // Check if offload is possible for given format, stream type, sample rate, + // bit rate, duration, video and streaming or offload property is enabled + virtual bool isOffloadSupported(const audio_offload_info_t& info) = 0; }; diff --git a/include/media/IAudioRecord.h b/include/media/IAudioRecord.h index d6e3141..eccc2ca 100644 --- a/include/media/IAudioRecord.h +++ b/include/media/IAudioRecord.h @@ -34,6 +34,9 @@ class IAudioRecord : public IInterface public: DECLARE_META_INTERFACE(AudioRecord); + /* get this tracks control block */ + virtual sp<IMemory> getCblk() const = 0; + /* After it's created the track is not active. Call start() to * make it active. */ @@ -44,9 +47,6 @@ public: * will be processed, unless flush() is called. */ virtual void stop() = 0; - - /* get this tracks control block */ - virtual sp<IMemory> getCblk() const = 0; }; // ---------------------------------------------------------------------------- diff --git a/include/media/IAudioTrack.h b/include/media/IAudioTrack.h index 144be0e..5c8a484 100644 --- a/include/media/IAudioTrack.h +++ b/include/media/IAudioTrack.h @@ -25,6 +25,8 @@ #include <binder/IInterface.h> #include <binder/IMemory.h> #include <utils/LinearTransform.h> +#include <utils/String8.h> +#include <media/AudioTimestamp.h> namespace android { @@ -82,6 +84,15 @@ public: or Tungsten time. The values for target are defined in AudioTrack.h */ virtual status_t setMediaTimeTransform(const LinearTransform& xform, int target) = 0; + + /* Send parameters to the audio hardware */ + virtual status_t setParameters(const String8& keyValuePairs) = 0; + + /* Return NO_ERROR if timestamp is valid */ + virtual status_t getTimestamp(AudioTimestamp& timestamp) = 0; + + /* Signal the playback thread for a change in control block */ + virtual void signal() = 0; }; // ---------------------------------------------------------------------------- diff --git a/include/media/IDrm.h b/include/media/IDrm.h index d630c40..5ef26af 100644 --- a/include/media/IDrm.h +++ b/include/media/IDrm.h @@ -32,7 +32,7 @@ struct IDrm : public IInterface { virtual status_t initCheck() const = 0; - virtual bool isCryptoSchemeSupported(const uint8_t uuid[16]) = 0; + virtual bool isCryptoSchemeSupported(const uint8_t uuid[16], const String8 &mimeType) = 0; virtual status_t createPlugin(const uint8_t uuid[16]) = 0; diff --git a/include/media/IHDCP.h b/include/media/IHDCP.h index 6d27b18..352561e 100644 --- a/include/media/IHDCP.h +++ b/include/media/IHDCP.h @@ -17,6 +17,7 @@ #include <binder/IInterface.h> #include <media/hardware/HDCPAPI.h> #include <media/stagefright/foundation/ABase.h> +#include <ui/GraphicBuffer.h> namespace android { @@ -45,6 +46,17 @@ struct IHDCP : public IInterface { // Request to shutdown the active HDCP session. virtual status_t shutdownAsync() = 0; + // Returns the capability bitmask of this HDCP session. + // Possible return values (please refer to HDCAPAPI.h): + // HDCP_CAPS_ENCRYPT: mandatory, meaning the HDCP module can encrypt + // from an input byte-array buffer to an output byte-array buffer + // HDCP_CAPS_ENCRYPT_NATIVE: the HDCP module supports encryption from + // a native buffer to an output byte-array buffer. The format of the + // input native buffer is specific to vendor's encoder implementation. + // It is the same format as that used by the encoder when + // "storeMetaDataInBuffers" extension is enabled on its output port. + virtual uint32_t getCaps() = 0; + // ENCRYPTION only: // Encrypt data according to the HDCP spec. "size" bytes of data are // available at "inData" (virtual address), "size" may not be a multiple @@ -59,6 +71,20 @@ struct IHDCP : public IInterface { const void *inData, size_t size, uint32_t streamCTR, uint64_t *outInputCTR, void *outData) = 0; + // Encrypt data according to the HDCP spec. "size" bytes of data starting + // at location "offset" are available in "buffer" (buffer handle). "size" + // may not be a multiple of 128 bits (16 bytes). An equal number of + // encrypted bytes should be written to the buffer at "outData" (virtual + // address). This operation is to be synchronous, i.e. this call does not + // return until outData contains size bytes of encrypted data. + // streamCTR will be assigned by the caller (to 0 for the first PES stream, + // 1 for the second and so on) + // inputCTR _will_be_maintained_by_the_callee_ for each PES stream. + virtual status_t encryptNative( + const sp<GraphicBuffer> &graphicBuffer, + size_t offset, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData) = 0; + // DECRYPTION only: // Decrypt data according to the HDCP spec. // "size" bytes of encrypted data are available at "inData" diff --git a/include/media/IMediaPlayerService.h b/include/media/IMediaPlayerService.h index fef7af2..2998b37 100644 --- a/include/media/IMediaPlayerService.h +++ b/include/media/IMediaPlayerService.h @@ -49,8 +49,12 @@ public: virtual sp<IMediaMetadataRetriever> createMetadataRetriever() = 0; virtual sp<IMediaPlayer> create(const sp<IMediaPlayerClient>& client, int audioSessionId = 0) = 0; - virtual sp<IMemory> decode(const char* url, uint32_t *pSampleRate, int* pNumChannels, audio_format_t* pFormat) = 0; - virtual sp<IMemory> decode(int fd, int64_t offset, int64_t length, uint32_t *pSampleRate, int* pNumChannels, audio_format_t* pFormat) = 0; + virtual status_t decode(const char* url, uint32_t *pSampleRate, int* pNumChannels, + audio_format_t* pFormat, + const sp<IMemoryHeap>& heap, size_t *pSize) = 0; + virtual status_t decode(int fd, int64_t offset, int64_t length, uint32_t *pSampleRate, + int* pNumChannels, audio_format_t* pFormat, + const sp<IMemoryHeap>& heap, size_t *pSize) = 0; virtual sp<IOMX> getOMX() = 0; virtual sp<ICrypto> makeCrypto() = 0; virtual sp<IDrm> makeDrm() = 0; diff --git a/include/media/IOMX.h b/include/media/IOMX.h index 0b1d1e4..9c8451c 100644 --- a/include/media/IOMX.h +++ b/include/media/IOMX.h @@ -83,6 +83,10 @@ public: virtual status_t storeMetaDataInBuffers( node_id node, OMX_U32 port_index, OMX_BOOL enable) = 0; + virtual status_t prepareForAdaptivePlayback( + node_id node, OMX_U32 portIndex, OMX_BOOL enable, + OMX_U32 maxFrameWidth, OMX_U32 maxFrameHeight) = 0; + virtual status_t enableGraphicBuffers( node_id node, OMX_U32 port_index, OMX_BOOL enable) = 0; @@ -97,6 +101,10 @@ public: node_id node, OMX_U32 port_index, const sp<GraphicBuffer> &graphicBuffer, buffer_id *buffer) = 0; + virtual status_t updateGraphicBufferInMeta( + node_id node, OMX_U32 port_index, + const sp<GraphicBuffer> &graphicBuffer, buffer_id buffer) = 0; + virtual status_t createInputSurface( node_id node, OMX_U32 port_index, sp<IGraphicBufferProducer> *bufferProducer) = 0; @@ -130,6 +138,17 @@ public: node_id node, const char *parameter_name, OMX_INDEXTYPE *index) = 0; + + enum InternalOptionType { + INTERNAL_OPTION_SUSPEND, // data is a bool + INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY, // data is an int64_t + }; + virtual status_t setInternalOption( + node_id node, + OMX_U32 port_index, + InternalOptionType type, + const void *data, + size_t size) = 0; }; struct omx_message { diff --git a/include/media/IRemoteDisplayClient.h b/include/media/IRemoteDisplayClient.h index 7b0fa9e..0e6d55d 100644 --- a/include/media/IRemoteDisplayClient.h +++ b/include/media/IRemoteDisplayClient.h @@ -49,7 +49,7 @@ public: // Provides a surface texture that the client should use to stream buffers to // the remote display. virtual void onDisplayConnected(const sp<IGraphicBufferProducer>& bufferProducer, - uint32_t width, uint32_t height, uint32_t flags) = 0; // one-way + uint32_t width, uint32_t height, uint32_t flags, uint32_t session) = 0; // one-way // Indicates that the remote display has been disconnected normally. // This method should only be called once the client has called 'dispose()' diff --git a/include/media/JetPlayer.h b/include/media/JetPlayer.h index 0616bf0..388f767 100644 --- a/include/media/JetPlayer.h +++ b/include/media/JetPlayer.h @@ -88,7 +88,7 @@ private: EAS_DATA_HANDLE mEasData; EAS_FILE_LOCATOR mEasJetFileLoc; EAS_PCM* mAudioBuffer;// EAS renders the MIDI data into this buffer, - AudioTrack* mAudioTrack; // and we play it in this audio track + sp<AudioTrack> mAudioTrack; // and we play it in this audio track int mTrackBufferSize; S_JET_STATUS mJetStatus; S_JET_STATUS mPreviousJetStatus; diff --git a/include/media/MediaPlayerInterface.h b/include/media/MediaPlayerInterface.h index 9a75f81..3b151ef 100644 --- a/include/media/MediaPlayerInterface.h +++ b/include/media/MediaPlayerInterface.h @@ -74,9 +74,18 @@ public: // AudioSink: abstraction layer for audio output class AudioSink : public RefBase { public: + enum cb_event_t { + CB_EVENT_FILL_BUFFER, // Request to write more data to buffer. + CB_EVENT_STREAM_END, // Sent after all the buffers queued in AF and HW are played + // back (after stop is called) + CB_EVENT_TEAR_DOWN // The AudioTrack was invalidated due to use case change: + // Need to re-evaluate offloading options + }; + // Callback returns the number of bytes actually written to the buffer. typedef size_t (*AudioCallback)( - AudioSink *audioSink, void *buffer, size_t size, void *cookie); + AudioSink *audioSink, void *buffer, size_t size, void *cookie, + cb_event_t event); virtual ~AudioSink() {} virtual bool ready() const = 0; // audio output is open and ready @@ -99,9 +108,10 @@ public: int bufferCount=DEFAULT_AUDIOSINK_BUFFERCOUNT, AudioCallback cb = NULL, void *cookie = NULL, - audio_output_flags_t flags = AUDIO_OUTPUT_FLAG_NONE) = 0; + audio_output_flags_t flags = AUDIO_OUTPUT_FLAG_NONE, + const audio_offload_info_t *offloadInfo = NULL) = 0; - virtual void start() = 0; + virtual status_t start() = 0; virtual ssize_t write(const void* buffer, size_t size) = 0; virtual void stop() = 0; virtual void flush() = 0; @@ -110,6 +120,9 @@ public: virtual status_t setPlaybackRatePermille(int32_t rate) { return INVALID_OPERATION; } virtual bool needsTrailingPadding() { return true; } + + virtual status_t setParameters(const String8& keyValuePairs) { return NO_ERROR; }; + virtual String8 getParameters(const String8& keys) { return String8::empty(); }; }; MediaPlayerBase() : mCookie(0), mNotify(0) {} diff --git a/include/media/SoundPool.h b/include/media/SoundPool.h index 7bf3069..2dd78cc 100644 --- a/include/media/SoundPool.h +++ b/include/media/SoundPool.h @@ -22,6 +22,8 @@ #include <utils/Vector.h> #include <utils/KeyedVector.h> #include <media/AudioTrack.h> +#include <binder/MemoryHeapBase.h> +#include <binder/MemoryBase.h> namespace android { @@ -85,6 +87,7 @@ private: int64_t mLength; char* mUrl; sp<IMemory> mData; + sp<MemoryHeapBase> mHeap; }; // stores pending events for stolen channels @@ -118,7 +121,7 @@ protected: class SoundChannel : public SoundEvent { public: enum state { IDLE, RESUMING, STOPPING, PAUSED, PLAYING }; - SoundChannel() : mAudioTrack(NULL), mState(IDLE), mNumChannels(1), + SoundChannel() : mState(IDLE), mNumChannels(1), mPos(0), mToggle(0), mAutoPaused(false) {} ~SoundChannel(); void init(SoundPool* soundPool); @@ -148,7 +151,7 @@ private: bool doStop_l(); SoundPool* mSoundPool; - AudioTrack* mAudioTrack; + sp<AudioTrack> mAudioTrack; SoundEvent mNextEvent; Mutex mLock; int mState; diff --git a/include/media/ToneGenerator.h b/include/media/ToneGenerator.h index 2183fbe..98c4332 100644 --- a/include/media/ToneGenerator.h +++ b/include/media/ToneGenerator.h @@ -160,7 +160,7 @@ public: bool isInited() { return (mState == TONE_IDLE)?false:true;} // returns the audio session this ToneGenerator belongs to or 0 if an error occured. - int getSessionId() { return (mpAudioTrack == NULL) ? 0 : mpAudioTrack->getSessionId(); } + int getSessionId() { return (mpAudioTrack == 0) ? 0 : mpAudioTrack->getSessionId(); } private: @@ -264,7 +264,7 @@ private: unsigned short mLoopCounter; // Current tone loopback count uint32_t mSamplingRate; // AudioFlinger Sampling rate - AudioTrack *mpAudioTrack; // Pointer to audio track used for playback + sp<AudioTrack> mpAudioTrack; // Pointer to audio track used for playback Mutex mLock; // Mutex to control concurent access to ToneGenerator object from audio callback and application API Mutex mCbkCondLock; // Mutex associated to mWaitCbkCond Condition mWaitCbkCond; // condition enabling interface to wait for audio callback completion after a change is requested diff --git a/include/media/Visualizer.h b/include/media/Visualizer.h index aa58905..6167dd6 100644 --- a/include/media/Visualizer.h +++ b/include/media/Visualizer.h @@ -19,7 +19,7 @@ #include <media/AudioEffect.h> #include <audio_effects/effect_visualizer.h> -#include <string.h> +#include <utils/Thread.h> /** * The Visualizer class enables application to retrieve part of the currently playing audio for @@ -114,6 +114,14 @@ public: status_t setScalingMode(uint32_t mode); uint32_t getScalingMode() { return mScalingMode; } + // set which measurements are done on the audio buffers processed by the effect. + // valid measurements (mask): MEASUREMENT_MODE_PEAK_RMS + status_t setMeasurementMode(uint32_t mode); + uint32_t getMeasurementMode() { return mMeasurementMode; } + + // return a set of int32_t measurements + status_t getIntMeasurements(uint32_t type, uint32_t number, int32_t *measurements); + // return a capture in PCM 8 bit unsigned format. The size of the capture is equal to // getCaptureSize() status_t getWaveForm(uint8_t *waveform); @@ -156,6 +164,7 @@ private: uint32_t mCaptureSize; uint32_t mSampleRate; uint32_t mScalingMode; + uint32_t mMeasurementMode; capture_cbk_t mCaptureCallBack; void *mCaptureCbkUser; sp<CaptureThread> mCaptureThread; diff --git a/include/media/mediaplayer.h b/include/media/mediaplayer.h index 14381c7..4c05fc3 100644 --- a/include/media/mediaplayer.h +++ b/include/media/mediaplayer.h @@ -42,9 +42,14 @@ enum media_event_type { MEDIA_BUFFERING_UPDATE = 3, MEDIA_SEEK_COMPLETE = 4, MEDIA_SET_VIDEO_SIZE = 5, + MEDIA_STARTED = 6, + MEDIA_PAUSED = 7, + MEDIA_STOPPED = 8, + MEDIA_SKIPPED = 9, MEDIA_TIMED_TEXT = 99, MEDIA_ERROR = 100, MEDIA_INFO = 200, + MEDIA_SUBTITLE_DATA = 201, }; // Generic error codes for the media player framework. Errors are fatal, the @@ -173,6 +178,7 @@ enum media_track_type { MEDIA_TRACK_TYPE_VIDEO = 1, MEDIA_TRACK_TYPE_AUDIO = 2, MEDIA_TRACK_TYPE_TIMEDTEXT = 3, + MEDIA_TRACK_TYPE_SUBTITLE = 4, }; // ---------------------------------------------------------------------------- @@ -218,8 +224,12 @@ public: bool isLooping(); status_t setVolume(float leftVolume, float rightVolume); void notify(int msg, int ext1, int ext2, const Parcel *obj = NULL); - static sp<IMemory> decode(const char* url, uint32_t *pSampleRate, int* pNumChannels, audio_format_t* pFormat); - static sp<IMemory> decode(int fd, int64_t offset, int64_t length, uint32_t *pSampleRate, int* pNumChannels, audio_format_t* pFormat); + static status_t decode(const char* url, uint32_t *pSampleRate, int* pNumChannels, + audio_format_t* pFormat, + const sp<IMemoryHeap>& heap, size_t *pSize); + static status_t decode(int fd, int64_t offset, int64_t length, uint32_t *pSampleRate, + int* pNumChannels, audio_format_t* pFormat, + const sp<IMemoryHeap>& heap, size_t *pSize); status_t invoke(const Parcel& request, Parcel *reply); status_t setMetadataFilter(const Parcel& filter); status_t getMetadata(bool update_only, bool apply_filter, Parcel *metadata); diff --git a/include/media/nbaio/AudioStreamOutSink.h b/include/media/nbaio/AudioStreamOutSink.h index 5976b18..7948d40 100644 --- a/include/media/nbaio/AudioStreamOutSink.h +++ b/include/media/nbaio/AudioStreamOutSink.h @@ -52,6 +52,8 @@ public: // implementation of GNWT (if any) virtual status_t getNextWriteTimestamp(int64_t *timestamp); + virtual status_t getTimestamp(AudioTimestamp& timestamp); + // NBAIO_Sink end #if 0 // until necessary diff --git a/include/media/nbaio/MonoPipe.h b/include/media/nbaio/MonoPipe.h index 5fcfe9e..d3802fe 100644 --- a/include/media/nbaio/MonoPipe.h +++ b/include/media/nbaio/MonoPipe.h @@ -20,9 +20,12 @@ #include <time.h> #include <utils/LinearTransform.h> #include "NBAIO.h" +#include <media/SingleStateQueue.h> namespace android { +typedef SingleStateQueue<AudioTimestamp> AudioTimestampSingleStateQueue; + // MonoPipe is similar to Pipe except: // - supports only a single reader, called MonoPipeReader // - write() cannot overrun; instead it will return a short actual count if insufficient space @@ -88,6 +91,9 @@ public: // Return true if the write side of a pipe is currently shutdown. bool isShutdown(); + // Return NO_ERROR if there is a timestamp available + status_t getTimestamp(AudioTimestamp& timestamp); + private: // A pair of methods and a helper variable which allows the reader and the // writer to update and observe the values of mFront and mNextRdPTS in an @@ -127,6 +133,10 @@ private: LinearTransform mSamplesToLocalTime; bool mIsShutdown; // whether shutdown(true) was called, no barriers are needed + + AudioTimestampSingleStateQueue::Shared mTimestampShared; + AudioTimestampSingleStateQueue::Mutator mTimestampMutator; + AudioTimestampSingleStateQueue::Observer mTimestampObserver; }; } // namespace android diff --git a/include/media/nbaio/MonoPipeReader.h b/include/media/nbaio/MonoPipeReader.h index 0e1c992..78fe867 100644 --- a/include/media/nbaio/MonoPipeReader.h +++ b/include/media/nbaio/MonoPipeReader.h @@ -49,6 +49,8 @@ public: virtual ssize_t read(void *buffer, size_t count, int64_t readPTS); + virtual void onTimestamp(const AudioTimestamp& timestamp); + // NBAIO_Source end #if 0 // until necessary diff --git a/include/media/nbaio/NBAIO.h b/include/media/nbaio/NBAIO.h index f5d6eb5..1da0c73 100644 --- a/include/media/nbaio/NBAIO.h +++ b/include/media/nbaio/NBAIO.h @@ -28,6 +28,7 @@ #include <stdlib.h> #include <utils/Errors.h> #include <utils/RefBase.h> +#include <media/AudioTimestamp.h> namespace android { @@ -213,6 +214,11 @@ public: // <other> Something unexpected happened internally. Check the logs and start debugging. virtual status_t getNextWriteTimestamp(int64_t *ts) { return INVALID_OPERATION; } + // Returns NO_ERROR if a timestamp is available. The timestamp includes the total number + // of frames presented to an external observer, together with the value of CLOCK_MONOTONIC + // as of this presentation count. + virtual status_t getTimestamp(AudioTimestamp& timestamp) { return INVALID_OPERATION; } + protected: NBAIO_Sink(NBAIO_Format format = Format_Invalid) : NBAIO_Port(format), mFramesWritten(0) { } virtual ~NBAIO_Sink() { } @@ -300,6 +306,10 @@ public: virtual ssize_t readVia(readVia_t via, size_t total, void *user, int64_t readPTS, size_t block = 0); + // Invoked asynchronously by corresponding sink when a new timestamp is available. + // Default implementation ignores the timestamp. + virtual void onTimestamp(const AudioTimestamp& timestamp) { } + protected: NBAIO_Source(NBAIO_Format format = Format_Invalid) : NBAIO_Port(format), mFramesRead(0) { } virtual ~NBAIO_Source() { } diff --git a/include/media/nbaio/NBLog.h b/include/media/nbaio/NBLog.h index 107ba66..6d59ea7 100644 --- a/include/media/nbaio/NBLog.h +++ b/include/media/nbaio/NBLog.h @@ -90,6 +90,8 @@ public: virtual ~Timeline(); #endif + // Input parameter 'size' is the desired size of the timeline in byte units. + // Returns the size rounded up to a power-of-2, plus the constant size overhead for indices. static size_t sharedSize(size_t size); #if 0 @@ -110,8 +112,12 @@ private: class Writer : public RefBase { public: Writer(); // dummy nop implementation without shared memory + + // Input parameter 'size' is the desired size of the timeline in byte units. + // The size of the shared memory must be at least Timeline::sharedSize(size). Writer(size_t size, void *shared); Writer(size_t size, const sp<IMemory>& iMemory); + virtual ~Writer() { } virtual void log(const char *string); @@ -165,8 +171,12 @@ private: class Reader : public RefBase { public: + + // Input parameter 'size' is the desired size of the timeline in byte units. + // The size of the shared memory must be at least Timeline::sharedSize(size). Reader(size_t size, const void *shared); Reader(size_t size, const sp<IMemory>& iMemory); + virtual ~Reader() { } void dump(int fd, size_t indent = 0); diff --git a/include/media/nbaio/SourceAudioBufferProvider.h b/include/media/nbaio/SourceAudioBufferProvider.h index c08331b..cdfb6fe 100644 --- a/include/media/nbaio/SourceAudioBufferProvider.h +++ b/include/media/nbaio/SourceAudioBufferProvider.h @@ -36,6 +36,8 @@ public: // ExtendedAudioBufferProvider interface virtual size_t framesReady() const; + virtual size_t framesReleased() const; + virtual void onTimestamp(const AudioTimestamp& timestamp); private: const sp<NBAIO_Source> mSource; // the wrapped source @@ -45,6 +47,7 @@ private: size_t mOffset; // frame offset within mAllocated of valid data size_t mRemaining; // frame count within mAllocated of valid data size_t mGetCount; // buffer.frameCount of the most recent getNextBuffer + uint32_t mFramesReleased; // counter of the total number of frames released }; } // namespace android diff --git a/include/media/stagefright/ACodec.h b/include/media/stagefright/ACodec.h index df25d7b..a8ffd4a 100644 --- a/include/media/stagefright/ACodec.h +++ b/include/media/stagefright/ACodec.h @@ -124,7 +124,8 @@ private: }; enum { - kFlagIsSecure = 1, + kFlagIsSecure = 1, + kFlagPushBlankBuffersToNativeWindowOnShutdown = 2, }; struct BufferInfo { @@ -138,6 +139,7 @@ private: IOMX::buffer_id mBufferID; Status mStatus; + unsigned mDequeuedAt; sp<ABuffer> mData; sp<GraphicBuffer> mGraphicBuffer; @@ -182,7 +184,7 @@ private: bool mSentFormat; bool mIsEncoder; - + bool mUseMetadataOnEncoderOutput; bool mShutdownInProgress; // If "mKeepComponentAllocated" we only transition back to Loaded state @@ -194,12 +196,22 @@ private: bool mChannelMaskPresent; int32_t mChannelMask; + unsigned mDequeueCounter; + bool mStoreMetaDataInOutputBuffers; + int32_t mMetaDataBuffersToSubmit; + + int64_t mRepeatFrameDelayUs; status_t setCyclicIntraMacroblockRefresh(const sp<AMessage> &msg, int32_t mode); status_t allocateBuffersOnPort(OMX_U32 portIndex); status_t freeBuffersOnPort(OMX_U32 portIndex); status_t freeBuffer(OMX_U32 portIndex, size_t i); + status_t configureOutputBuffersFromNativeWindow( + OMX_U32 *nBufferCount, OMX_U32 *nBufferSize, + OMX_U32 *nMinUndequeuedBuffers); + status_t allocateOutputMetaDataBuffers(); + status_t submitOutputMetaDataBuffer(); status_t allocateOutputBuffersFromNativeWindow(); status_t cancelBufferToNativeWindow(BufferInfo *info); status_t freeOutputBuffersNotOwnedByComponent(); diff --git a/include/media/stagefright/AudioPlayer.h b/include/media/stagefright/AudioPlayer.h index 1dc408f..912a43c 100644 --- a/include/media/stagefright/AudioPlayer.h +++ b/include/media/stagefright/AudioPlayer.h @@ -36,8 +36,16 @@ public: SEEK_COMPLETE }; + enum { + ALLOW_DEEP_BUFFERING = 0x01, + USE_OFFLOAD = 0x02, + HAS_VIDEO = 0x1000, + IS_STREAMING = 0x2000 + + }; + AudioPlayer(const sp<MediaPlayerBase::AudioSink> &audioSink, - bool allowDeepBuffering = false, + uint32_t flags = 0, AwesomePlayer *audioObserver = NULL); virtual ~AudioPlayer(); @@ -51,7 +59,7 @@ public: status_t start(bool sourceAlreadyStarted = false); void pause(bool playPendingSamples = false); - void resume(); + status_t resume(); // Returns the timestamp of the last buffer played (in us). int64_t getMediaTimeUs(); @@ -67,10 +75,12 @@ public: status_t setPlaybackRatePermille(int32_t ratePermille); + void notifyAudioEOS(); + private: friend class VideoEditorAudioPlayer; sp<MediaSource> mSource; - AudioTrack *mAudioTrack; + sp<AudioTrack> mAudioTrack; MediaBuffer *mInputBuffer; @@ -97,17 +107,20 @@ private: MediaBuffer *mFirstBuffer; sp<MediaPlayerBase::AudioSink> mAudioSink; - bool mAllowDeepBuffering; // allow audio deep audio buffers. Helps with low power audio - // playback but implies high latency AwesomePlayer *mObserver; int64_t mPinnedTimeUs; + bool mPlaying; + int64_t mStartPosUs; + const uint32_t mCreateFlags; + static void AudioCallback(int event, void *user, void *info); void AudioCallback(int event, void *info); static size_t AudioSinkCallback( MediaPlayerBase::AudioSink *audioSink, - void *data, size_t size, void *me); + void *data, size_t size, void *me, + MediaPlayerBase::AudioSink::cb_event_t event); size_t fillBuffer(void *data, size_t size); @@ -116,6 +129,10 @@ private: void reset(); uint32_t getNumFramesPendingPlayout() const; + int64_t getOutputPlayPositionUs_l() const; + + bool allowDeepBuffering() const { return (mCreateFlags & ALLOW_DEEP_BUFFERING) != 0; } + bool useOffload() const { return (mCreateFlags & USE_OFFLOAD) != 0; } AudioPlayer(const AudioPlayer &); AudioPlayer &operator=(const AudioPlayer &); diff --git a/include/media/stagefright/AudioSource.h b/include/media/stagefright/AudioSource.h index 99f3c3b..4c9aaad 100644 --- a/include/media/stagefright/AudioSource.h +++ b/include/media/stagefright/AudioSource.h @@ -73,7 +73,7 @@ private: Condition mFrameAvailableCondition; Condition mFrameEncodingCompletionCondition; - AudioRecord *mRecord; + sp<AudioRecord> mRecord; status_t mInitCheck; bool mStarted; int32_t mSampleRate; diff --git a/include/media/stagefright/MediaCodecList.h b/include/media/stagefright/MediaCodecList.h index dfb845b..590623b 100644 --- a/include/media/stagefright/MediaCodecList.h +++ b/include/media/stagefright/MediaCodecList.h @@ -50,7 +50,8 @@ struct MediaCodecList { status_t getCodecCapabilities( size_t index, const char *type, Vector<ProfileLevel> *profileLevels, - Vector<uint32_t> *colorFormats) const; + Vector<uint32_t> *colorFormats, + uint32_t *flags) const; private: enum Section { diff --git a/include/media/stagefright/MediaDefs.h b/include/media/stagefright/MediaDefs.h index 81de6e4..85693d4 100644 --- a/include/media/stagefright/MediaDefs.h +++ b/include/media/stagefright/MediaDefs.h @@ -22,7 +22,8 @@ namespace android { extern const char *MEDIA_MIMETYPE_IMAGE_JPEG; -extern const char *MEDIA_MIMETYPE_VIDEO_VPX; +extern const char *MEDIA_MIMETYPE_VIDEO_VP8; +extern const char *MEDIA_MIMETYPE_VIDEO_VP9; extern const char *MEDIA_MIMETYPE_VIDEO_AVC; extern const char *MEDIA_MIMETYPE_VIDEO_MPEG4; extern const char *MEDIA_MIMETYPE_VIDEO_H263; diff --git a/include/media/stagefright/MediaErrors.h b/include/media/stagefright/MediaErrors.h index ee5e4e2..686f286 100644 --- a/include/media/stagefright/MediaErrors.h +++ b/include/media/stagefright/MediaErrors.h @@ -56,14 +56,11 @@ enum { ERROR_DRM_TAMPER_DETECTED = DRM_ERROR_BASE - 7, ERROR_DRM_NOT_PROVISIONED = DRM_ERROR_BASE - 8, ERROR_DRM_DEVICE_REVOKED = DRM_ERROR_BASE - 9, + ERROR_DRM_RESOURCE_BUSY = DRM_ERROR_BASE - 10, ERROR_DRM_VENDOR_MAX = DRM_ERROR_BASE - 500, ERROR_DRM_VENDOR_MIN = DRM_ERROR_BASE - 999, - // Deprecated - ERROR_DRM_WV_VENDOR_MAX = ERROR_DRM_VENDOR_MAX, - ERROR_DRM_WV_VENDOR_MIN = ERROR_DRM_VENDOR_MIN, - // Heartbeat Error Codes HEARTBEAT_ERROR_BASE = -3000, ERROR_HEARTBEAT_TERMINATE_REQUESTED = HEARTBEAT_ERROR_BASE, diff --git a/include/media/stagefright/MediaMuxer.h b/include/media/stagefright/MediaMuxer.h index c1fdbad..ff6a66e 100644 --- a/include/media/stagefright/MediaMuxer.h +++ b/include/media/stagefright/MediaMuxer.h @@ -79,6 +79,16 @@ public: status_t setOrientationHint(int degrees); /** + * Set the location. + * @param latitude The latitude in degree x 1000. Its value must be in the range + * [-900000, 900000]. + * @param longitude The longitude in degree x 1000. Its value must be in the range + * [-1800000, 1800000]. + * @return OK if no error. + */ + status_t setLocation(int latitude, int longitude); + + /** * Stop muxing. * This method is a blocking call. Depending on how * much data is bufferred internally, the time needed for stopping diff --git a/include/media/stagefright/OMXCodec.h b/include/media/stagefright/OMXCodec.h index 583c3b3..daaf20f 100644 --- a/include/media/stagefright/OMXCodec.h +++ b/include/media/stagefright/OMXCodec.h @@ -361,9 +361,14 @@ private: }; struct CodecCapabilities { + enum { + kFlagSupportsAdaptivePlayback = 1 << 0, + }; + String8 mComponentName; Vector<CodecProfileLevel> mProfileLevels; Vector<OMX_U32> mColorFormats; + uint32_t mFlags; }; // Return a vector of componentNames with supported profile/level pairs diff --git a/include/media/stagefright/SurfaceMediaSource.h b/include/media/stagefright/SurfaceMediaSource.h index 5f21da9..db5f947 100644 --- a/include/media/stagefright/SurfaceMediaSource.h +++ b/include/media/stagefright/SurfaceMediaSource.h @@ -56,7 +56,7 @@ class GraphicBuffer; class SurfaceMediaSource : public MediaSource, public MediaBufferObserver, - protected BufferQueue::ConsumerListener { + protected ConsumerListener { public: enum { MIN_UNDEQUEUED_BUFFERS = 4}; @@ -146,9 +146,13 @@ private: // this consumer sp<BufferQueue> mBufferQueue; - // mBufferSlot caches GraphicBuffers from the buffer queue - sp<GraphicBuffer> mBufferSlot[BufferQueue::NUM_BUFFER_SLOTS]; + struct SlotData { + sp<GraphicBuffer> mGraphicBuffer; + uint64_t mFrameNumber; + }; + // mSlots caches GraphicBuffers and frameNumbers from the buffer queue + SlotData mSlots[BufferQueue::NUM_BUFFER_SLOTS]; // The permenent width and height of SMS buffers int mWidth; diff --git a/include/media/stagefright/Utils.h b/include/media/stagefright/Utils.h index 73940d3..c24f612 100644 --- a/include/media/stagefright/Utils.h +++ b/include/media/stagefright/Utils.h @@ -22,6 +22,8 @@ #include <stdint.h> #include <utils/Errors.h> #include <utils/RefBase.h> +#include <system/audio.h> +#include <media/MediaPlayerInterface.h> namespace android { @@ -48,6 +50,15 @@ void convertMessageToMetaData( AString MakeUserAgent(); +// Convert a MIME type to a AudioSystem::audio_format +status_t mapMimeToAudioFormat(audio_format_t& format, const char* mime); + +// Send information from MetaData to the HAL via AudioSink +status_t sendMetaDataToHal(sp<MediaPlayerBase::AudioSink>& sink, const sp<MetaData>& meta); + +// Check whether the stream defined by meta can be offloaded to hardware +bool canOffloadStream(const sp<MetaData>& meta, bool hasVideo, bool isStreaming); + } // namespace android #endif // UTILS_H_ diff --git a/include/media/stagefright/foundation/ALooperRoster.h b/include/media/stagefright/foundation/ALooperRoster.h index 2e5fd73..940fc55 100644 --- a/include/media/stagefright/foundation/ALooperRoster.h +++ b/include/media/stagefright/foundation/ALooperRoster.h @@ -30,6 +30,7 @@ struct ALooperRoster { const sp<ALooper> looper, const sp<AHandler> &handler); void unregisterHandler(ALooper::handler_id handlerID); + void unregisterStaleHandlers(); status_t postMessage(const sp<AMessage> &msg, int64_t delayUs = 0); void deliverMessage(const sp<AMessage> &msg); diff --git a/media/libstagefright/wifi-display/ANetworkSession.h b/include/media/stagefright/foundation/ANetworkSession.h index 7c62b29..fd3ebaa 100644 --- a/media/libstagefright/wifi-display/ANetworkSession.h +++ b/include/media/stagefright/foundation/ANetworkSession.h @@ -77,6 +77,8 @@ struct ANetworkSession : public RefBase { int32_t sessionID, const void *data, ssize_t size = -1, bool timeValid = false, int64_t timeUs = -1ll); + status_t switchToWebSocketMode(int32_t sessionID); + enum NotificationReason { kWhatError, kWhatConnected, @@ -84,6 +86,7 @@ struct ANetworkSession : public RefBase { kWhatData, kWhatDatagram, kWhatBinaryData, + kWhatWebSocketMessage, kWhatNetworkStall, }; diff --git a/media/libstagefright/wifi-display/ParsedMessage.h b/include/media/stagefright/foundation/ParsedMessage.h index e9a1859..9d43a93 100644 --- a/media/libstagefright/wifi-display/ParsedMessage.h +++ b/include/media/stagefright/foundation/ParsedMessage.h @@ -32,7 +32,7 @@ struct ParsedMessage : public RefBase { const char *getContent() const; - void getRequestField(size_t index, AString *field) const; + bool getRequestField(size_t index, AString *field) const; bool getStatusCode(int32_t *statusCode) const; AString debugString() const; diff --git a/include/private/media/AudioTrackShared.h b/include/private/media/AudioTrackShared.h index 41e20f8..395f164 100644 --- a/include/private/media/AudioTrackShared.h +++ b/include/private/media/AudioTrackShared.h @@ -22,32 +22,51 @@ #include <utils/threads.h> #include <utils/Log.h> +#include <utils/RefBase.h> +#include <media/nbaio/roundup.h> +#include <media/SingleStateQueue.h> +#include <private/media/StaticAudioTrackState.h> namespace android { // ---------------------------------------------------------------------------- -// Maximum cumulated timeout milliseconds before restarting audioflinger thread -#define MAX_STARTUP_TIMEOUT_MS 3000 // Longer timeout period at startup to cope with A2DP - // init time -#define MAX_RUN_TIMEOUT_MS 1000 -#define WAIT_PERIOD_MS 10 - -#define CBLK_UNDERRUN 0x01 // set: underrun (out) or overrrun (in), clear: no underrun or overrun +// for audio_track_cblk_t::mFlags +#define CBLK_UNDERRUN 0x01 // set by server immediately on output underrun, cleared by client #define CBLK_FORCEREADY 0x02 // set: track is considered ready immediately by AudioFlinger, // clear: track is ready when buffer full #define CBLK_INVALID 0x04 // track buffer invalidated by AudioFlinger, need to re-create -#define CBLK_DISABLED 0x08 // track disabled by AudioFlinger due to underrun, need to re-start +#define CBLK_DISABLED 0x08 // output track disabled by AudioFlinger due to underrun, + // need to re-start. Unlike CBLK_UNDERRUN, this is not set + // immediately, but only after a long string of underruns. +// 0x10 unused +#define CBLK_LOOP_CYCLE 0x20 // set by server each time a loop cycle other than final one completes +#define CBLK_LOOP_FINAL 0x40 // set by server when the final loop cycle completes +#define CBLK_BUFFER_END 0x80 // set by server when the position reaches end of buffer if not looping +#define CBLK_OVERRUN 0x100 // set by server immediately on input overrun, cleared by client +#define CBLK_INTERRUPT 0x200 // set by client on interrupt(), cleared by client in obtainBuffer() +#define CBLK_STREAM_END_DONE 0x400 // set by server on render completion, cleared by client + +//EL_FIXME 20 seconds may not be enough and must be reconciled with new obtainBuffer implementation +#define MAX_RUN_OFFLOADED_TIMEOUT_MS 20000 //assuming upto a maximum of 20 seconds of offloaded struct AudioTrackSharedStreaming { // similar to NBAIO MonoPipe - volatile int32_t mFront; - volatile int32_t mRear; + // in continuously incrementing frame units, take modulo buffer size, which must be a power of 2 + volatile int32_t mFront; // read by server + volatile int32_t mRear; // write by client + volatile int32_t mFlush; // incremented by client to indicate a request to flush; + // server notices and discards all data between mFront and mRear + volatile uint32_t mUnderrunFrames; // server increments for each unavailable but desired frame }; -// future +typedef SingleStateQueue<StaticAudioTrackState> StaticAudioTrackSingleStateQueue; + struct AudioTrackSharedStatic { - int mReserved; + StaticAudioTrackSingleStateQueue::Shared + mSingleStateQueue; + size_t mBufferPosition; // updated asynchronously by server, + // "for entertainment purposes only" }; // ---------------------------------------------------------------------------- @@ -55,65 +74,63 @@ struct AudioTrackSharedStatic { // Important: do not add any virtual methods, including ~ struct audio_track_cblk_t { + // Since the control block is always located in shared memory, this constructor + // is only used for placement new(). It is never used for regular new() or stack. + audio_track_cblk_t(); + /*virtual*/ ~audio_track_cblk_t() { } + friend class Proxy; + friend class ClientProxy; friend class AudioTrackClientProxy; friend class AudioRecordClientProxy; friend class ServerProxy; + friend class AudioTrackServerProxy; + friend class AudioRecordServerProxy; // The data members are grouped so that members accessed frequently and in the same context // are in the same line of data cache. - Mutex lock; // sizeof(int) - Condition cv; // sizeof(int) - // next 4 are offsets within "buffers" - volatile uint32_t user; - volatile uint32_t server; - uint32_t userBase; - uint32_t serverBase; - - int mPad1; // unused, but preserves cache line alignment + uint32_t mServer; // Number of filled frames consumed by server (mIsOut), + // or filled frames provided by server (!mIsOut). + // It is updated asynchronously by server without a barrier. + // The value should be used "for entertainment purposes only", + // which means don't make important decisions based on it. size_t frameCount_; // used during creation to pass actual track buffer size // from AudioFlinger to client, and not referenced again - // FIXME remove here and replace by createTrack() in/out parameter + // FIXME remove here and replace by createTrack() in/out + // parameter // renamed to "_" to detect incorrect use - // Cache line boundary (32 bytes) + volatile int32_t mFutex; // event flag: down (P) by client, + // up (V) by server or binderDied() or interrupt() +#define CBLK_FUTEX_WAKE 1 // if event flag bit is set, then a deferred wake is pending - uint32_t loopStart; - uint32_t loopEnd; // read-only for server, read/write for client - int loopCount; // read/write for client +private: + + size_t mMinimum; // server wakes up client if available >= mMinimum // Channel volumes are fixed point U4.12, so 0x1000 means 1.0. // Left channel is in [0:15], right channel is in [16:31]. // Always read and write the combined pair atomically. // For AudioTrack only, not used by AudioRecord. -private: uint32_t mVolumeLR; uint32_t mSampleRate; // AudioTrack only: client's requested sample rate in Hz // or 0 == default. Write-only client, read-only server. - uint8_t mPad2; // unused - -public: - // read-only for client, server writes once at initialization and is then read-only - uint8_t mName; // normal tracks: track name, fast tracks: track index - - // used by client only - uint16_t bufferTimeoutMs; // Maximum cumulated timeout before restarting - // audioflinger - - uint16_t waitTimeMs; // Cumulated wait time, used by client only -private: // client write-only, server read-only uint16_t mSendLevel; // Fixed point U4.12 so 0x1000 means 1.0 + + uint16_t mPad2; // unused + public: - volatile int32_t flags; + + volatile int32_t mFlags; // combinations of CBLK_* // Cache line boundary (32 bytes) -#if 0 +public: union { AudioTrackSharedStreaming mStreaming; AudioTrackSharedStatic mStatic; @@ -121,25 +138,6 @@ public: } u; // Cache line boundary (32 bytes) -#endif - - // Since the control block is always located in shared memory, this constructor - // is only used for placement new(). It is never used for regular new() or stack. - audio_track_cblk_t(); - -private: - // if there is a shared buffer, "buffers" is the value of pointer() for the shared - // buffer, otherwise "buffers" points immediately after the control block - void* buffer(void *buffers, uint32_t frameSize, size_t offset) const; - - bool tryLock(); - - // isOut == true means AudioTrack, isOut == false means AudioRecord - bool stepServer(size_t stepCount, size_t frameCount, bool isOut); - uint32_t stepUser(size_t stepCount, size_t frameCount, bool isOut); - uint32_t framesAvailable(size_t frameCount, bool isOut); - uint32_t framesAvailable_l(size_t frameCount, bool isOut); - uint32_t framesReady(bool isOut); }; // ---------------------------------------------------------------------------- @@ -147,29 +145,32 @@ private: // Proxy for shared memory control block, to isolate callers from needing to know the details. // There is exactly one ClientProxy and one ServerProxy per shared memory control block. // The proxies are located in normal memory, and are not multi-thread safe within a given side. -class Proxy { +class Proxy : public RefBase { protected: - Proxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, size_t frameSize) - : mCblk(cblk), mBuffers(buffers), mFrameCount(frameCount), mFrameSize(frameSize) { } + Proxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, size_t frameSize, bool isOut, + bool clientInServer); virtual ~Proxy() { } public: - void* buffer(size_t offset) const { - return mCblk->buffer(mBuffers, mFrameSize, offset); - } + struct Buffer { + size_t mFrameCount; // number of frames available in this buffer + void* mRaw; // pointer to first frame + size_t mNonContig; // number of additional non-contiguous frames available + }; protected: // These refer to shared memory, and are virtual addresses with respect to the current process. // They may have different virtual addresses within the other process. - audio_track_cblk_t* const mCblk; // the control block - void* const mBuffers; // starting address of buffers - - const size_t mFrameCount; // not necessarily a power of 2 - const size_t mFrameSize; // in bytes -#if 0 - const size_t mFrameCountP2; // mFrameCount rounded to power of 2, streaming mode -#endif - + audio_track_cblk_t* const mCblk; // the control block + void* const mBuffers; // starting address of buffers + + const size_t mFrameCount; // not necessarily a power of 2 + const size_t mFrameSize; // in bytes + const size_t mFrameCountP2; // mFrameCount rounded to power of 2, streaming mode + const bool mIsOut; // true for AudioTrack, false for AudioRecord + const bool mClientInServer; // true for OutputTrack, false for AudioTrack & AudioRecord + bool mIsShutdown; // latch set to true when shared memory corruption detected + size_t mUnreleased; // unreleased frames remaining from most recent obtainBuffer }; // ---------------------------------------------------------------------------- @@ -177,9 +178,88 @@ protected: // Proxy seen by AudioTrack client and AudioRecord client class ClientProxy : public Proxy { protected: - ClientProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, size_t frameSize) - : Proxy(cblk, buffers, frameCount, frameSize) { } + ClientProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, size_t frameSize, + bool isOut, bool clientInServer); virtual ~ClientProxy() { } + +public: + static const struct timespec kForever; + static const struct timespec kNonBlocking; + + // Obtain a buffer with filled frames (reading) or empty frames (writing). + // It is permitted to call obtainBuffer() multiple times in succession, without any intervening + // calls to releaseBuffer(). In that case, the final obtainBuffer() is the one that effectively + // sets or extends the unreleased frame count. + // On entry: + // buffer->mFrameCount should be initialized to maximum number of desired frames, + // which must be > 0. + // buffer->mNonContig is unused. + // buffer->mRaw is unused. + // requested is the requested timeout in local monotonic delta time units: + // NULL or &kNonBlocking means non-blocking (zero timeout). + // &kForever means block forever (infinite timeout). + // Other values mean a specific timeout in local monotonic delta time units. + // elapsed is a pointer to a location that will hold the total local monotonic time that + // elapsed while blocked, or NULL if not needed. + // On exit: + // buffer->mFrameCount has the actual number of contiguous available frames, + // which is always 0 when the return status != NO_ERROR. + // buffer->mNonContig is the number of additional non-contiguous available frames. + // buffer->mRaw is a pointer to the first available frame, + // or NULL when buffer->mFrameCount == 0. + // The return status is one of: + // NO_ERROR Success, buffer->mFrameCount > 0. + // WOULD_BLOCK Non-blocking mode and no frames are available. + // TIMED_OUT Timeout occurred before any frames became available. + // This can happen even for infinite timeout, due to a spurious wakeup. + // In this case, the caller should investigate and then re-try as appropriate. + // DEAD_OBJECT Server has died or invalidated, caller should destroy this proxy and re-create. + // -EINTR Call has been interrupted. Look around to see why, and then perhaps try again. + // NO_INIT Shared memory is corrupt. + // Assertion failure on entry, if buffer == NULL or buffer->mFrameCount == 0. + status_t obtainBuffer(Buffer* buffer, const struct timespec *requested = NULL, + struct timespec *elapsed = NULL); + + // Release (some of) the frames last obtained. + // On entry, buffer->mFrameCount should have the number of frames to release, + // which must (cumulatively) be <= the number of frames last obtained but not yet released. + // buffer->mRaw is ignored, but is normally same pointer returned by last obtainBuffer(). + // It is permitted to call releaseBuffer() multiple times to release the frames in chunks. + // On exit: + // buffer->mFrameCount is zero. + // buffer->mRaw is NULL. + void releaseBuffer(Buffer* buffer); + + // Call after detecting server's death + void binderDied(); + + // Call to force an obtainBuffer() to return quickly with -EINTR + void interrupt(); + + size_t getPosition() { + return mEpoch + mCblk->mServer; + } + + void setEpoch(size_t epoch) { + mEpoch = epoch; + } + + void setMinimum(size_t minimum) { + mCblk->mMinimum = minimum; + } + + // Return the number of frames that would need to be obtained and released + // in order for the client to be aligned at start of buffer + virtual size_t getMisalignment(); + + size_t getEpoch() const { + return mEpoch; + } + + size_t getFramesFilled(); + +private: + size_t mEpoch; }; // ---------------------------------------------------------------------------- @@ -187,8 +267,10 @@ protected: // Proxy used by AudioTrack client, which also includes AudioFlinger::PlaybackThread::OutputTrack class AudioTrackClientProxy : public ClientProxy { public: - AudioTrackClientProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, size_t frameSize) - : ClientProxy(cblk, buffers, frameCount, frameSize) { } + AudioTrackClientProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, + size_t frameSize, bool clientInServer = false) + : ClientProxy(cblk, buffers, frameCount, frameSize, true /*isOut*/, + clientInServer) { } virtual ~AudioTrackClientProxy() { } // No barriers on the following operations, so the ordering of loads/stores @@ -208,27 +290,42 @@ public: mCblk->mSampleRate = sampleRate; } - // called by: - // PlaybackThread::OutputTrack::write - // AudioTrack::createTrack_l - // AudioTrack::releaseBuffer - // AudioTrack::reload - // AudioTrack::restoreTrack_l (2 places) - size_t stepUser(size_t stepCount) { - return mCblk->stepUser(stepCount, mFrameCount, true /*isOut*/); + virtual void flush(); + + virtual uint32_t getUnderrunFrames() const { + return mCblk->u.mStreaming.mUnderrunFrames; } - // called by AudioTrack::obtainBuffer and AudioTrack::processBuffer - size_t framesAvailable() { - return mCblk->framesAvailable(mFrameCount, true /*isOut*/); + bool clearStreamEndDone(); // and return previous value + + bool getStreamEndDone() const; + + status_t waitStreamEndDone(const struct timespec *requested); +}; + +class StaticAudioTrackClientProxy : public AudioTrackClientProxy { +public: + StaticAudioTrackClientProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, + size_t frameSize); + virtual ~StaticAudioTrackClientProxy() { } + + virtual void flush(); + +#define MIN_LOOP 16 // minimum length of each loop iteration in frames + void setLoop(size_t loopStart, size_t loopEnd, int loopCount); + size_t getBufferPosition(); + + virtual size_t getMisalignment() { + return 0; } - // called by AudioTrack::obtainBuffer and PlaybackThread::OutputTrack::obtainBuffer - // FIXME remove this API since it assumes a lock that should be invisible to caller - size_t framesAvailable_l() { - return mCblk->framesAvailable_l(mFrameCount, true /*isOut*/); + virtual uint32_t getUnderrunFrames() const { + return 0; } +private: + StaticAudioTrackSingleStateQueue::Mutator mMutator; + size_t mBufferPosition; // so that getBufferPosition() appears to be synchronous }; // ---------------------------------------------------------------------------- @@ -236,60 +333,133 @@ public: // Proxy used by AudioRecord client class AudioRecordClientProxy : public ClientProxy { public: - AudioRecordClientProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, size_t frameSize) - : ClientProxy(cblk, buffers, frameCount, frameSize) { } + AudioRecordClientProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, + size_t frameSize) + : ClientProxy(cblk, buffers, frameCount, frameSize, + false /*isOut*/, false /*clientInServer*/) { } ~AudioRecordClientProxy() { } - - // called by AudioRecord::releaseBuffer - size_t stepUser(size_t stepCount) { - return mCblk->stepUser(stepCount, mFrameCount, false /*isOut*/); - } - - // called by AudioRecord::processBuffer - size_t framesAvailable() { - return mCblk->framesAvailable(mFrameCount, false /*isOut*/); - } - - // called by AudioRecord::obtainBuffer - size_t framesReady() { - return mCblk->framesReady(false /*isOut*/); - } - }; // ---------------------------------------------------------------------------- // Proxy used by AudioFlinger server class ServerProxy : public Proxy { +protected: + ServerProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, size_t frameSize, + bool isOut, bool clientInServer); public: - ServerProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, size_t frameSize, bool isOut) - : Proxy(cblk, buffers, frameCount, frameSize), mIsOut(isOut) { } virtual ~ServerProxy() { } - // for AudioTrack and AudioRecord - bool step(size_t stepCount) { return mCblk->stepServer(stepCount, mFrameCount, mIsOut); } + // Obtain a buffer with filled frames (writing) or empty frames (reading). + // It is permitted to call obtainBuffer() multiple times in succession, without any intervening + // calls to releaseBuffer(). In that case, the final obtainBuffer() is the one that effectively + // sets or extends the unreleased frame count. + // Always non-blocking. + // On entry: + // buffer->mFrameCount should be initialized to maximum number of desired frames, + // which must be > 0. + // buffer->mNonContig is unused. + // buffer->mRaw is unused. + // On exit: + // buffer->mFrameCount has the actual number of contiguous available frames, + // which is always 0 when the return status != NO_ERROR. + // buffer->mNonContig is the number of additional non-contiguous available frames. + // buffer->mRaw is a pointer to the first available frame, + // or NULL when buffer->mFrameCount == 0. + // The return status is one of: + // NO_ERROR Success, buffer->mFrameCount > 0. + // WOULD_BLOCK No frames are available. + // NO_INIT Shared memory is corrupt. + virtual status_t obtainBuffer(Buffer* buffer); + + // Release (some of) the frames last obtained. + // On entry, buffer->mFrameCount should have the number of frames to release, + // which must (cumulatively) be <= the number of frames last obtained but not yet released. + // It is permitted to call releaseBuffer() multiple times to release the frames in chunks. + // buffer->mRaw is ignored, but is normally same pointer returned by last obtainBuffer(). + // On exit: + // buffer->mFrameCount is zero. + // buffer->mRaw is NULL. + virtual void releaseBuffer(Buffer* buffer); +protected: + size_t mAvailToClient; // estimated frames available to client prior to releaseBuffer() + int32_t mFlush; // our copy of cblk->u.mStreaming.mFlush, for streaming output only +}; + +// Proxy used by AudioFlinger for servicing AudioTrack +class AudioTrackServerProxy : public ServerProxy { +public: + AudioTrackServerProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, + size_t frameSize, bool clientInServer = false) + : ServerProxy(cblk, buffers, frameCount, frameSize, true /*isOut*/, clientInServer) { } +protected: + virtual ~AudioTrackServerProxy() { } + +public: // return value of these methods must be validated by the caller uint32_t getSampleRate() const { return mCblk->mSampleRate; } uint16_t getSendLevel_U4_12() const { return mCblk->mSendLevel; } uint32_t getVolumeLR() const { return mCblk->mVolumeLR; } - // for AudioTrack only - size_t framesReady() { - ALOG_ASSERT(mIsOut); - return mCblk->framesReady(true); - } + // estimated total number of filled frames available to server to read, + // which may include non-contiguous frames + virtual size_t framesReady(); - // for AudioRecord only, called by RecordThread::RecordTrack::getNextBuffer - // FIXME remove this API since it assumes a lock that should be invisible to caller - size_t framesAvailableIn_l() { - ALOG_ASSERT(!mIsOut); - return mCblk->framesAvailable_l(mFrameCount, false); - } + // Currently AudioFlinger will call framesReady() for a fast track from two threads: + // FastMixer thread, and normal mixer thread. This is dangerous, as the proxy is intended + // to be called from at most one thread of server, and one thread of client. + // As a temporary workaround, this method informs the proxy implementation that it + // should avoid doing a state queue poll from within framesReady(). + // FIXME Change AudioFlinger to not call framesReady() from normal mixer thread. + virtual void framesReadyIsCalledByMultipleThreads() { } + + bool setStreamEndDone(); // and return previous value + + // Add to the tally of underrun frames, and inform client of underrun + virtual void tallyUnderrunFrames(uint32_t frameCount); + + // Return the total number of frames which AudioFlinger desired but were unavailable, + // and thus which resulted in an underrun. + virtual uint32_t getUnderrunFrames() const { return mCblk->u.mStreaming.mUnderrunFrames; } + + // Return the total number of frames that AudioFlinger has obtained and released + virtual size_t framesReleased() const { return mCblk->mServer; } +}; + +class StaticAudioTrackServerProxy : public AudioTrackServerProxy { +public: + StaticAudioTrackServerProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, + size_t frameSize); +protected: + virtual ~StaticAudioTrackServerProxy() { } + +public: + virtual size_t framesReady(); + virtual void framesReadyIsCalledByMultipleThreads(); + virtual status_t obtainBuffer(Buffer* buffer); + virtual void releaseBuffer(Buffer* buffer); + virtual void tallyUnderrunFrames(uint32_t frameCount); + virtual uint32_t getUnderrunFrames() const { return 0; } private: - const bool mIsOut; // true for AudioTrack, false for AudioRecord + ssize_t pollPosition(); // poll for state queue update, and return current position + StaticAudioTrackSingleStateQueue::Observer mObserver; + size_t mPosition; // server's current play position in frames, relative to 0 + size_t mEnd; // cached value computed from mState, safe for asynchronous read + bool mFramesReadyIsCalledByMultipleThreads; + StaticAudioTrackState mState; +}; +// Proxy used by AudioFlinger for servicing AudioRecord +class AudioRecordServerProxy : public ServerProxy { +public: + AudioRecordServerProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, + size_t frameSize) + : ServerProxy(cblk, buffers, frameCount, frameSize, false /*isOut*/, + false /*clientInServer*/) { } +protected: + virtual ~AudioRecordServerProxy() { } }; // ---------------------------------------------------------------------------- diff --git a/libvideoeditor/lvpp/NativeWindowRenderer.cpp b/libvideoeditor/lvpp/NativeWindowRenderer.cpp index 702900b..8b362ef 100755 --- a/libvideoeditor/lvpp/NativeWindowRenderer.cpp +++ b/libvideoeditor/lvpp/NativeWindowRenderer.cpp @@ -568,8 +568,9 @@ void NativeWindowRenderer::destroyRenderInput(RenderInput* input) { RenderInput::RenderInput(NativeWindowRenderer* renderer, GLuint textureId) : mRenderer(renderer) , mTextureId(textureId) { - mST = new GLConsumer(mTextureId); - mSTC = new Surface(mST->getBufferQueue()); + sp<BufferQueue> bq = new BufferQueue(); + mST = new GLConsumer(bq, mTextureId); + mSTC = new Surface(bq); native_window_connect(mSTC.get(), NATIVE_WINDOW_API_MEDIA); } diff --git a/libvideoeditor/lvpp/VideoEditorAudioPlayer.cpp b/libvideoeditor/lvpp/VideoEditorAudioPlayer.cpp index c111ba8..176f8e9 100755 --- a/libvideoeditor/lvpp/VideoEditorAudioPlayer.cpp +++ b/libvideoeditor/lvpp/VideoEditorAudioPlayer.cpp @@ -35,8 +35,7 @@ namespace android { VideoEditorAudioPlayer::VideoEditorAudioPlayer( const sp<MediaPlayerBase::AudioSink> &audioSink, PreviewPlayer *observer) - : mAudioTrack(NULL), - mInputBuffer(NULL), + : mInputBuffer(NULL), mSampleRate(0), mLatencyUs(0), mFrameSize(0), @@ -111,8 +110,7 @@ void VideoEditorAudioPlayer::clear() { } else { mAudioTrack->stop(); - delete mAudioTrack; - mAudioTrack = NULL; + mAudioTrack.clear(); } // Make sure to release any buffer we hold onto so that the @@ -151,7 +149,7 @@ void VideoEditorAudioPlayer::clear() { mStarted = false; } -void VideoEditorAudioPlayer::resume() { +status_t VideoEditorAudioPlayer::resume() { ALOGV("resume"); AudioMixSettings audioMixSettings; @@ -182,6 +180,7 @@ void VideoEditorAudioPlayer::resume() { } else { mAudioTrack->start(); } + return OK; } status_t VideoEditorAudioPlayer::seekTo(int64_t time_us) { @@ -538,8 +537,7 @@ status_t VideoEditorAudioPlayer::start(bool sourceAlreadyStarted) { 0, AUDIO_OUTPUT_FLAG_NONE, &AudioCallback, this, 0); if ((err = mAudioTrack->initCheck()) != OK) { - delete mAudioTrack; - mAudioTrack = NULL; + mAudioTrack.clear(); if (mFirstBuffer != NULL) { mFirstBuffer->release(); @@ -578,10 +576,15 @@ void VideoEditorAudioPlayer::reset() { size_t VideoEditorAudioPlayer::AudioSinkCallback( MediaPlayerBase::AudioSink *audioSink, - void *buffer, size_t size, void *cookie) { + void *buffer, size_t size, void *cookie, + MediaPlayerBase::AudioSink::cb_event_t event) { VideoEditorAudioPlayer *me = (VideoEditorAudioPlayer *)cookie; - return me->fillBuffer(buffer, size); + if (event == MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER ) { + return me->fillBuffer(buffer, size); + } else { + return 0; + } } diff --git a/libvideoeditor/lvpp/VideoEditorAudioPlayer.h b/libvideoeditor/lvpp/VideoEditorAudioPlayer.h index 626df39..2caf5e8 100755 --- a/libvideoeditor/lvpp/VideoEditorAudioPlayer.h +++ b/libvideoeditor/lvpp/VideoEditorAudioPlayer.h @@ -58,7 +58,7 @@ public: status_t start(bool sourceAlreadyStarted = false); void pause(bool playPendingSamples = false); - void resume(); + status_t resume(); status_t seekTo(int64_t time_us); bool isSeeking(); bool reachedEOS(status_t *finalStatus); @@ -91,7 +91,7 @@ private: int64_t mBGAudioStoryBoardCurrentMediaVolumeVal; sp<MediaSource> mSource; - AudioTrack *mAudioTrack; + sp<AudioTrack> mAudioTrack; MediaBuffer *mInputBuffer; @@ -124,7 +124,8 @@ private: size_t fillBuffer(void *data, size_t size); static size_t AudioSinkCallback( MediaPlayerBase::AudioSink *audioSink, - void *data, size_t size, void *me); + void *data, size_t size, void *me, + MediaPlayerBase::AudioSink::cb_event_t event); void reset(); void clear(); diff --git a/libvideoeditor/lvpp/VideoEditorPlayer.cpp b/libvideoeditor/lvpp/VideoEditorPlayer.cpp index 91a4415..5aeba4f 100755 --- a/libvideoeditor/lvpp/VideoEditorPlayer.cpp +++ b/libvideoeditor/lvpp/VideoEditorPlayer.cpp @@ -310,7 +310,6 @@ bool VideoEditorPlayer::VeAudioOutput::mIsOnEmulator = false; VideoEditorPlayer::VeAudioOutput::VeAudioOutput() : mCallback(NULL), mCallbackCookie(NULL) { - mTrack = 0; mStreamType = AUDIO_STREAM_MUSIC; mLeftVolume = 1.0; mRightVolume = 1.0; @@ -392,7 +391,8 @@ status_t VideoEditorPlayer::VeAudioOutput::getFramesWritten(uint32_t *written) c status_t VideoEditorPlayer::VeAudioOutput::open( uint32_t sampleRate, int channelCount, audio_channel_mask_t channelMask, audio_format_t format, int bufferCount, - AudioCallback cb, void *cookie, audio_output_flags_t flags) { + AudioCallback cb, void *cookie, audio_output_flags_t flags, + const audio_offload_info_t *offloadInfo) { mCallback = cb; mCallbackCookie = cookie; @@ -405,7 +405,7 @@ status_t VideoEditorPlayer::VeAudioOutput::open( } ALOGV("open(%u, %d, %d, %d)", sampleRate, channelCount, format, bufferCount); - if (mTrack) close(); + if (mTrack != 0) close(); uint32_t afSampleRate; size_t afFrameCount; int frameCount; @@ -434,7 +434,7 @@ status_t VideoEditorPlayer::VeAudioOutput::open( } } - AudioTrack *t; + sp<AudioTrack> t; if (mCallback != NULL) { t = new AudioTrack( mStreamType, @@ -457,7 +457,6 @@ status_t VideoEditorPlayer::VeAudioOutput::open( if ((t == 0) || (t->initCheck() != NO_ERROR)) { ALOGE("Unable to create audio track"); - delete t; return NO_INIT; } @@ -469,14 +468,18 @@ status_t VideoEditorPlayer::VeAudioOutput::open( return NO_ERROR; } -void VideoEditorPlayer::VeAudioOutput::start() { +status_t VideoEditorPlayer::VeAudioOutput::start() { ALOGV("start"); - if (mTrack) { + if (mTrack != 0) { mTrack->setVolume(mLeftVolume, mRightVolume); - mTrack->start(); - mTrack->getPosition(&mNumFramesWritten); + status_t status = mTrack->start(); + if (status == NO_ERROR) { + mTrack->getPosition(&mNumFramesWritten); + } + return status; } + return NO_INIT; } void VideoEditorPlayer::VeAudioOutput::snoopWrite( @@ -492,7 +495,7 @@ ssize_t VideoEditorPlayer::VeAudioOutput::write( LOG_FATAL_IF(mCallback != NULL, "Don't call write if supplying a callback."); //ALOGV("write(%p, %u)", buffer, size); - if (mTrack) { + if (mTrack != 0) { snoopWrite(buffer, size); ssize_t ret = mTrack->write(buffer, size); mNumFramesWritten += ret / 4; // assume 16 bit stereo @@ -504,26 +507,25 @@ ssize_t VideoEditorPlayer::VeAudioOutput::write( void VideoEditorPlayer::VeAudioOutput::stop() { ALOGV("stop"); - if (mTrack) mTrack->stop(); + if (mTrack != 0) mTrack->stop(); } void VideoEditorPlayer::VeAudioOutput::flush() { ALOGV("flush"); - if (mTrack) mTrack->flush(); + if (mTrack != 0) mTrack->flush(); } void VideoEditorPlayer::VeAudioOutput::pause() { ALOGV("VeAudioOutput::pause"); - if (mTrack) mTrack->pause(); + if (mTrack != 0) mTrack->pause(); } void VideoEditorPlayer::VeAudioOutput::close() { ALOGV("close"); - delete mTrack; - mTrack = 0; + mTrack.clear(); } void VideoEditorPlayer::VeAudioOutput::setVolume(float left, float right) { @@ -531,7 +533,7 @@ void VideoEditorPlayer::VeAudioOutput::setVolume(float left, float right) { ALOGV("setVolume(%f, %f)", left, right); mLeftVolume = left; mRightVolume = right; - if (mTrack) { + if (mTrack != 0) { mTrack->setVolume(left, right); } } @@ -548,7 +550,8 @@ void VideoEditorPlayer::VeAudioOutput::CallbackWrapper( AudioTrack::Buffer *buffer = (AudioTrack::Buffer *)info; size_t actualSize = (*me->mCallback)( - me, buffer->raw, buffer->size, me->mCallbackCookie); + me, buffer->raw, buffer->size, me->mCallbackCookie, + MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER); buffer->size = actualSize; diff --git a/libvideoeditor/lvpp/VideoEditorPlayer.h b/libvideoeditor/lvpp/VideoEditorPlayer.h index 77194ab..ab6d731 100755 --- a/libvideoeditor/lvpp/VideoEditorPlayer.h +++ b/libvideoeditor/lvpp/VideoEditorPlayer.h @@ -52,9 +52,10 @@ class VideoEditorPlayer : public MediaPlayerInterface { virtual status_t open( uint32_t sampleRate, int channelCount, audio_channel_mask_t channelMask, audio_format_t format, int bufferCount, - AudioCallback cb, void *cookie, audio_output_flags_t flags); + AudioCallback cb, void *cookie, audio_output_flags_t flags, + const audio_offload_info_t *offloadInfo); - virtual void start(); + virtual status_t start(); virtual ssize_t write(const void* buffer, size_t size); virtual void stop(); virtual void flush(); @@ -71,7 +72,7 @@ class VideoEditorPlayer : public MediaPlayerInterface { static void CallbackWrapper( int event, void *me, void *info); - AudioTrack* mTrack; + sp<AudioTrack> mTrack; AudioCallback mCallback; void * mCallbackCookie; audio_stream_type_t mStreamType; diff --git a/media/libcpustats/Android.mk b/media/libcpustats/Android.mk new file mode 100644 index 0000000..b506353 --- /dev/null +++ b/media/libcpustats/Android.mk @@ -0,0 +1,11 @@ +LOCAL_PATH:= $(call my-dir) + +include $(CLEAR_VARS) + +LOCAL_SRC_FILES := \ + CentralTendencyStatistics.cpp \ + ThreadCpuUsage.cpp + +LOCAL_MODULE := libcpustats + +include $(BUILD_STATIC_LIBRARY) diff --git a/media/libcpustats/CentralTendencyStatistics.cpp b/media/libcpustats/CentralTendencyStatistics.cpp new file mode 100644 index 0000000..42ab62b --- /dev/null +++ b/media/libcpustats/CentralTendencyStatistics.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdlib.h> + +#include <cpustats/CentralTendencyStatistics.h> + +void CentralTendencyStatistics::sample(double x) +{ + // update min and max + if (x < mMinimum) + mMinimum = x; + if (x > mMaximum) + mMaximum = x; + // Knuth + if (mN == 0) { + mMean = 0; + } + ++mN; + double delta = x - mMean; + mMean += delta / mN; + mM2 += delta * (x - mMean); +} + +void CentralTendencyStatistics::reset() +{ + mMean = NAN; + mMedian = NAN; + mMinimum = INFINITY; + mMaximum = -INFINITY; + mN = 0; + mM2 = 0; + mVariance = NAN; + mVarianceKnownForN = 0; + mStddev = NAN; + mStddevKnownForN = 0; +} + +double CentralTendencyStatistics::variance() const +{ + double variance; + if (mVarianceKnownForN != mN) { + if (mN > 1) { + // double variance_n = M2/n; + variance = mM2 / (mN - 1); + } else { + variance = NAN; + } + mVariance = variance; + mVarianceKnownForN = mN; + } else { + variance = mVariance; + } + return variance; +} + +double CentralTendencyStatistics::stddev() const +{ + double stddev; + if (mStddevKnownForN != mN) { + stddev = sqrt(variance()); + mStddev = stddev; + mStddevKnownForN = mN; + } else { + stddev = mStddev; + } + return stddev; +} diff --git a/media/libcpustats/ThreadCpuUsage.cpp b/media/libcpustats/ThreadCpuUsage.cpp new file mode 100644 index 0000000..637402a --- /dev/null +++ b/media/libcpustats/ThreadCpuUsage.cpp @@ -0,0 +1,255 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "ThreadCpuUsage" +//#define LOG_NDEBUG 0 + +#include <errno.h> +#include <stdlib.h> +#include <time.h> + +#include <utils/Debug.h> +#include <utils/Log.h> + +#include <cpustats/ThreadCpuUsage.h> + +namespace android { + +bool ThreadCpuUsage::setEnabled(bool isEnabled) +{ + bool wasEnabled = mIsEnabled; + // only do something if there is a change + if (isEnabled != wasEnabled) { + ALOGV("setEnabled(%d)", isEnabled); + int rc; + // enabling + if (isEnabled) { + rc = clock_gettime(CLOCK_THREAD_CPUTIME_ID, &mPreviousTs); + if (rc) { + ALOGE("clock_gettime(CLOCK_THREAD_CPUTIME_ID) errno=%d", errno); + isEnabled = false; + } else { + mWasEverEnabled = true; + // record wall clock time at first enable + if (!mMonotonicKnown) { + rc = clock_gettime(CLOCK_MONOTONIC, &mMonotonicTs); + if (rc) { + ALOGE("clock_gettime(CLOCK_MONOTONIC) errno=%d", errno); + } else { + mMonotonicKnown = true; + } + } + } + // disabling + } else { + struct timespec ts; + rc = clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts); + if (rc) { + ALOGE("clock_gettime(CLOCK_THREAD_CPUTIME_ID) errno=%d", errno); + } else { + long long delta = (ts.tv_sec - mPreviousTs.tv_sec) * 1000000000LL + + (ts.tv_nsec - mPreviousTs.tv_nsec); + mAccumulator += delta; +#if 0 + mPreviousTs = ts; +#endif + } + } + mIsEnabled = isEnabled; + } + return wasEnabled; +} + +bool ThreadCpuUsage::sampleAndEnable(double& ns) +{ + bool ret; + bool wasEverEnabled = mWasEverEnabled; + if (enable()) { + // already enabled, so add a new sample relative to previous + return sample(ns); + } else if (wasEverEnabled) { + // was disabled, but add sample for accumulated time while enabled + ns = (double) mAccumulator; + mAccumulator = 0; + ALOGV("sampleAndEnable %.0f", ns); + return true; + } else { + // first time called + ns = 0.0; + ALOGV("sampleAndEnable false"); + return false; + } +} + +bool ThreadCpuUsage::sample(double &ns) +{ + if (mWasEverEnabled) { + if (mIsEnabled) { + struct timespec ts; + int rc; + rc = clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts); + if (rc) { + ALOGE("clock_gettime(CLOCK_THREAD_CPUTIME_ID) errno=%d", errno); + ns = 0.0; + return false; + } else { + long long delta = (ts.tv_sec - mPreviousTs.tv_sec) * 1000000000LL + + (ts.tv_nsec - mPreviousTs.tv_nsec); + mAccumulator += delta; + mPreviousTs = ts; + } + } else { + mWasEverEnabled = false; + } + ns = (double) mAccumulator; + ALOGV("sample %.0f", ns); + mAccumulator = 0; + return true; + } else { + ALOGW("Can't add sample because measurements have never been enabled"); + ns = 0.0; + return false; + } +} + +long long ThreadCpuUsage::elapsed() const +{ + long long elapsed; + if (mMonotonicKnown) { + struct timespec ts; + int rc; + rc = clock_gettime(CLOCK_MONOTONIC, &ts); + if (rc) { + ALOGE("clock_gettime(CLOCK_MONOTONIC) errno=%d", errno); + elapsed = 0; + } else { + // mMonotonicTs is updated only at first enable and resetStatistics + elapsed = (ts.tv_sec - mMonotonicTs.tv_sec) * 1000000000LL + + (ts.tv_nsec - mMonotonicTs.tv_nsec); + } + } else { + ALOGW("Can't compute elapsed time because measurements have never been enabled"); + elapsed = 0; + } + ALOGV("elapsed %lld", elapsed); + return elapsed; +} + +void ThreadCpuUsage::resetElapsed() +{ + ALOGV("resetElapsed"); + if (mMonotonicKnown) { + int rc; + rc = clock_gettime(CLOCK_MONOTONIC, &mMonotonicTs); + if (rc) { + ALOGE("clock_gettime(CLOCK_MONOTONIC) errno=%d", errno); + mMonotonicKnown = false; + } + } +} + +/*static*/ +int ThreadCpuUsage::sScalingFds[ThreadCpuUsage::MAX_CPU]; +pthread_once_t ThreadCpuUsage::sOnceControl = PTHREAD_ONCE_INIT; +int ThreadCpuUsage::sKernelMax; +pthread_mutex_t ThreadCpuUsage::sMutex = PTHREAD_MUTEX_INITIALIZER; + +/*static*/ +void ThreadCpuUsage::init() +{ + // read the number of CPUs + sKernelMax = 1; + int fd = open("/sys/devices/system/cpu/kernel_max", O_RDONLY); + if (fd >= 0) { +#define KERNEL_MAX_SIZE 12 + char kernelMax[KERNEL_MAX_SIZE]; + ssize_t actual = read(fd, kernelMax, sizeof(kernelMax)); + if (actual >= 2 && kernelMax[actual-1] == '\n') { + sKernelMax = atoi(kernelMax); + if (sKernelMax >= MAX_CPU - 1) { + ALOGW("kernel_max %d but MAX_CPU %d", sKernelMax, MAX_CPU); + sKernelMax = MAX_CPU; + } else if (sKernelMax < 0) { + ALOGW("kernel_max invalid %d", sKernelMax); + sKernelMax = 1; + } else { + ++sKernelMax; + ALOGV("number of CPUs %d", sKernelMax); + } + } else { + ALOGW("Can't read number of CPUs"); + } + (void) close(fd); + } else { + ALOGW("Can't open number of CPUs"); + } + int i; + for (i = 0; i < MAX_CPU; ++i) { + sScalingFds[i] = -1; + } +} + +uint32_t ThreadCpuUsage::getCpukHz(int cpuNum) +{ + if (cpuNum < 0 || cpuNum >= MAX_CPU) { + ALOGW("getCpukHz called with invalid CPU %d", cpuNum); + return 0; + } + // double-checked locking idiom is not broken for atomic values such as fd + int fd = sScalingFds[cpuNum]; + if (fd < 0) { + // some kernels can't open a scaling file until hot plug complete + pthread_mutex_lock(&sMutex); + fd = sScalingFds[cpuNum]; + if (fd < 0) { +#define FREQ_SIZE 64 + char freq_path[FREQ_SIZE]; +#define FREQ_DIGIT 27 + COMPILE_TIME_ASSERT_FUNCTION_SCOPE(MAX_CPU <= 10); +#define FREQ_PATH "/sys/devices/system/cpu/cpu?/cpufreq/scaling_cur_freq" + strlcpy(freq_path, FREQ_PATH, sizeof(freq_path)); + freq_path[FREQ_DIGIT] = cpuNum + '0'; + fd = open(freq_path, O_RDONLY | O_CLOEXEC); + // keep this fd until process exit or exec + sScalingFds[cpuNum] = fd; + } + pthread_mutex_unlock(&sMutex); + if (fd < 0) { + ALOGW("getCpukHz can't open CPU %d", cpuNum); + return 0; + } + } +#define KHZ_SIZE 12 + char kHz[KHZ_SIZE]; // kHz base 10 + ssize_t actual = pread(fd, kHz, sizeof(kHz), (off_t) 0); + uint32_t ret; + if (actual >= 2 && kHz[actual-1] == '\n') { + ret = atoi(kHz); + } else { + ret = 0; + } + if (ret != mCurrentkHz[cpuNum]) { + if (ret > 0) { + ALOGV("CPU %d frequency %u kHz", cpuNum, ret); + } else { + ALOGW("Can't read CPU %d frequency", cpuNum); + } + mCurrentkHz[cpuNum] = ret; + } + return ret; +} + +} // namespace android diff --git a/media/libeffects/data/audio_effects.conf b/media/libeffects/data/audio_effects.conf index 93f27cb..c3c4b67 100644 --- a/media/libeffects/data/audio_effects.conf +++ b/media/libeffects/data/audio_effects.conf @@ -6,6 +6,23 @@ # } # } libraries { +# This is a proxy library that will be an abstraction for +# the HW and SW effects + + #proxy { + #path /system/lib/soundfx/libeffectproxy.so + #} + +# This is the SW implementation library of the effect + #libSW { + #path /system/lib/soundfx/libswwrapper.so + #} + +# This is the HW implementation library for the effect + #libHW { + #path /system/lib/soundfx/libhwwrapper.so + #} + bundle { path /system/lib/soundfx/libbundlewrapper.so } @@ -18,6 +35,9 @@ libraries { downmix { path /system/lib/soundfx/libdownmix.so } + loudness_enhancer { + path /system/lib/soundfx/libldnhncr.so + } } # Default pre-processing library. Add to audio_effect.conf "libraries" section if @@ -43,6 +63,28 @@ libraries { # } effects { + +# additions for the proxy implementation +# Proxy implementation + #effectname { + #library proxy + #uuid xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + + # SW implemetation of the effect. Added as a node under the proxy to + # indicate this as a sub effect. + #libsw { + #library libSW + #uuid yyyyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy + #} End of SW effect + + # HW implementation of the effect. Added as a node under the proxy to + # indicate this as a sub effect. + #libhw { + #library libHW + #uuid zzzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzzz + #}End of HW effect + #} End of effect proxy + bassboost { library bundle uuid 8631f300-72e2-11df-b57e-0002a5d5c51b @@ -83,6 +125,10 @@ effects { library downmix uuid 93f04452-e4fe-41cc-91f9-e475b6d1d69f } + loudness_enhancer { + library loudness_enhancer + uuid fa415329-2034-4bea-b5dc-5b381c8d1e2c + } } # Default pre-processing effects. Add to audio_effect.conf "effects" section if diff --git a/media/libeffects/factory/EffectsFactory.c b/media/libeffects/factory/EffectsFactory.c index f158929..f8d6041 100644 --- a/media/libeffects/factory/EffectsFactory.c +++ b/media/libeffects/factory/EffectsFactory.c @@ -28,6 +28,9 @@ static list_elem_t *gEffectList; // list of effect_entry_t: all currently created effects static list_elem_t *gLibraryList; // list of lib_entry_t: all currently loaded libraries +// list of effect_descriptor and list of sub effects : all currently loaded +// It does not contain effects without sub effects. +static list_sub_elem_t *gSubEffectList; static pthread_mutex_t gLibLock = PTHREAD_MUTEX_INITIALIZER; // controls access to gLibraryList static uint32_t gNumEffects; // total number number of effects static list_elem_t *gCurLib; // current library in enumeration process @@ -50,6 +53,8 @@ static int loadLibraries(cnode *root); static int loadLibrary(cnode *root, const char *name); static int loadEffects(cnode *root); static int loadEffect(cnode *node); +// To get and add the effect pointed by the passed node to the gSubEffectList +static int addSubEffect(cnode *root); static lib_entry_t *getLibrary(const char *path); static void resetEffectEnumeration(); static uint32_t updateNumEffects(); @@ -57,6 +62,10 @@ static int findEffect(const effect_uuid_t *type, const effect_uuid_t *uuid, lib_entry_t **lib, effect_descriptor_t **desc); +// To search a subeffect in the gSubEffectList +int findSubEffect(const effect_uuid_t *uuid, + lib_entry_t **lib, + effect_descriptor_t **desc); static void dumpEffectDescriptor(effect_descriptor_t *desc, char *str, size_t len); static int stringToUuid(const char *str, effect_uuid_t *uuid); static int uuidToString(const effect_uuid_t *uuid, char *str, size_t maxLen); @@ -287,7 +296,12 @@ int EffectCreate(const effect_uuid_t *uuid, int32_t sessionId, int32_t ioId, eff ret = findEffect(NULL, uuid, &l, &d); if (ret < 0){ - goto exit; + // Sub effects are not associated with the library->effects, + // so, findEffect will fail. Search for the effect in gSubEffectList. + ret = findSubEffect(uuid, &l, &d); + if (ret < 0 ) { + goto exit; + } } // create effect in library @@ -354,21 +368,27 @@ int EffectRelease(effect_handle_t handle) } if (e1 == NULL) { ret = -ENOENT; + pthread_mutex_unlock(&gLibLock); goto exit; } // release effect in library if (fx->lib == NULL) { ALOGW("EffectRelease() fx %p library already unloaded", handle); + pthread_mutex_unlock(&gLibLock); } else { pthread_mutex_lock(&fx->lib->lock); + // Releasing the gLibLock here as the list access is over as the + // effect is removed from the list. + // If the gLibLock is not released, we will have a deadlock situation + // since we call the sub effect release inside the EffectRelease of Proxy + pthread_mutex_unlock(&gLibLock); fx->lib->desc->release_effect(fx->subItfe); pthread_mutex_unlock(&fx->lib->lock); } free(fx); exit: - pthread_mutex_unlock(&gLibLock); return ret; } @@ -380,6 +400,49 @@ int EffectIsNullUuid(const effect_uuid_t *uuid) return 1; } +// Function to get the sub effect descriptors of the effect whose uuid +// is pointed by the first argument. It searches the gSubEffectList for the +// matching uuid and then copies the corresponding sub effect descriptors +// to the inout param +int EffectGetSubEffects(const effect_uuid_t *uuid, + effect_descriptor_t *pDescriptors, size_t size) +{ + ALOGV("EffectGetSubEffects() UUID: %08X-%04X-%04X-%04X-%02X%02X%02X%02X%02X" + "%02X\n",uuid->timeLow, uuid->timeMid, uuid->timeHiAndVersion, + uuid->clockSeq, uuid->node[0], uuid->node[1],uuid->node[2], + uuid->node[3],uuid->node[4],uuid->node[5]); + + // Check if the size of the desc buffer is large enough for 2 subeffects + if ((uuid == NULL) || (pDescriptors == NULL) || + (size < 2*sizeof(effect_descriptor_t))) { + ALOGW("NULL pointer or insufficient memory. Cannot query subeffects"); + return -EINVAL; + } + int ret = init(); + if (ret < 0) + return ret; + list_sub_elem_t *e = gSubEffectList; + sub_effect_entry_t *subeffect; + effect_descriptor_t *d; + int count = 0; + while (e != NULL) { + d = (effect_descriptor_t*)e->object; + if (memcmp(uuid, &d->uuid, sizeof(effect_uuid_t)) == 0) { + ALOGV("EffectGetSubEffects: effect found in the list"); + list_elem_t *subefx = e->sub_elem; + while (subefx != NULL) { + subeffect = (sub_effect_entry_t*)subefx->object; + d = (effect_descriptor_t*)(subeffect->object); + pDescriptors[count++] = *d; + subefx = subefx->next; + } + ALOGV("EffectGetSubEffects end - copied the sub effect descriptors"); + return count; + } + e = e->next; + } + return -ENOENT; +} ///////////////////////////////////////////////// // Local functions ///////////////////////////////////////////////// @@ -503,6 +566,65 @@ error: return -EINVAL; } +// This will find the library and UUID tags of the sub effect pointed by the +// node, gets the effect descriptor and lib_entry_t and adds the subeffect - +// sub_entry_t to the gSubEffectList +int addSubEffect(cnode *root) +{ + ALOGV("addSubEffect"); + cnode *node; + effect_uuid_t uuid; + effect_descriptor_t *d; + lib_entry_t *l; + list_elem_t *e; + node = config_find(root, LIBRARY_TAG); + if (node == NULL) { + return -EINVAL; + } + l = getLibrary(node->value); + if (l == NULL) { + ALOGW("addSubEffect() could not get library %s", node->value); + return -EINVAL; + } + node = config_find(root, UUID_TAG); + if (node == NULL) { + return -EINVAL; + } + if (stringToUuid(node->value, &uuid) != 0) { + ALOGW("addSubEffect() invalid uuid %s", node->value); + return -EINVAL; + } + d = malloc(sizeof(effect_descriptor_t)); + if (l->desc->get_descriptor(&uuid, d) != 0) { + char s[40]; + uuidToString(&uuid, s, 40); + ALOGW("Error querying effect %s on lib %s", s, l->name); + free(d); + return -EINVAL; + } +#if (LOG_NDEBUG==0) + char s[256]; + dumpEffectDescriptor(d, s, 256); + ALOGV("addSubEffect() read descriptor %p:%s",d, s); +#endif + if (EFFECT_API_VERSION_MAJOR(d->apiVersion) != + EFFECT_API_VERSION_MAJOR(EFFECT_CONTROL_API_VERSION)) { + ALOGW("Bad API version %08x on lib %s", d->apiVersion, l->name); + free(d); + return -EINVAL; + } + sub_effect_entry_t *sub_effect = malloc(sizeof(sub_effect_entry_t)); + sub_effect->object = d; + // lib_entry_t is stored since the sub effects are not linked to the library + sub_effect->lib = l; + e = malloc(sizeof(list_elem_t)); + e->object = sub_effect; + e->next = gSubEffectList->sub_elem; + gSubEffectList->sub_elem = e; + ALOGV("addSubEffect end"); + return 0; +} + int loadEffects(cnode *root) { cnode *node; @@ -571,9 +693,101 @@ int loadEffect(cnode *root) e->next = l->effects; l->effects = e; + // After the UUID node in the config_tree, if node->next is valid, + // that would be sub effect node. + // Find the sub effects and add them to the gSubEffectList + node = node->next; + int count = 2; + bool hwSubefx = false, swSubefx = false; + list_sub_elem_t *sube = NULL; + if (node != NULL) { + ALOGV("Adding the effect to gEffectSubList as there are sub effects"); + sube = malloc(sizeof(list_sub_elem_t)); + sube->object = d; + sube->sub_elem = NULL; + sube->next = gSubEffectList; + gSubEffectList = sube; + } + while (node != NULL && count) { + if (addSubEffect(node)) { + ALOGW("loadEffect() could not add subEffect %s", node->value); + // Change the gSubEffectList to point to older list; + gSubEffectList = sube->next; + free(sube->sub_elem);// Free an already added sub effect + sube->sub_elem = NULL; + free(sube); + return -ENOENT; + } + sub_effect_entry_t *subEntry = (sub_effect_entry_t*)gSubEffectList->sub_elem->object; + effect_descriptor_t *subEffectDesc = (effect_descriptor_t*)(subEntry->object); + // Since we return a dummy descriptor for the proxy during + // get_descriptor call,we replace it with the correspoding + // sw effect descriptor, but with Proxy UUID + // check for Sw desc + if (!((subEffectDesc->flags & EFFECT_FLAG_HW_ACC_MASK) == + EFFECT_FLAG_HW_ACC_TUNNEL)) { + swSubefx = true; + *d = *subEffectDesc; + d->uuid = uuid; + ALOGV("loadEffect() Changed the Proxy desc"); + } else + hwSubefx = true; + count--; + node = node->next; + } + // 1 HW and 1 SW sub effect found. Set the offload flag in the Proxy desc + if (hwSubefx && swSubefx) { + d->flags |= EFFECT_FLAG_OFFLOAD_SUPPORTED; + } return 0; } +// Searches the sub effect matching to the specified uuid +// in the gSubEffectList. It gets the lib_entry_t for +// the matched sub_effect . Used in EffectCreate of sub effects +int findSubEffect(const effect_uuid_t *uuid, + lib_entry_t **lib, + effect_descriptor_t **desc) +{ + list_sub_elem_t *e = gSubEffectList; + list_elem_t *subefx; + sub_effect_entry_t *effect; + lib_entry_t *l = NULL; + effect_descriptor_t *d = NULL; + int found = 0; + int ret = 0; + + if (uuid == NULL) + return -EINVAL; + + while (e != NULL && !found) { + subefx = (list_elem_t*)(e->sub_elem); + while (subefx != NULL) { + effect = (sub_effect_entry_t*)subefx->object; + l = (lib_entry_t *)effect->lib; + d = (effect_descriptor_t *)effect->object; + if (memcmp(&d->uuid, uuid, sizeof(effect_uuid_t)) == 0) { + ALOGV("uuid matched"); + found = 1; + break; + } + subefx = subefx->next; + } + e = e->next; + } + if (!found) { + ALOGV("findSubEffect() effect not found"); + ret = -ENOENT; + } else { + ALOGV("findSubEffect() found effect: %s in lib %s", d->name, l->name); + *lib = l; + if (desc != NULL) { + *desc = d; + } + } + return ret; +} + lib_entry_t *getLibrary(const char *name) { list_elem_t *e; diff --git a/media/libeffects/factory/EffectsFactory.h b/media/libeffects/factory/EffectsFactory.h index c1d4319..147ff18 100644 --- a/media/libeffects/factory/EffectsFactory.h +++ b/media/libeffects/factory/EffectsFactory.h @@ -32,6 +32,15 @@ typedef struct list_elem_s { struct list_elem_s *next; } list_elem_t; +// Structure used for storing effects with their sub effects. +// Used in creating gSubEffectList. Here, +// object holds the effect desc and the list sub_elem holds the sub effects +typedef struct list_sub_elem_s { + void *object; + list_elem_t *sub_elem; + struct list_sub_elem_s *next; +} list_sub_elem_t; + typedef struct lib_entry_s { audio_effect_library_t *desc; char *name; @@ -47,6 +56,16 @@ typedef struct effect_entry_s { lib_entry_t *lib; } effect_entry_t; +// Structure used to store the lib entry +// and the descriptor of the sub effects. +// The library entry is to be stored in case of +// sub effects as the sub effects are not linked +// to the library list - gLibraryList. +typedef struct sub_effect_entry_s { + lib_entry_t *lib; + void *object; +} sub_effect_entry_t; + #if __cplusplus } // extern "C" #endif diff --git a/media/libeffects/loudness/Android.mk b/media/libeffects/loudness/Android.mk new file mode 100644 index 0000000..dcb7b27 --- /dev/null +++ b/media/libeffects/loudness/Android.mk @@ -0,0 +1,27 @@ +LOCAL_PATH:= $(call my-dir) + +# LoudnessEnhancer library +include $(CLEAR_VARS) + +LOCAL_SRC_FILES:= \ + EffectLoudnessEnhancer.cpp \ + dsp/core/dynamic_range_compression.cpp + +LOCAL_CFLAGS+= -O2 -fvisibility=hidden + +LOCAL_SHARED_LIBRARIES := \ + libcutils \ + liblog \ + libstlport + +LOCAL_MODULE_PATH := $(TARGET_OUT_SHARED_LIBRARIES)/soundfx +LOCAL_MODULE:= libldnhncr + +LOCAL_C_INCLUDES := \ + $(call include-path-for, audio-effects) \ + bionic \ + bionic/libstdc++/include \ + external/stlport/stlport + + +include $(BUILD_SHARED_LIBRARY) diff --git a/media/libeffects/loudness/EffectLoudnessEnhancer.cpp b/media/libeffects/loudness/EffectLoudnessEnhancer.cpp new file mode 100644 index 0000000..91ed677 --- /dev/null +++ b/media/libeffects/loudness/EffectLoudnessEnhancer.cpp @@ -0,0 +1,466 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "EffectLE" +//#define LOG_NDEBUG 0 +#include <cutils/log.h> +#include <assert.h> +#include <stdlib.h> +#include <string.h> +#include <new> +#include <time.h> +#include <math.h> +#include <audio_effects/effect_loudnessenhancer.h> +#include "dsp/core/dynamic_range_compression.h" + +extern "C" { + +// effect_handle_t interface implementation for LE effect +extern const struct effect_interface_s gLEInterface; + +// AOSP Loudness Enhancer UUID: fa415329-2034-4bea-b5dc-5b381c8d1e2c +const effect_descriptor_t gLEDescriptor = { + {0xfe3199be, 0xaed0, 0x413f, 0x87bb, {0x11, 0x26, 0x0e, 0xb6, 0x3c, 0xf1}}, // type + {0xfa415329, 0x2034, 0x4bea, 0xb5dc, {0x5b, 0x38, 0x1c, 0x8d, 0x1e, 0x2c}}, // uuid + EFFECT_CONTROL_API_VERSION, + (EFFECT_FLAG_TYPE_INSERT | EFFECT_FLAG_INSERT_FIRST), + 0, // TODO + 1, + "Loudness Enhancer", + "The Android Open Source Project", +}; + +enum le_state_e { + LOUDNESS_ENHANCER_STATE_UNINITIALIZED, + LOUDNESS_ENHANCER_STATE_INITIALIZED, + LOUDNESS_ENHANCER_STATE_ACTIVE, +}; + +struct LoudnessEnhancerContext { + const struct effect_interface_s *mItfe; + effect_config_t mConfig; + uint8_t mState; + int32_t mTargetGainmB;// target gain in mB + // in this implementation, there is no coupling between the compression on the left and right + // channels + le_fx::AdaptiveDynamicRangeCompression* mCompressor; +}; + +// +//--- Local functions (not directly used by effect interface) +// + +void LE_reset(LoudnessEnhancerContext *pContext) +{ + ALOGV(" > LE_reset(%p)", pContext); + + if (pContext->mCompressor != NULL) { + float targetAmp = pow(10, pContext->mTargetGainmB/2000.0f); // mB to linear amplification + ALOGV("LE_reset(): Target gain=%dmB <=> factor=%.2fX", pContext->mTargetGainmB, targetAmp); + pContext->mCompressor->Initialize(targetAmp, pContext->mConfig.inputCfg.samplingRate); + } else { + ALOGE("LE_reset(%p): null compressors, can't apply target gain", pContext); + } +} + +static inline int16_t clamp16(int32_t sample) +{ + if ((sample>>15) ^ (sample>>31)) + sample = 0x7FFF ^ (sample>>31); + return sample; +} + +//---------------------------------------------------------------------------- +// LE_setConfig() +//---------------------------------------------------------------------------- +// Purpose: Set input and output audio configuration. +// +// Inputs: +// pContext: effect engine context +// pConfig: pointer to effect_config_t structure holding input and output +// configuration parameters +// +// Outputs: +// +//---------------------------------------------------------------------------- + +int LE_setConfig(LoudnessEnhancerContext *pContext, effect_config_t *pConfig) +{ + ALOGV("LE_setConfig(%p)", pContext); + + if (pConfig->inputCfg.samplingRate != pConfig->outputCfg.samplingRate) return -EINVAL; + if (pConfig->inputCfg.channels != pConfig->outputCfg.channels) return -EINVAL; + if (pConfig->inputCfg.format != pConfig->outputCfg.format) return -EINVAL; + if (pConfig->inputCfg.channels != AUDIO_CHANNEL_OUT_STEREO) return -EINVAL; + if (pConfig->outputCfg.accessMode != EFFECT_BUFFER_ACCESS_WRITE && + pConfig->outputCfg.accessMode != EFFECT_BUFFER_ACCESS_ACCUMULATE) return -EINVAL; + if (pConfig->inputCfg.format != AUDIO_FORMAT_PCM_16_BIT) return -EINVAL; + + pContext->mConfig = *pConfig; + + LE_reset(pContext); + + return 0; +} + + +//---------------------------------------------------------------------------- +// LE_getConfig() +//---------------------------------------------------------------------------- +// Purpose: Get input and output audio configuration. +// +// Inputs: +// pContext: effect engine context +// pConfig: pointer to effect_config_t structure holding input and output +// configuration parameters +// +// Outputs: +// +//---------------------------------------------------------------------------- + +void LE_getConfig(LoudnessEnhancerContext *pContext, effect_config_t *pConfig) +{ + *pConfig = pContext->mConfig; +} + + +//---------------------------------------------------------------------------- +// LE_init() +//---------------------------------------------------------------------------- +// Purpose: Initialize engine with default configuration. +// +// Inputs: +// pContext: effect engine context +// +// Outputs: +// +//---------------------------------------------------------------------------- + +int LE_init(LoudnessEnhancerContext *pContext) +{ + ALOGV("LE_init(%p)", pContext); + + pContext->mConfig.inputCfg.accessMode = EFFECT_BUFFER_ACCESS_READ; + pContext->mConfig.inputCfg.channels = AUDIO_CHANNEL_OUT_STEREO; + pContext->mConfig.inputCfg.format = AUDIO_FORMAT_PCM_16_BIT; + pContext->mConfig.inputCfg.samplingRate = 44100; + pContext->mConfig.inputCfg.bufferProvider.getBuffer = NULL; + pContext->mConfig.inputCfg.bufferProvider.releaseBuffer = NULL; + pContext->mConfig.inputCfg.bufferProvider.cookie = NULL; + pContext->mConfig.inputCfg.mask = EFFECT_CONFIG_ALL; + pContext->mConfig.outputCfg.accessMode = EFFECT_BUFFER_ACCESS_ACCUMULATE; + pContext->mConfig.outputCfg.channels = AUDIO_CHANNEL_OUT_STEREO; + pContext->mConfig.outputCfg.format = AUDIO_FORMAT_PCM_16_BIT; + pContext->mConfig.outputCfg.samplingRate = 44100; + pContext->mConfig.outputCfg.bufferProvider.getBuffer = NULL; + pContext->mConfig.outputCfg.bufferProvider.releaseBuffer = NULL; + pContext->mConfig.outputCfg.bufferProvider.cookie = NULL; + pContext->mConfig.outputCfg.mask = EFFECT_CONFIG_ALL; + + pContext->mTargetGainmB = LOUDNESS_ENHANCER_DEFAULT_TARGET_GAIN_MB; + float targetAmp = pow(10, pContext->mTargetGainmB/2000.0f); // mB to linear amplification + ALOGV("LE_init(): Target gain=%dmB <=> factor=%.2fX", pContext->mTargetGainmB, targetAmp); + + if (pContext->mCompressor == NULL) { + pContext->mCompressor = new le_fx::AdaptiveDynamicRangeCompression(); + pContext->mCompressor->Initialize(targetAmp, pContext->mConfig.inputCfg.samplingRate); + } + + LE_setConfig(pContext, &pContext->mConfig); + + return 0; +} + +// +//--- Effect Library Interface Implementation +// + +int LELib_Create(const effect_uuid_t *uuid, + int32_t sessionId, + int32_t ioId, + effect_handle_t *pHandle) { + ALOGV("LELib_Create()"); + int ret; + int i; + + if (pHandle == NULL || uuid == NULL) { + return -EINVAL; + } + + if (memcmp(uuid, &gLEDescriptor.uuid, sizeof(effect_uuid_t)) != 0) { + return -EINVAL; + } + + LoudnessEnhancerContext *pContext = new LoudnessEnhancerContext; + + pContext->mItfe = &gLEInterface; + pContext->mState = LOUDNESS_ENHANCER_STATE_UNINITIALIZED; + + pContext->mCompressor = NULL; + ret = LE_init(pContext); + if (ret < 0) { + ALOGW("LELib_Create() init failed"); + delete pContext; + return ret; + } + + *pHandle = (effect_handle_t)pContext; + + pContext->mState = LOUDNESS_ENHANCER_STATE_INITIALIZED; + + ALOGV(" LELib_Create context is %p", pContext); + + return 0; + +} + +int LELib_Release(effect_handle_t handle) { + LoudnessEnhancerContext * pContext = (LoudnessEnhancerContext *)handle; + + ALOGV("LELib_Release %p", handle); + if (pContext == NULL) { + return -EINVAL; + } + pContext->mState = LOUDNESS_ENHANCER_STATE_UNINITIALIZED; + if (pContext->mCompressor != NULL) { + delete pContext->mCompressor; + pContext->mCompressor = NULL; + } + delete pContext; + + return 0; +} + +int LELib_GetDescriptor(const effect_uuid_t *uuid, + effect_descriptor_t *pDescriptor) { + + if (pDescriptor == NULL || uuid == NULL){ + ALOGV("LELib_GetDescriptor() called with NULL pointer"); + return -EINVAL; + } + + if (memcmp(uuid, &gLEDescriptor.uuid, sizeof(effect_uuid_t)) == 0) { + *pDescriptor = gLEDescriptor; + return 0; + } + + return -EINVAL; +} /* end LELib_GetDescriptor */ + +// +//--- Effect Control Interface Implementation +// +int LE_process( + effect_handle_t self, audio_buffer_t *inBuffer, audio_buffer_t *outBuffer) +{ + LoudnessEnhancerContext * pContext = (LoudnessEnhancerContext *)self; + + if (pContext == NULL) { + return -EINVAL; + } + + if (inBuffer == NULL || inBuffer->raw == NULL || + outBuffer == NULL || outBuffer->raw == NULL || + inBuffer->frameCount != outBuffer->frameCount || + inBuffer->frameCount == 0) { + return -EINVAL; + } + + //ALOGV("LE about to process %d samples", inBuffer->frameCount); + uint16_t inIdx; + float inputAmp = pow(10, pContext->mTargetGainmB/2000.0f); + float leftSample, rightSample; + for (inIdx = 0 ; inIdx < inBuffer->frameCount ; inIdx++) { + // makeup gain is applied on the input of the compressor + leftSample = inputAmp * (float)inBuffer->s16[2*inIdx]; + rightSample = inputAmp * (float)inBuffer->s16[2*inIdx +1]; + pContext->mCompressor->Compress(&leftSample, &rightSample); + inBuffer->s16[2*inIdx] = (int16_t) leftSample; + inBuffer->s16[2*inIdx +1] = (int16_t) rightSample; + } + + if (inBuffer->raw != outBuffer->raw) { + if (pContext->mConfig.outputCfg.accessMode == EFFECT_BUFFER_ACCESS_ACCUMULATE) { + for (size_t i = 0; i < outBuffer->frameCount*2; i++) { + outBuffer->s16[i] = clamp16(outBuffer->s16[i] + inBuffer->s16[i]); + } + } else { + memcpy(outBuffer->raw, inBuffer->raw, outBuffer->frameCount * 2 * sizeof(int16_t)); + } + } + if (pContext->mState != LOUDNESS_ENHANCER_STATE_ACTIVE) { + return -ENODATA; + } + return 0; +} + +int LE_command(effect_handle_t self, uint32_t cmdCode, uint32_t cmdSize, + void *pCmdData, uint32_t *replySize, void *pReplyData) { + + LoudnessEnhancerContext * pContext = (LoudnessEnhancerContext *)self; + int retsize; + + if (pContext == NULL || pContext->mState == LOUDNESS_ENHANCER_STATE_UNINITIALIZED) { + return -EINVAL; + } + +// ALOGV("LE_command command %d cmdSize %d",cmdCode, cmdSize); + switch (cmdCode) { + case EFFECT_CMD_INIT: + if (pReplyData == NULL || *replySize != sizeof(int)) { + return -EINVAL; + } + *(int *) pReplyData = LE_init(pContext); + break; + case EFFECT_CMD_SET_CONFIG: + if (pCmdData == NULL || cmdSize != sizeof(effect_config_t) + || pReplyData == NULL || *replySize != sizeof(int)) { + return -EINVAL; + } + *(int *) pReplyData = LE_setConfig(pContext, + (effect_config_t *) pCmdData); + break; + case EFFECT_CMD_GET_CONFIG: + if (pReplyData == NULL || + *replySize != sizeof(effect_config_t)) { + return -EINVAL; + } + LE_getConfig(pContext, (effect_config_t *)pReplyData); + break; + case EFFECT_CMD_RESET: + LE_reset(pContext); + break; + case EFFECT_CMD_ENABLE: + if (pReplyData == NULL || *replySize != sizeof(int)) { + return -EINVAL; + } + if (pContext->mState != LOUDNESS_ENHANCER_STATE_INITIALIZED) { + return -ENOSYS; + } + pContext->mState = LOUDNESS_ENHANCER_STATE_ACTIVE; + ALOGV("EFFECT_CMD_ENABLE() OK"); + *(int *)pReplyData = 0; + break; + case EFFECT_CMD_DISABLE: + if (pReplyData == NULL || *replySize != sizeof(int)) { + return -EINVAL; + } + if (pContext->mState != LOUDNESS_ENHANCER_STATE_ACTIVE) { + return -ENOSYS; + } + pContext->mState = LOUDNESS_ENHANCER_STATE_INITIALIZED; + ALOGV("EFFECT_CMD_DISABLE() OK"); + *(int *)pReplyData = 0; + break; + case EFFECT_CMD_GET_PARAM: { + if (pCmdData == NULL || + cmdSize != (int)(sizeof(effect_param_t) + sizeof(uint32_t)) || + pReplyData == NULL || + *replySize < (int)(sizeof(effect_param_t) + sizeof(uint32_t) + sizeof(uint32_t))) { + return -EINVAL; + } + memcpy(pReplyData, pCmdData, sizeof(effect_param_t) + sizeof(uint32_t)); + effect_param_t *p = (effect_param_t *)pReplyData; + p->status = 0; + *replySize = sizeof(effect_param_t) + sizeof(uint32_t); + if (p->psize != sizeof(uint32_t)) { + p->status = -EINVAL; + break; + } + switch (*(uint32_t *)p->data) { + case LOUDNESS_ENHANCER_PARAM_TARGET_GAIN_MB: + ALOGV("get target gain(mB) = %d", pContext->mTargetGainmB); + *((int32_t *)p->data + 1) = pContext->mTargetGainmB; + p->vsize = sizeof(int32_t); + *replySize += sizeof(int32_t); + break; + default: + p->status = -EINVAL; + } + } break; + case EFFECT_CMD_SET_PARAM: { + if (pCmdData == NULL || + cmdSize != (int)(sizeof(effect_param_t) + sizeof(uint32_t) + sizeof(uint32_t)) || + pReplyData == NULL || *replySize != sizeof(int32_t)) { + return -EINVAL; + } + *(int32_t *)pReplyData = 0; + effect_param_t *p = (effect_param_t *)pCmdData; + if (p->psize != sizeof(uint32_t) || p->vsize != sizeof(uint32_t)) { + *(int32_t *)pReplyData = -EINVAL; + break; + } + switch (*(uint32_t *)p->data) { + case LOUDNESS_ENHANCER_PARAM_TARGET_GAIN_MB: + pContext->mTargetGainmB = *((int32_t *)p->data + 1); + ALOGV("set target gain(mB) = %d", pContext->mTargetGainmB); + LE_reset(pContext); // apply parameter update + break; + default: + *(int32_t *)pReplyData = -EINVAL; + } + } break; + case EFFECT_CMD_SET_DEVICE: + case EFFECT_CMD_SET_VOLUME: + case EFFECT_CMD_SET_AUDIO_MODE: + break; + + default: + ALOGW("LE_command invalid command %d",cmdCode); + return -EINVAL; + } + + return 0; +} + +/* Effect Control Interface Implementation: get_descriptor */ +int LE_getDescriptor(effect_handle_t self, + effect_descriptor_t *pDescriptor) +{ + LoudnessEnhancerContext * pContext = (LoudnessEnhancerContext *) self; + + if (pContext == NULL || pDescriptor == NULL) { + ALOGV("LE_getDescriptor() invalid param"); + return -EINVAL; + } + + *pDescriptor = gLEDescriptor; + + return 0; +} /* end LE_getDescriptor */ + +// effect_handle_t interface implementation for DRC effect +const struct effect_interface_s gLEInterface = { + LE_process, + LE_command, + LE_getDescriptor, + NULL, +}; + +// This is the only symbol that needs to be exported +__attribute__ ((visibility ("default"))) +audio_effect_library_t AUDIO_EFFECT_LIBRARY_INFO_SYM = { + tag : AUDIO_EFFECT_LIBRARY_TAG, + version : EFFECT_LIBRARY_API_VERSION, + name : "Loudness Enhancer Library", + implementor : "The Android Open Source Project", + create_effect : LELib_Create, + release_effect : LELib_Release, + get_descriptor : LELib_GetDescriptor, +}; + +}; // extern "C" + diff --git a/media/libeffects/loudness/MODULE_LICENSE_APACHE2 b/media/libeffects/loudness/MODULE_LICENSE_APACHE2 new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/media/libeffects/loudness/MODULE_LICENSE_APACHE2 diff --git a/media/libeffects/loudness/NOTICE b/media/libeffects/loudness/NOTICE new file mode 100644 index 0000000..ad6ed94 --- /dev/null +++ b/media/libeffects/loudness/NOTICE @@ -0,0 +1,190 @@ + + Copyright (c) 2013, The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + diff --git a/media/libeffects/loudness/common/core/basic_types.h b/media/libeffects/loudness/common/core/basic_types.h new file mode 100644 index 0000000..593e914 --- /dev/null +++ b/media/libeffects/loudness/common/core/basic_types.h @@ -0,0 +1,114 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LE_FX_ENGINE_COMMON_CORE_BASIC_TYPES_H_ +#define LE_FX_ENGINE_COMMON_CORE_BASIC_TYPES_H_ + +#include <stddef.h> +#include <stdlib.h> +#include <string> +using ::std::string; +using ::std::basic_string; +#include <vector> +using ::std::vector; + +#include "common/core/os.h" + +// ----------------------------------------------------------------------------- +// Definitions of common basic types: +// ----------------------------------------------------------------------------- + +#if !defined(G_COMPILE) && !defined(BASE_INTEGRAL_TYPES_H_) + +namespace le_fx { + +typedef signed char schar; +typedef signed char int8; +typedef short int16; +typedef int int32; +typedef long long int64; + +typedef unsigned char uint8; +typedef unsigned short uint16; +typedef unsigned int uint32; +typedef unsigned long long uint64; + +} // namespace le_fx + +#endif + +namespace le_fx { + +struct FloatArray { + int length; + float *data; + + FloatArray(void) { + data = NULL; + length = 0; + } +}; + +struct Int16Array { + int length; + int16 *data; + + Int16Array(void) { + data = NULL; + length = 0; + } +}; + +struct Int32Array { + int length; + int32 *data; + + Int32Array(void) { + data = NULL; + length = 0; + } +}; + +struct Int8Array { + int length; + uint8 *data; + + Int8Array(void) { + data = NULL; + length = 0; + } +}; + +// +// Simple wrapper for waveform data: +// +class WaveData : public vector<int16> { + public: + WaveData(); + ~WaveData(); + + void Set(int number_samples, int sampling_rate, int16 *data); + int sample_rate(void) const; + void set_sample_rate(int sample_rate); + bool Equals(const WaveData &wave_data, int threshold = 0) const; + + private: + int sample_rate_; +}; + +} // namespace le_fx + +#endif // LE_FX_ENGINE_COMMON_CORE_BASIC_TYPES_H_ diff --git a/media/libeffects/loudness/common/core/byte_swapper.h b/media/libeffects/loudness/common/core/byte_swapper.h new file mode 100644 index 0000000..8f0caf3 --- /dev/null +++ b/media/libeffects/loudness/common/core/byte_swapper.h @@ -0,0 +1,151 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LE_FX_ENGINE_COMMON_CORE_BYTE_SWAPPER_H_ +#define LE_FX_ENGINE_COMMON_CORE_BYTE_SWAPPER_H_ + +#include <stdio.h> +#include <string.h> + +#include "common/core/basic_types.h" +#include "common/core/os.h" + +namespace le_fx { + +namespace arch { + +inline bool IsLittleEndian(void) { + int16 word = 1; + char *cp = reinterpret_cast<char *>(&word); + return cp[0] != 0; +} + +inline bool IsBigEndian(void) { + return !IsLittleEndian(); +} + +template <typename T, unsigned int kValSize> +struct ByteSwapper { + static T Swap(const T &val) { + T new_val = val; + char *first = &new_val, *last = first + kValSize - 1, x; + for (; first < last; ++first, --last) { + x = *last; + *last = *first; + *first = x; + } + return new_val; + } +}; + +template <typename T> +struct ByteSwapper<T, 1> { + static T Swap(const T &val) { + return val; + } +}; + +template <typename T> +struct ByteSwapper<T, 2> { + static T Swap(const T &val) { + T new_val; + const char *o = (const char *)&val; + char *p = reinterpret_cast<char *>(&new_val); + p[0] = o[1]; + p[1] = o[0]; + return new_val; + } +}; + +template <typename T> +struct ByteSwapper<T, 4> { + static T Swap(const T &val) { + T new_val; + const char *o = (const char *)&val; + char *p = reinterpret_cast<char *>(&new_val); + p[0] = o[3]; + p[1] = o[2]; + p[2] = o[1]; + p[3] = o[0]; + return new_val; + } +}; + +template <typename T> +struct ByteSwapper<T, 8> { + static T Swap(const T &val) { + T new_val = val; + const char *o = (const char *)&val; + char *p = reinterpret_cast<char *>(&new_val); + p[0] = o[7]; + p[1] = o[6]; + p[2] = o[5]; + p[3] = o[4]; + p[4] = o[3]; + p[5] = o[2]; + p[6] = o[1]; + p[7] = o[0]; + return new_val; + } +}; + +template <typename T> +T SwapBytes(const T &val, bool force_swap) { + if (force_swap) { +#if !defined(LE_FX__NEED_BYTESWAP) + return ByteSwapper<T, sizeof(T)>::Swap(val); +#else + return val; +#endif // !LE_FX_NEED_BYTESWAP + } else { +#if !defined(LE_FX_NEED_BYTESWAP) + return val; +#else + return ByteSwapper<T, sizeof(T)>::Swap(val); +#endif // !LE_FX_NEED_BYTESWAP + } +} + +template <typename T> +const T *SwapBytes(const T *vals, unsigned int num_items, bool force_swap) { + if (force_swap) { +#if !defined(LE_FX_NEED_BYTESWAP) + T *writeable_vals = const_cast<T *>(vals); + for (unsigned int i = 0; i < num_items; i++) { + writeable_vals[i] = ByteSwapper<T, sizeof(T)>::Swap(vals[i]); + } + return writeable_vals; +#else + return vals; +#endif // !LE_FX_NEED_BYTESWAP + } else { +#if !defined(LE_FX_NEED_BYTESWAP) + return vals; +#else + T *writeable_vals = const_cast<T *>(vals); + for (unsigned int i = 0; i < num_items; i++) { + writeable_vals[i] = ByteSwapper<T, sizeof(T)>::Swap(vals[i]); + } + return writeable_vals; +#endif // !LE_FX_NEED_BYTESWAP + } +} + +} // namespace arch + +} // namespace le_fx + +#endif // LE_FX_ENGINE_COMMON_CORE_BYTE_SWAPPER_H_ diff --git a/media/libeffects/loudness/common/core/math.h b/media/libeffects/loudness/common/core/math.h new file mode 100644 index 0000000..3f302cc --- /dev/null +++ b/media/libeffects/loudness/common/core/math.h @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LE_FX_ENGINE_COMMON_CORE_MATH_H_ +#define LE_FX_ENGINE_COMMON_CORE_MATH_H_ + +#include <math.h> +#include <algorithm> +using ::std::min; +using ::std::max; +using ::std::fill; +using ::std::fill_n;using ::std::lower_bound; +#include <cmath> +#include <math.h> +//using ::std::fpclassify; + +#include "common/core/os.h" +#include "common/core/types.h" + +namespace le_fx { +namespace math { + +// A fast approximation to log2(.) +inline float fast_log2(float val) { + int* const exp_ptr = reinterpret_cast <int *> (&val); + int x = *exp_ptr; + const int log_2 = ((x >> 23) & 255) - 128; + x &= ~(255 << 23); + x += 127 << 23; + *exp_ptr = x; + val = ((-1.0f / 3) * val + 2) * val - 2.0f / 3; + return static_cast<float>(val + log_2); +} + +// A fast approximation to log(.) +inline float fast_log(float val) { + return fast_log2(val) * + 0.693147180559945286226763982995180413126945495605468750f; +} + +// An approximation of the exp(.) function using a 5-th order Taylor expansion. +// It's pretty accurate between +-0.1 and accurate to 10e-3 between +-1 +template <typename T> +inline T ExpApproximationViaTaylorExpansionOrder5(T x) { + const T x2 = x * x; + const T x3 = x2 * x; + const T x4 = x2 * x2; + const T x5 = x3 * x2; + return 1.0f + x + 0.5f * x2 + + 0.16666666666666665741480812812369549646973609924316406250f * x3 + + 0.0416666666666666643537020320309238741174340248107910156250f * x4 + + 0.008333333333333333217685101601546193705871701240539550781250f * x5; +} + +} // namespace math +} // namespace le_fx + +// Math functions missing in Android NDK: +#if defined(LE_FX_OS_ANDROID) + +namespace std { + +// +// Round to the nearest integer: We need this implementation +// since std::round is missing on android. +// +template <typename T> +inline T round(const T &x) { + return static_cast<T>(std::floor(static_cast<double>(x) + 0.5)); +} + +} // namespace std + +#endif // LE_FX_OS_ANDROID + +#endif // LE_FX_ENGINE_COMMON_CORE_MATH_H_ diff --git a/media/libeffects/loudness/common/core/os.h b/media/libeffects/loudness/common/core/os.h new file mode 100644 index 0000000..4a8ce82 --- /dev/null +++ b/media/libeffects/loudness/common/core/os.h @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LE_FX_ENGINE_COMMON_CORE_OS_H_ +#define LE_FX_ENGINE_COMMON_CORE_OS_H_ + +// ----------------------------------------------------------------------------- +// OS Identification: +// ----------------------------------------------------------------------------- + +#define LE_FX_OS_UNIX +#if defined(__ANDROID__) +# define LE_FX_OS_ANDROID +#endif // Android + +#endif // LE_FX_ENGINE_COMMON_CORE_OS_H_ diff --git a/media/libeffects/loudness/common/core/types.h b/media/libeffects/loudness/common/core/types.h new file mode 100644 index 0000000..d1b6c6a --- /dev/null +++ b/media/libeffects/loudness/common/core/types.h @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LE_FX_ENGINE_COMMON_CORE_TYPES_H_ +#define LE_FX_ENGINE_COMMON_CORE_TYPES_H_ + +#include "common/core/os.h" + +#include "common/core/basic_types.h" + +#ifndef LE_FX_DISALLOW_COPY_AND_ASSIGN +#define LE_FX_DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&); \ + void operator=(const TypeName&) +#endif // LE_FX_DISALLOW_COPY_AND_ASSIGN + + +#endif // LE_FX_ENGINE_COMMON_CORE_TYPES_H_ diff --git a/media/libeffects/loudness/dsp/core/basic-inl.h b/media/libeffects/loudness/dsp/core/basic-inl.h new file mode 100644 index 0000000..3f77147 --- /dev/null +++ b/media/libeffects/loudness/dsp/core/basic-inl.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LE_FX_ENGINE_DSP_CORE_BASIC_INL_H_ +#define LE_FX_ENGINE_DSP_CORE_BASIC_INL_H_ + +#include <math.h> + +namespace le_fx { + +namespace sigmod { + +template <typename T> +int SearchIndex(const T x_data[], + T x, + int start_index, + int end_index) { + int start = start_index; + int end = end_index; + while (end > start + 1) { + int i = (end + start) / 2; + if (x_data[i] > x) { + end = i; + } else { + start = i; + } + } + return start; +} + +} // namespace sigmod + +} // namespace le_fx + +#endif // LE_FX_ENGINE_DSP_CORE_BASIC_INL_H_ diff --git a/media/libeffects/loudness/dsp/core/basic.h b/media/libeffects/loudness/dsp/core/basic.h new file mode 100644 index 0000000..27e0a8d --- /dev/null +++ b/media/libeffects/loudness/dsp/core/basic.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LE_FX_ENGINE_DSP_CORE_BASIC_H_ +#define LE_FX_ENGINE_DSP_CORE_BASIC_H_ + +#include <limits.h> +#include "common/core/math.h" +#include "common/core/types.h" + +namespace le_fx { + +namespace sigmod { + +// Searchs for the interval that contains <x> using a divide-and-conquer +// algorithm. +// X[]: a vector of sorted values (X[i+1] > X[i]) +// x: a value +// StartIndex: the minimum searched index +// EndIndex: the maximum searched index +// returns: the index <i> that satisfies: X[i] <= x <= X[i+1] && +// StartIndex <= i <= (EndIndex-1) +template <typename T> +int SearchIndex(const T x_data[], + T x, + int start_index, + int end_index); + +} // namespace sigmod + +} // namespace le_fx + +#include "dsp/core/basic-inl.h" + +#endif // LE_FX_ENGINE_DSP_CORE_BASIC_H_ diff --git a/media/libeffects/loudness/dsp/core/dynamic_range_compression-inl.h b/media/libeffects/loudness/dsp/core/dynamic_range_compression-inl.h new file mode 100644 index 0000000..da75ceb --- /dev/null +++ b/media/libeffects/loudness/dsp/core/dynamic_range_compression-inl.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LE_FX_ENGINE_DSP_CORE_DYNAMIC_RANGE_COMPRESSION_INL_H_ +#define LE_FX_ENGINE_DSP_CORE_DYNAMIC_RANGE_COMPRESSION_INL_H_ + +//#define LOG_NDEBUG 0 +#include <cutils/log.h> + + +namespace le_fx { + + +inline void AdaptiveDynamicRangeCompression::set_knee_threshold(float decibel) { + // Converts to 1og-base + knee_threshold_in_decibel_ = decibel; + knee_threshold_ = 0.1151292546497023061569109358970308676362037658691406250f * + decibel + 10.39717719035538401328722102334722876548767089843750f; +} + + +inline void AdaptiveDynamicRangeCompression::set_knee_threshold_via_target_gain( + float target_gain) { + const float decibel = target_gain_to_knee_threshold_.Interpolate( + target_gain); + ALOGV("set_knee_threshold_via_target_gain: decibel =%.3fdB", decibel); + set_knee_threshold(decibel); +} + +} // namespace le_fx + + +#endif // LE_FX_ENGINE_DSP_CORE_DYNAMIC_RANGE_COMPRESSION_INL_H_ diff --git a/media/libeffects/loudness/dsp/core/dynamic_range_compression.cpp b/media/libeffects/loudness/dsp/core/dynamic_range_compression.cpp new file mode 100644 index 0000000..7bd068e --- /dev/null +++ b/media/libeffects/loudness/dsp/core/dynamic_range_compression.cpp @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <cmath> + +#include "common/core/math.h" +#include "common/core/types.h" +#include "dsp/core/basic.h" +#include "dsp/core/interpolation.h" +#include "dsp/core/dynamic_range_compression.h" + +//#define LOG_NDEBUG 0 +#include <cutils/log.h> + + +namespace le_fx { + +// Definitions for static const class members declared in +// dynamic_range_compression.h. +const float AdaptiveDynamicRangeCompression::kMinAbsValue = 0.000001f; +const float AdaptiveDynamicRangeCompression::kMinLogAbsValue = + 0.032766999999999997517097227728299912996590137481689453125f; +const float AdaptiveDynamicRangeCompression::kFixedPointLimit = 32767.0f; +const float AdaptiveDynamicRangeCompression::kInverseFixedPointLimit = + 1.0f / AdaptiveDynamicRangeCompression::kFixedPointLimit; +const float AdaptiveDynamicRangeCompression::kDefaultKneeThresholdInDecibel = + -8.0f; +const float AdaptiveDynamicRangeCompression::kCompressionRatio = 7.0f; +const float AdaptiveDynamicRangeCompression::kTauAttack = 0.001f; +const float AdaptiveDynamicRangeCompression::kTauRelease = 0.015f; + +AdaptiveDynamicRangeCompression::AdaptiveDynamicRangeCompression() { + static const float kTargetGain[] = { + 1.0f, 2.0f, 3.0f, 4.0f, 5.0f }; + static const float kKneeThreshold[] = { + -8.0f, -8.0f, -8.5f, -9.0f, -10.0f }; + target_gain_to_knee_threshold_.Initialize( + &kTargetGain[0], &kKneeThreshold[0], + sizeof(kTargetGain) / sizeof(kTargetGain[0])); +} + +bool AdaptiveDynamicRangeCompression::Initialize( + float target_gain, float sampling_rate) { + set_knee_threshold_via_target_gain(target_gain); + sampling_rate_ = sampling_rate; + state_ = 0.0f; + compressor_gain_ = 1.0f; + if (kTauAttack > 0.0f) { + const float taufs = kTauAttack * sampling_rate_; + alpha_attack_ = std::exp(-1.0f / taufs); + } else { + alpha_attack_ = 0.0f; + } + if (kTauRelease > 0.0f) { + const float taufs = kTauRelease * sampling_rate_; + alpha_release_ = std::exp(-1.0f / taufs); + } else { + alpha_release_ = 0.0f; + } + // Feed-forward topology + slope_ = 1.0f / kCompressionRatio - 1.0f; + return true; +} + +float AdaptiveDynamicRangeCompression::Compress(float x) { + const float max_abs_x = std::max(std::fabs(x), kMinLogAbsValue); + const float max_abs_x_dB = math::fast_log(max_abs_x); + // Subtract Threshold from log-encoded input to get the amount of overshoot + const float overshoot = max_abs_x_dB - knee_threshold_; + // Hard half-wave rectifier + const float rect = std::max(overshoot, 0.0f); + // Multiply rectified overshoot with slope + const float cv = rect * slope_; + const float prev_state = state_; + if (cv <= state_) { + state_ = alpha_attack_ * state_ + (1.0f - alpha_attack_) * cv; + } else { + state_ = alpha_release_ * state_ + (1.0f - alpha_release_) * cv; + } + compressor_gain_ *= + math::ExpApproximationViaTaylorExpansionOrder5(state_ - prev_state); + x *= compressor_gain_; + if (x > kFixedPointLimit) { + return kFixedPointLimit; + } + if (x < -kFixedPointLimit) { + return -kFixedPointLimit; + } + return x; +} + +void AdaptiveDynamicRangeCompression::Compress(float *x1, float *x2) { + // Taking the maximum amplitude of both channels + const float max_abs_x = std::max(std::fabs(*x1), + std::max(std::fabs(*x2), kMinLogAbsValue)); + const float max_abs_x_dB = math::fast_log(max_abs_x); + // Subtract Threshold from log-encoded input to get the amount of overshoot + const float overshoot = max_abs_x_dB - knee_threshold_; + // Hard half-wave rectifier + const float rect = std::max(overshoot, 0.0f); + // Multiply rectified overshoot with slope + const float cv = rect * slope_; + const float prev_state = state_; + if (cv <= state_) { + state_ = alpha_attack_ * state_ + (1.0f - alpha_attack_) * cv; + } else { + state_ = alpha_release_ * state_ + (1.0f - alpha_release_) * cv; + } + compressor_gain_ *= + math::ExpApproximationViaTaylorExpansionOrder5(state_ - prev_state); + *x1 *= compressor_gain_; + if (*x1 > kFixedPointLimit) { + *x1 = kFixedPointLimit; + } + if (*x1 < -kFixedPointLimit) { + *x1 = -kFixedPointLimit; + } + *x2 *= compressor_gain_; + if (*x2 > kFixedPointLimit) { + *x2 = kFixedPointLimit; + } + if (*x2 < -kFixedPointLimit) { + *x2 = -kFixedPointLimit; + } +} + +} // namespace le_fx + diff --git a/media/libeffects/loudness/dsp/core/dynamic_range_compression.h b/media/libeffects/loudness/dsp/core/dynamic_range_compression.h new file mode 100644 index 0000000..2821a78 --- /dev/null +++ b/media/libeffects/loudness/dsp/core/dynamic_range_compression.h @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LE_FX_ENGINE_DSP_CORE_DYNAMIC_RANGE_COMPRESSION_H_ +#define LE_FX_ENGINE_DSP_CORE_DYNAMIC_RANGE_COMPRESSION_H_ + +#include "common/core/types.h" +#include "common/core/math.h" +#include "dsp/core/basic.h" +#include "dsp/core/interpolation.h" + +//#define LOG_NDEBUG 0 +#include <cutils/log.h> + + +namespace le_fx { + +// An adaptive dynamic range compression algorithm. The gain adaptation is made +// at the logarithmic domain and it is based on a Branching-Smooth compensated +// digital peak detector with different time constants for attack and release. +class AdaptiveDynamicRangeCompression { + public: + AdaptiveDynamicRangeCompression(); + + // Initializes the compressor using prior information. It assumes that the + // input signal is speech from high-quality recordings that is scaled and then + // fed to the compressor. The compressor is tuned according to the target gain + // that is expected to be applied. + // + // Target gain receives values between 0.0 and 10.0. The knee threshold is + // reduced as the target gain increases in order to fit the increased range of + // values. + // + // Values between 1.0 and 2.0 will only mildly affect your signal. Higher + // values will reduce the dynamic range of the signal to the benefit of + // increased loudness. + // + // If nothing is known regarding the input, a `target_gain` of 1.0f is a + // relatively safe choice for many signals. + bool Initialize(float target_gain, float sampling_rate); + + // A fast version of the algorithm that uses approximate computations for the + // log(.) and exp(.). + float Compress(float x); + + // Stereo channel version of the compressor + void Compress(float *x1, float *x2); + + // This version is slower than Compress(.) but faster than CompressSlow(.) + float CompressNormalSpeed(float x); + + // A slow version of the algorithm that is easier for further developement, + // tuning and debugging + float CompressSlow(float x); + + // Sets knee threshold (in decibel). + void set_knee_threshold(float decibel); + + // Sets knee threshold via the target gain using an experimentally derived + // relationship. + void set_knee_threshold_via_target_gain(float target_gain); + + private: + // The minimum accepted absolute input value and it's natural logarithm. This + // is to prevent numerical issues when the input is close to zero + static const float kMinAbsValue; + static const float kMinLogAbsValue; + // Fixed-point arithmetic limits + static const float kFixedPointLimit; + static const float kInverseFixedPointLimit; + // The default knee threshold in decibel. The knee threshold defines when the + // compressor is actually starting to compress the value of the input samples + static const float kDefaultKneeThresholdInDecibel; + // The compression ratio is the reciprocal of the slope of the line segment + // above the threshold (in the log-domain). The ratio controls the + // effectiveness of the compression. + static const float kCompressionRatio; + // The attack time of the envelope detector + static const float kTauAttack; + // The release time of the envelope detector + static const float kTauRelease; + + float sampling_rate_; + // the internal state of the envelope detector + float state_; + // the latest gain factor that was applied to the input signal + float compressor_gain_; + // attack constant for exponential dumping + float alpha_attack_; + // release constant for exponential dumping + float alpha_release_; + float slope_; + // The knee threshold + float knee_threshold_; + float knee_threshold_in_decibel_; + // This interpolator provides the function that relates target gain to knee + // threshold. + sigmod::InterpolatorLinear<float> target_gain_to_knee_threshold_; + + LE_FX_DISALLOW_COPY_AND_ASSIGN(AdaptiveDynamicRangeCompression); +}; + +} // namespace le_fx + +#include "dsp/core/dynamic_range_compression-inl.h" + +#endif // LE_FX_ENGINE_DSP_CORE_DYNAMIC_RANGE_COMPRESSION_H_ diff --git a/media/libeffects/loudness/dsp/core/interpolation.h b/media/libeffects/loudness/dsp/core/interpolation.h new file mode 100644 index 0000000..23c287c --- /dev/null +++ b/media/libeffects/loudness/dsp/core/interpolation.h @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LE_FX_ENGINE_DSP_CORE_INTERPOLATION_H_ +#define LE_FX_ENGINE_DSP_CORE_INTERPOLATION_H_ + +#include "common/core/math.h" +#include "dsp/core/interpolator_base.h" +#include "dsp/core/interpolator_linear.h" + +#endif // LE_FX_ENGINE_DSP_CORE_INTERPOLATION_H_ + diff --git a/media/libeffects/loudness/dsp/core/interpolator_base-inl.h b/media/libeffects/loudness/dsp/core/interpolator_base-inl.h new file mode 100644 index 0000000..bd08b65 --- /dev/null +++ b/media/libeffects/loudness/dsp/core/interpolator_base-inl.h @@ -0,0 +1,180 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LE_FX_ENGINE_DSP_CORE_INTERPOLATOR_BASE_INL_H_ +#define LE_FX_ENGINE_DSP_CORE_INTERPOLATOR_BASE_INL_H_ + +#include "dsp/core/basic.h" + +//#define LOG_NDEBUG 0 +#include <cutils/log.h> + + +namespace le_fx { + +namespace sigmod { + +template <typename T, class Algorithm> +InterpolatorBase<T, Algorithm>::InterpolatorBase() { + status_ = false; + cached_index_ = 0; + x_data_ = NULL; + y_data_ = NULL; + data_length_ = 0; + own_x_data_ = false; + x_start_offset_ = 0.0; + last_element_index_ = -1; + x_inverse_sampling_interval_ = 0.0; + state_ = NULL; +} + +template <typename T, class Algorithm> +InterpolatorBase<T, Algorithm>::~InterpolatorBase() { + delete [] state_; + if (own_x_data_) { + delete [] x_data_; + } +} + +template <typename T, class Algorithm> +bool InterpolatorBase<T, Algorithm>::Initialize(const vector<T> &x_data, + const vector<T> &y_data) { +#ifndef NDEBUG + if (x_data.size() != y_data.size()) { + LoggerError("InterpolatorBase::Initialize: xData size (%d) != yData size" + " (%d)", x_data.size(), y_data.size()); + } +#endif + return Initialize(&x_data[0], &y_data[0], x_data.size()); +} + +template <typename T, class Algorithm> +bool InterpolatorBase<T, Algorithm>::Initialize(double x_start_offset, + double x_sampling_interval, + const vector<T> &y_data) { + return Initialize(x_start_offset, + x_sampling_interval, + &y_data[0], + y_data.size()); +} + +template <typename T, class Algorithm> +bool InterpolatorBase<T, Algorithm>::Initialize(double x_start_offset, + double x_sampling_interval, + const T *y_data, + int data_length) { + // Constructs and populate x-axis data: `x_data_` + T *x_data_tmp = new T[data_length]; + float time_offset = x_start_offset; + for (int n = 0; n < data_length; n++) { + x_data_tmp[n] = time_offset; + time_offset += x_sampling_interval; + } + Initialize(x_data_tmp, y_data, data_length); + // Sets-up the regularly sampled interpolation mode + x_start_offset_ = x_start_offset; + x_inverse_sampling_interval_ = 1.0 / x_sampling_interval; + own_x_data_ = true; + return status_; +} + + +template <typename T, class Algorithm> +bool InterpolatorBase<T, Algorithm>::Initialize( + const T *x_data, const T *y_data, int data_length) { + // Default settings + cached_index_ = 0; + data_length_ = 0; + x_start_offset_ = 0; + x_inverse_sampling_interval_ = 0; + state_ = NULL; + // Input data is externally owned + own_x_data_ = false; + x_data_ = x_data; + y_data_ = y_data; + data_length_ = data_length; + last_element_index_ = data_length - 1; + // Check input data sanity + for (int n = 0; n < last_element_index_; ++n) { + if (x_data_[n + 1] <= x_data_[n]) { + ALOGE("InterpolatorBase::Initialize: xData are not ordered or " + "contain equal values (X[%d] <= X[%d]) (%.5e <= %.5e)", + n + 1, n, x_data_[n + 1], x_data_[n]); + status_ = false; + return false; + } + } + // Pre-compute internal state by calling the corresponding function of the + // derived class. + status_ = static_cast<Algorithm*>(this)->SetInternalState(); + return status_; +} + +template <typename T, class Algorithm> +T InterpolatorBase<T, Algorithm>::Interpolate(T x) { +#ifndef NDEBUG + if (cached_index_ < 0 || cached_index_ > data_length_ - 2) { + LoggerError("InterpolatorBase:Interpolate: CachedIndex_ out of bounds " + "[0, %d, %d]", cached_index_, data_length_ - 2); + } +#endif + // Search for the containing interval + if (x <= x_data_[cached_index_]) { + if (cached_index_ <= 0) { + cached_index_ = 0; + return y_data_[0]; + } + if (x >= x_data_[cached_index_ - 1]) { + cached_index_--; // Fast descending + } else { + if (x <= x_data_[0]) { + cached_index_ = 0; + return y_data_[0]; + } + cached_index_ = SearchIndex(x_data_, x, 0, cached_index_); + } + } else { + if (cached_index_ >= last_element_index_) { + cached_index_ = last_element_index_; + return y_data_[last_element_index_]; + } + if (x > x_data_[cached_index_ + 1]) { + if (cached_index_ + 2 > last_element_index_) { + cached_index_ = last_element_index_ - 1; + return y_data_[last_element_index_]; + } + if (x <= x_data_[cached_index_ + 2]) { + cached_index_++; // Fast ascending + } else { + if (x >= x_data_[last_element_index_]) { + cached_index_ = last_element_index_ - 1; + return y_data_[last_element_index_]; + } + cached_index_ = SearchIndex( + x_data_, x, cached_index_, last_element_index_); + } + } + } + // Compute interpolated value by calling the corresponding function of the + // derived class. + return static_cast<Algorithm*>(this)->MethodSpecificInterpolation(x); +} + +} // namespace sigmod + +} // namespace le_fx + +#endif // LE_FX_ENGINE_DSP_CORE_INTERPOLATOR_BASE_INL_H_ diff --git a/media/libeffects/loudness/dsp/core/interpolator_base.h b/media/libeffects/loudness/dsp/core/interpolator_base.h new file mode 100644 index 0000000..0cd1a35 --- /dev/null +++ b/media/libeffects/loudness/dsp/core/interpolator_base.h @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LE_FX_ENGINE_DSP_CORE_INTERPOLATOR_BASE_H_ +#define LE_FX_ENGINE_DSP_CORE_INTERPOLATOR_BASE_H_ + +#include "common/core/types.h" + +namespace le_fx { + +namespace sigmod { + +// Interpolation base-class that provides the interface, while it is the derived +// class that provides the specific interpolation algorithm. The following list +// of interpolation algorithms are currently present: +// +// InterpolationSine<T>: weighted interpolation between y_data[n] and +// y_data[n+1] using a sin(.) weighting factor from +// 0 to pi/4. +// InterpolationLinear<T>: linear interpolation +// InterpolationSplines<T>: spline-based interpolation +// +// Example (using derived spline-based interpolation class): +// InterpolatorSplines<float> interp(x_data, y_data, data_length); +// for (int n = 0; n < data_length; n++) Y[n] = interp.Interpolate(X[n]); +// +template <typename T, class Algorithm> +class InterpolatorBase { + public: + InterpolatorBase(); + ~InterpolatorBase(); + + // Generic random-access interpolation with arbitrary spaced x-axis samples. + // Below X[0], the interpolator returns Y[0]. Above X[data_length-1], it + // returns Y[data_length-1]. + T Interpolate(T x); + + bool get_status() const { + return status_; + } + + // Initializes internal buffers. + // x_data: [(data_length)x1] x-axis coordinates (searching axis) + // y_data: [(data_length)x1] y-axis coordinates (interpolation axis) + // data_length: number of points + // returns `true` if everything is ok, `false`, otherwise + bool Initialize(const T *x_data, const T *y_data, int data_length); + + // Initializes internal buffers. + // x_data: x-axis coordinates (searching axis) + // y_data: y-axis coordinates (interpolating axis) + // returns `true` if everything is ok, `false`, otherwise + bool Initialize(const vector<T> &x_data, const vector<T> &y_data); + + // Initialization for regularly sampled sequences, where: + // x_data[i] = x_start_offset + i * x_sampling_interval + bool Initialize(double x_start_offset, + double x_sampling_interval, + const vector<T> &y_data); + + // Initialization for regularly sampled sequences, where: + // x_data[i] = x_start_offset + i * x_sampling_interval + bool Initialize(double x_start_offset, + double x_sampling_interval, + const T *y_data, + int data_length); + + protected: + // Is set to false if something goes wrong, and to true if everything is ok. + bool status_; + + // The start-index of the previously searched interval + int cached_index_; + + // Data points + const T *x_data_; // Externally or internally owned, depending on own_x_data_ + const T *y_data_; // Externally owned (always) + int data_length_; + // Index of the last element `data_length_ - 1` kept here for optimization + int last_element_index_; + bool own_x_data_; + // For regularly-samples sequences, keep only the boundaries and the intervals + T x_start_offset_; + float x_inverse_sampling_interval_; + + // Algorithm state (internally owned) + double *state_; + + private: + LE_FX_DISALLOW_COPY_AND_ASSIGN(InterpolatorBase); +}; + +} // namespace sigmod + +} // namespace le_fx + +#include "dsp/core/interpolator_base-inl.h" + +#endif // LE_FX_ENGINE_DSP_CORE_INTERPOLATOR_BASE_H_ diff --git a/media/libeffects/loudness/dsp/core/interpolator_linear.h b/media/libeffects/loudness/dsp/core/interpolator_linear.h new file mode 100644 index 0000000..434698a --- /dev/null +++ b/media/libeffects/loudness/dsp/core/interpolator_linear.h @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LE_FX_ENGINE_DSP_CORE_INTERPOLATOR_LINEAR_H_ +#define LE_FX_ENGINE_DSP_CORE_INTERPOLATOR_LINEAR_H_ + +#include <math.h> +#include "dsp/core/interpolator_base.h" + +namespace le_fx { + +namespace sigmod { + +// Linear interpolation class. +// +// The main functionality of this class is provided by it's base-class, so +// please refer to: InterpolatorBase +// +// Example: +// InterpolatorLinear<float> interp(x_data, y_data, data_length); +// for (int n = 0; n < data_length; n++) Y[n] = interp.Interpolate(X[n]); +// +template <typename T> +class InterpolatorLinear: public InterpolatorBase<T, InterpolatorLinear<T> > { + public: + InterpolatorLinear() { } + ~InterpolatorLinear() { } + + protected: + // Provides the main implementation of the linear interpolation algorithm. + // Assumes that: X[cached_index_] < x < X[cached_index_ + 1] + T MethodSpecificInterpolation(T x); + + // Pre-compute internal state_ parameters. + bool SetInternalState(); + + private: + friend class InterpolatorBase<T, InterpolatorLinear<T> >; + typedef InterpolatorBase<T, InterpolatorLinear<T> > BaseClass; + using BaseClass::status_; + using BaseClass::cached_index_; + using BaseClass::x_data_; + using BaseClass::y_data_; + using BaseClass::data_length_; + using BaseClass::state_; + + LE_FX_DISALLOW_COPY_AND_ASSIGN(InterpolatorLinear<T>); +}; + +template <typename T> +inline T InterpolatorLinear<T>::MethodSpecificInterpolation(T x) { + T dX = x_data_[cached_index_ + 1] - x_data_[cached_index_]; + T dY = y_data_[cached_index_ + 1] - y_data_[cached_index_]; + T dx = x - x_data_[cached_index_]; + return y_data_[cached_index_] + (dY * dx) / dX; +} + +template <typename T> +bool InterpolatorLinear<T>::SetInternalState() { + state_ = NULL; + return true; +} + +} // namespace sigmod + +} // namespace le_fx + +#endif // LE_FX_ENGINE_DSP_CORE_INTERPOLATOR_LINEAR_H_ diff --git a/media/libeffects/proxy/Android.mk b/media/libeffects/proxy/Android.mk new file mode 100644 index 0000000..01b3be1 --- /dev/null +++ b/media/libeffects/proxy/Android.mk @@ -0,0 +1,34 @@ +# Copyright 2013 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +LOCAL_PATH:= $(call my-dir) +include $(CLEAR_VARS) +LOCAL_MODULE:= libeffectproxy +LOCAL_MODULE_PATH := $(TARGET_OUT_SHARED_LIBRARIES)/soundfx +LOCAL_MODULE_TAGS := optional + + +LOCAL_SRC_FILES := \ + EffectProxy.cpp + +LOCAL_CFLAGS+= -fvisibility=hidden + +LOCAL_SHARED_LIBRARIES := liblog libcutils libutils libdl libeffects + +LOCAL_C_INCLUDES := \ + system/media/audio_effects/include \ + bionic/libc/include + +include $(BUILD_SHARED_LIBRARY) + diff --git a/media/libeffects/proxy/EffectProxy.cpp b/media/libeffects/proxy/EffectProxy.cpp new file mode 100644 index 0000000..dd4ad08 --- /dev/null +++ b/media/libeffects/proxy/EffectProxy.cpp @@ -0,0 +1,339 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "EffectProxy" +//#define LOG_NDEBUG 0 + +#include <cutils/log.h> +#include <assert.h> +#include <stdlib.h> +#include <string.h> +#include <new> +#include <EffectProxy.h> +#include <utils/threads.h> +#include <media/EffectsFactoryApi.h> + +namespace android { +// This is a dummy proxy descriptor just to return to Factory during the initial +// GetDescriptor call. Later in the factory, it is replaced with the +// SW sub effect descriptor +// proxy UUID af8da7e0-2ca1-11e3-b71d-0002a5d5c51b +const effect_descriptor_t gProxyDescriptor = { + EFFECT_UUID_INITIALIZER, // type + {0xaf8da7e0, 0x2ca1, 0x11e3, 0xb71d, { 0x00, 0x02, 0xa5, 0xd5, 0xc5, 0x1b }}, // uuid + EFFECT_CONTROL_API_VERSION, //version of effect control API + (EFFECT_FLAG_TYPE_INSERT | EFFECT_FLAG_INSERT_LAST | + EFFECT_FLAG_VOLUME_CTRL), // effect capability flags + 0, // CPU load + 1, // Data memory + "Proxy", //effect name + "AOSP", //implementor name +}; + + +static const effect_descriptor_t *const gDescriptors[] = +{ + &gProxyDescriptor, +}; + + +int EffectProxyCreate(const effect_uuid_t *uuid, + int32_t sessionId, + int32_t ioId, + effect_handle_t *pHandle) { + + effect_descriptor_t* desc; + EffectContext* pContext; + if (pHandle == NULL || uuid == NULL) { + ALOGE("EffectProxyCreate() called with NULL pointer"); + return -EINVAL; + } + ALOGV("EffectProxyCreate start.."); + pContext = new EffectContext; + pContext->sessionId = sessionId; + pContext->ioId = ioId; + pContext->uuid = *uuid; + pContext->common_itfe = &gEffectInterface; + + // The sub effects will be created in effect_command when the first command + // for the effect is received + pContext->eHandle[SUB_FX_HOST] = pContext->eHandle[SUB_FX_OFFLOAD] = NULL; + + // Get the HW and SW sub effect descriptors from the effects factory + desc = new effect_descriptor_t[SUB_FX_COUNT]; + pContext->desc = new effect_descriptor_t[SUB_FX_COUNT]; + int retValue = EffectGetSubEffects(uuid, desc, + sizeof(effect_descriptor_t) * SUB_FX_COUNT); + // EffectGetSubEffects returns the number of sub-effects copied. + if (retValue != SUB_FX_COUNT) { + ALOGE("EffectCreate() could not get the sub effects"); + delete desc; + delete pContext->desc; + return -EINVAL; + } + // Check which is the HW descriptor and copy the descriptors + // to the Context desc array + // Also check if there is only one HW and one SW descriptor. + // HW descriptor alone has the HW_TUNNEL flag. + if ((desc[0].flags & EFFECT_FLAG_HW_ACC_TUNNEL) && + !(desc[1].flags & EFFECT_FLAG_HW_ACC_TUNNEL)) { + pContext->desc[SUB_FX_OFFLOAD] = desc[0]; + pContext->desc[SUB_FX_HOST] = desc[1]; + } + else if ((desc[1].flags & EFFECT_FLAG_HW_ACC_TUNNEL) && + !(desc[0].flags & EFFECT_FLAG_HW_ACC_TUNNEL)) { + pContext->desc[SUB_FX_HOST] = desc[0]; + pContext->desc[SUB_FX_OFFLOAD] = desc[1]; + } + delete desc; +#if (LOG_NDEBUG == 0) + effect_uuid_t uuid_print = pContext->desc[SUB_FX_HOST].uuid; + ALOGV("EffectCreate() UUID of HOST: %08X-%04X-%04X-%04X-%02X%02X%02X%02X" + "%02X%02X\n",uuid_print.timeLow, uuid_print.timeMid, + uuid_print.timeHiAndVersion, uuid_print.clockSeq, uuid_print.node[0], + uuid_print.node[1], uuid_print.node[2], uuid_print.node[3], + uuid_print.node[4], uuid_print.node[5]); + ALOGV("EffectCreate() UUID of OFFLOAD: %08X-%04X-%04X-%04X-%02X%02X%02X%02X" + "%02X%02X\n", uuid_print.timeLow, uuid_print.timeMid, + uuid_print.timeHiAndVersion, uuid_print.clockSeq, uuid_print.node[0], + uuid_print.node[1], uuid_print.node[2], uuid_print.node[3], + uuid_print.node[4], uuid_print.node[5]); +#endif + + pContext->replySize = PROXY_REPLY_SIZE_DEFAULT; + pContext->replyData = (char *)malloc(PROXY_REPLY_SIZE_DEFAULT); + + *pHandle = (effect_handle_t)pContext; + ALOGV("EffectCreate end"); + return 0; +} //end EffectProxyCreate + +int EffectProxyRelease(effect_handle_t handle) { + EffectContext * pContext = (EffectContext *)handle; + if (pContext == NULL) { + ALOGV("ERROR : EffectRelease called with NULL pointer"); + return -EINVAL; + } + ALOGV("EffectRelease"); + delete pContext->desc; + free(pContext->replyData); + + if (pContext->eHandle[SUB_FX_HOST]) + EffectRelease(pContext->eHandle[SUB_FX_HOST]); + if (pContext->eHandle[SUB_FX_OFFLOAD]) + EffectRelease(pContext->eHandle[SUB_FX_OFFLOAD]); + delete pContext; + pContext = NULL; + return 0; +} /*end EffectProxyRelease */ + +int EffectProxyGetDescriptor(const effect_uuid_t *uuid, + effect_descriptor_t *pDescriptor) { + const effect_descriptor_t *desc = NULL; + + if (pDescriptor == NULL || uuid == NULL) { + ALOGV("EffectGetDescriptor() called with NULL pointer"); + return -EINVAL; + } + desc = &gProxyDescriptor; + *pDescriptor = *desc; + return 0; +} /* end EffectProxyGetDescriptor */ + +/* Effect Control Interface Implementation: Process */ +int Effect_process(effect_handle_t self, + audio_buffer_t *inBuffer, + audio_buffer_t *outBuffer) { + + EffectContext *pContext = (EffectContext *) self; + int ret = 0; + if (pContext != NULL) { + int index = pContext->index; + // if the index refers to HW , do not do anything. Just return. + if (index == SUB_FX_HOST) { + ret = (*pContext->eHandle[index])->process(pContext->eHandle[index], + inBuffer, outBuffer); + } + } + return ret; +} /* end Effect_process */ + +/* Effect Control Interface Implementation: Command */ +int Effect_command(effect_handle_t self, + uint32_t cmdCode, + uint32_t cmdSize, + void *pCmdData, + uint32_t *replySize, + void *pReplyData) { + + EffectContext *pContext = (EffectContext *) self; + int status = 0; + if (pContext == NULL) { + ALOGV("Effect_command() Proxy context is NULL"); + return -EINVAL; + } + if (pContext->eHandle[SUB_FX_HOST] == NULL) { + ALOGV("Effect_command() Calling HOST EffectCreate"); + status = EffectCreate(&pContext->desc[SUB_FX_HOST].uuid, + pContext->sessionId, pContext->ioId, + &(pContext->eHandle[SUB_FX_HOST])); + if (status != NO_ERROR || (pContext->eHandle[SUB_FX_HOST] == NULL)) { + ALOGV("Effect_command() Error creating SW sub effect"); + return status; + } + } + if (pContext->eHandle[SUB_FX_OFFLOAD] == NULL) { + ALOGV("Effect_command() Calling OFFLOAD EffectCreate"); + status = EffectCreate(&pContext->desc[SUB_FX_OFFLOAD].uuid, + pContext->sessionId, pContext->ioId, + &(pContext->eHandle[SUB_FX_OFFLOAD])); + if (status != NO_ERROR || (pContext->eHandle[SUB_FX_OFFLOAD] == NULL)) { + ALOGV("Effect_command() Error creating HW effect"); + // Do not return error here as SW effect is created + // Return error if the CMD_OFFLOAD sends the index as OFFLOAD + } + pContext->index = SUB_FX_HOST; + } + // EFFECT_CMD_OFFLOAD used to (1) send whether the thread is offload or not + // (2) Send the ioHandle of the effectThread when the effect + // is moved from one type of thread to another. + // pCmdData points to a memory holding effect_offload_param_t structure + if (cmdCode == EFFECT_CMD_OFFLOAD) { + ALOGV("Effect_command() cmdCode = EFFECT_CMD_OFFLOAD"); + if (cmdSize == 0 || pCmdData == NULL) { + ALOGV("effectsOffload: Effect_command: CMD_OFFLOAD has no data"); + *(int*)pReplyData = FAILED_TRANSACTION; + return FAILED_TRANSACTION; + } + effect_offload_param_t* offloadParam = (effect_offload_param_t*)pCmdData; + // Assign the effect context index based on isOffload field of the structure + pContext->index = offloadParam->isOffload ? SUB_FX_OFFLOAD : SUB_FX_HOST; + // if the index is HW and the HW effect is unavailable, return error + // and reset the index to SW + if (pContext->eHandle[pContext->index] == NULL) { + ALOGV("Effect_command()CMD_OFFLOAD sub effect unavailable"); + *(int*)pReplyData = FAILED_TRANSACTION; + return FAILED_TRANSACTION; + } + pContext->ioId = offloadParam->ioHandle; + ALOGV("Effect_command()CMD_OFFLOAD index:%d io %d", pContext->index, pContext->ioId); + // Update the DSP wrapper with the new ioHandle. + // Pass the OFFLOAD command to the wrapper. + // The DSP wrapper needs to handle this CMD + if (pContext->eHandle[SUB_FX_OFFLOAD]) + status = (*pContext->eHandle[SUB_FX_OFFLOAD])->command( + pContext->eHandle[SUB_FX_OFFLOAD], cmdCode, cmdSize, + pCmdData, replySize, pReplyData); + return status; + } + + int index = pContext->index; + if (index != SUB_FX_HOST && index != SUB_FX_OFFLOAD) { + ALOGV("Effect_command: effect index is neither offload nor host"); + return -EINVAL; + } + + // Getter commands are only sent to the active sub effect. + int *subStatus[SUB_FX_COUNT]; + uint32_t *subReplySize[SUB_FX_COUNT]; + void *subReplyData[SUB_FX_COUNT]; + uint32_t tmpSize; + int tmpStatus; + + // grow temp reply buffer if needed + if (replySize != NULL) { + tmpSize = pContext->replySize; + while (tmpSize < *replySize && tmpSize < PROXY_REPLY_SIZE_MAX) { + tmpSize *= 2; + } + if (tmpSize > pContext->replySize) { + ALOGV("Effect_command grow reply buf to %d", tmpSize); + pContext->replyData = (char *)realloc(pContext->replyData, tmpSize); + pContext->replySize = tmpSize; + } + if (tmpSize > *replySize) { + tmpSize = *replySize; + } + } else { + tmpSize = 0; + } + // tmpSize is now the actual reply size for the non active sub effect + + // Send command to sub effects. The command is sent to all sub effects so that their internal + // state is kept in sync. + // Only the reply from the active sub effect is returned to the caller. The reply from the + // other sub effect is lost in pContext->replyData + for (int i = 0; i < SUB_FX_COUNT; i++) { + if (pContext->eHandle[i] == NULL) { + continue; + } + if (i == index) { + subStatus[i] = &status; + subReplySize[i] = replySize; + subReplyData[i] = pReplyData; + } else { + subStatus[i] = &tmpStatus; + subReplySize[i] = replySize == NULL ? NULL : &tmpSize; + subReplyData[i] = pReplyData == NULL ? NULL : pContext->replyData; + } + *subStatus[i] = (*pContext->eHandle[i])->command( + pContext->eHandle[i], cmdCode, cmdSize, + pCmdData, subReplySize[i], subReplyData[i]); + } + + return status; +} /* end Effect_command */ + + +/* Effect Control Interface Implementation: get_descriptor */ +int Effect_getDescriptor(effect_handle_t self, + effect_descriptor_t *pDescriptor) { + + EffectContext * pContext = (EffectContext *) self; + const effect_descriptor_t *desc; + + ALOGV("Effect_getDescriptor"); + if (pContext == NULL || pDescriptor == NULL) { + ALOGV("Effect_getDescriptor() invalid param"); + return -EINVAL; + } + if (pContext->desc == NULL) { + ALOGV("Effect_getDescriptor() could not get descriptor"); + return -EINVAL; + } + desc = &pContext->desc[SUB_FX_HOST]; + *pDescriptor = *desc; + pDescriptor->uuid = pContext->uuid; // Replace the uuid with the Proxy UUID + // Also set/clear the EFFECT_FLAG_OFFLOAD_SUPPORTED flag based on the sub effects availability + if (pContext->eHandle[SUB_FX_OFFLOAD] != NULL) + pDescriptor->flags |= EFFECT_FLAG_OFFLOAD_SUPPORTED; + else + pDescriptor->flags &= ~EFFECT_FLAG_OFFLOAD_SUPPORTED; + return 0; +} /* end Effect_getDescriptor */ + +} // namespace android + +__attribute__ ((visibility ("default"))) +audio_effect_library_t AUDIO_EFFECT_LIBRARY_INFO_SYM = { + tag : AUDIO_EFFECT_LIBRARY_TAG, + version : EFFECT_LIBRARY_API_VERSION, + name : "Effect Proxy", + implementor : "AOSP", + create_effect : android::EffectProxyCreate, + release_effect : android::EffectProxyRelease, + get_descriptor : android::EffectProxyGetDescriptor, +}; diff --git a/media/libeffects/proxy/EffectProxy.h b/media/libeffects/proxy/EffectProxy.h new file mode 100644 index 0000000..acbe17e --- /dev/null +++ b/media/libeffects/proxy/EffectProxy.h @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <hardware/audio.h> +#include <hardware/audio_effect.h> +namespace android { +enum { + SUB_FX_HOST, // Index of HOST in the descriptor and handle arrays + // of the Proxy context + SUB_FX_OFFLOAD, // Index of OFFLOAD in the descriptor and handle arrays + // of the Proxy context + SUB_FX_COUNT // The number of sub effects for a Proxy(1 HW, 1 SW) +}; +#if __cplusplus +extern "C" { +#endif + +int EffectProxyCreate(const effect_uuid_t *uuid, + int32_t sessionId, + int32_t ioId, + effect_handle_t *pHandle); +int EffectProxyRelease(effect_handle_t handle); +int EffectProxyGetDescriptor(const effect_uuid_t *uuid, + effect_descriptor_t *pDescriptor); +/* Effect Control Interface Implementation: Process */ +int Effect_process(effect_handle_t self, + audio_buffer_t *inBuffer, + audio_buffer_t *outBuffer); + +/* Effect Control Interface Implementation: Command */ +int Effect_command(effect_handle_t self, + uint32_t cmdCode, + uint32_t cmdSize, + void *pCmdData, + uint32_t *replySize, + void *pReplyData); +int Effect_getDescriptor(effect_handle_t self, + effect_descriptor_t *pDescriptor); + +const struct effect_interface_s gEffectInterface = { + Effect_process, + Effect_command, + Effect_getDescriptor, + NULL, +}; + +#define PROXY_REPLY_SIZE_MAX (64 * 1024) // must be power of two +#define PROXY_REPLY_SIZE_DEFAULT 32 // must be power of two + +struct EffectContext { + const struct effect_interface_s *common_itfe; // Holds the itfe of the Proxy + effect_descriptor_t* desc; // Points to the sub effect descriptors + effect_handle_t eHandle[SUB_FX_COUNT]; // The effect handles of the sub effects + int index; // The index that is currently active - HOST or OFFLOAD + int32_t sessionId; // The sessiond in which the effect is created. + // Stored in context to pass on to sub effect creation + int32_t ioId; // The ioId in which the effect is created. + // Stored in context to pass on to sub effect creation + effect_uuid_t uuid; // UUID of the Proxy + char* replyData; // temporary buffer for non active sub effect command reply + uint32_t replySize; // current size of temporary reply buffer +}; + +#if __cplusplus +} // extern "C" +#endif +} //namespace android diff --git a/media/libeffects/testlibs/AudioFormatAdapter.h b/media/libeffects/testlibs/AudioFormatAdapter.h index 41f1810..dea2734 100644 --- a/media/libeffects/testlibs/AudioFormatAdapter.h +++ b/media/libeffects/testlibs/AudioFormatAdapter.h @@ -75,6 +75,7 @@ public: while (numSamples > 0) { uint32_t numSamplesIter = min(numSamples, mMaxSamplesPerCall); uint32_t nSamplesChannels = numSamplesIter * mNumChannels; + // This branch of "if" is untested if (mPcmFormat == AUDIO_FORMAT_PCM_8_24_BIT) { if (mBehavior == EFFECT_BUFFER_ACCESS_WRITE) { mpProcessor->process( diff --git a/media/libeffects/testlibs/EffectEqualizer.cpp b/media/libeffects/testlibs/EffectEqualizer.cpp index c35453b..8d00206 100644 --- a/media/libeffects/testlibs/EffectEqualizer.cpp +++ b/media/libeffects/testlibs/EffectEqualizer.cpp @@ -234,8 +234,7 @@ int Equalizer_setConfig(EqualizerContext *pContext, effect_config_t *pConfig) (pConfig->inputCfg.channels == AUDIO_CHANNEL_OUT_STEREO)); CHECK_ARG(pConfig->outputCfg.accessMode == EFFECT_BUFFER_ACCESS_WRITE || pConfig->outputCfg.accessMode == EFFECT_BUFFER_ACCESS_ACCUMULATE); - CHECK_ARG(pConfig->inputCfg.format == AUDIO_FORMAT_PCM_8_24_BIT - || pConfig->inputCfg.format == AUDIO_FORMAT_PCM_16_BIT); + CHECK_ARG(pConfig->inputCfg.format == AUDIO_FORMAT_PCM_16_BIT); int channelCount; if (pConfig->inputCfg.channels == AUDIO_CHANNEL_OUT_MONO) { diff --git a/media/libeffects/visualizer/EffectVisualizer.cpp b/media/libeffects/visualizer/EffectVisualizer.cpp index 9f8139c..2d66eef 100644 --- a/media/libeffects/visualizer/EffectVisualizer.cpp +++ b/media/libeffects/visualizer/EffectVisualizer.cpp @@ -22,6 +22,7 @@ #include <string.h> #include <new> #include <time.h> +#include <math.h> #include <audio_effects/effect_visualizer.h> @@ -54,6 +55,18 @@ enum visualizer_state_e { #define CAPTURE_BUF_SIZE 65536 // "64k should be enough for everyone" +#define DISCARD_MEASUREMENTS_TIME_MS 2000 // discard measurements older than this number of ms + +// maximum number of buffers for which we keep track of the measurements +#define MEASUREMENT_WINDOW_MAX_SIZE_IN_BUFFERS 25 // note: buffer index is stored in uint8_t + + +struct BufferStats { + bool mIsValid; + uint16_t mPeakU16; // the positive peak of the absolute value of the samples in a buffer + float mRmsSquared; // the average square of the samples in a buffer +}; + struct VisualizerContext { const struct effect_interface_s *mItfe; effect_config_t mConfig; @@ -61,15 +74,38 @@ struct VisualizerContext { uint32_t mCaptureSize; uint32_t mScalingMode; uint8_t mState; - uint8_t mLastCaptureIdx; + uint32_t mLastCaptureIdx; uint32_t mLatency; struct timespec mBufferUpdateTime; uint8_t mCaptureBuf[CAPTURE_BUF_SIZE]; + // for measurements + uint8_t mChannelCount; // to avoid recomputing it every time a buffer is processed + uint32_t mMeasurementMode; + uint8_t mMeasurementWindowSizeInBuffers; + uint8_t mMeasurementBufferIdx; + BufferStats mPastMeasurements[MEASUREMENT_WINDOW_MAX_SIZE_IN_BUFFERS]; }; // //--- Local functions // +uint32_t Visualizer_getDeltaTimeMsFromUpdatedTime(VisualizerContext* pContext) { + uint32_t deltaMs = 0; + if (pContext->mBufferUpdateTime.tv_sec != 0) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) { + time_t secs = ts.tv_sec - pContext->mBufferUpdateTime.tv_sec; + long nsec = ts.tv_nsec - pContext->mBufferUpdateTime.tv_nsec; + if (nsec < 0) { + --secs; + nsec += 1000000000; + } + deltaMs = secs * 1000 + nsec / 1000000; + } + } + return deltaMs; +} + void Visualizer_reset(VisualizerContext *pContext) { @@ -165,9 +201,21 @@ int Visualizer_init(VisualizerContext *pContext) pContext->mConfig.outputCfg.bufferProvider.cookie = NULL; pContext->mConfig.outputCfg.mask = EFFECT_CONFIG_ALL; + // visualization initialization pContext->mCaptureSize = VISUALIZER_CAPTURE_SIZE_MAX; pContext->mScalingMode = VISUALIZER_SCALING_MODE_NORMALIZED; + // measurement initialization + pContext->mChannelCount = popcount(pContext->mConfig.inputCfg.channels); + pContext->mMeasurementMode = MEASUREMENT_MODE_NONE; + pContext->mMeasurementWindowSizeInBuffers = MEASUREMENT_WINDOW_MAX_SIZE_IN_BUFFERS; + pContext->mMeasurementBufferIdx = 0; + for (uint32_t i=0 ; i<pContext->mMeasurementWindowSizeInBuffers ; i++) { + pContext->mPastMeasurements[i].mIsValid = false; + pContext->mPastMeasurements[i].mPeakU16 = 0; + pContext->mPastMeasurements[i].mRmsSquared = 0; + } + Visualizer_setConfig(pContext, &pContext->mConfig); return 0; @@ -270,6 +318,30 @@ int Visualizer_process( return -EINVAL; } + // perform measurements if needed + if (pContext->mMeasurementMode & MEASUREMENT_MODE_PEAK_RMS) { + // find the peak and RMS squared for the new buffer + uint32_t inIdx; + int16_t maxSample = 0; + float rmsSqAcc = 0; + for (inIdx = 0 ; inIdx < inBuffer->frameCount * pContext->mChannelCount ; inIdx++) { + if (inBuffer->s16[inIdx] > maxSample) { + maxSample = inBuffer->s16[inIdx]; + } else if (-inBuffer->s16[inIdx] > maxSample) { + maxSample = -inBuffer->s16[inIdx]; + } + rmsSqAcc += (inBuffer->s16[inIdx] * inBuffer->s16[inIdx]); + } + // store the measurement + pContext->mPastMeasurements[pContext->mMeasurementBufferIdx].mPeakU16 = (uint16_t)maxSample; + pContext->mPastMeasurements[pContext->mMeasurementBufferIdx].mRmsSquared = + rmsSqAcc / (inBuffer->frameCount * pContext->mChannelCount); + pContext->mPastMeasurements[pContext->mMeasurementBufferIdx].mIsValid = true; + if (++pContext->mMeasurementBufferIdx >= pContext->mMeasurementWindowSizeInBuffers) { + pContext->mMeasurementBufferIdx = 0; + } + } + // all code below assumes stereo 16 bit PCM output and input int32_t shift; @@ -423,6 +495,12 @@ int Visualizer_command(effect_handle_t self, uint32_t cmdCode, uint32_t cmdSize, p->vsize = sizeof(uint32_t); *replySize += sizeof(uint32_t); break; + case VISUALIZER_PARAM_MEASUREMENT_MODE: + ALOGV("get mMeasurementMode = %d", pContext->mMeasurementMode); + *((uint32_t *)p->data + 1) = pContext->mMeasurementMode; + p->vsize = sizeof(uint32_t); + *replySize += sizeof(uint32_t); + break; default: p->status = -EINVAL; } @@ -452,6 +530,10 @@ int Visualizer_command(effect_handle_t self, uint32_t cmdCode, uint32_t cmdSize, pContext->mLatency = *((uint32_t *)p->data + 1); ALOGV("set mLatency = %d", pContext->mLatency); break; + case VISUALIZER_PARAM_MEASUREMENT_MODE: + pContext->mMeasurementMode = *((uint32_t *)p->data + 1); + ALOGV("set mMeasurementMode = %d", pContext->mMeasurementMode); + break; default: *(int32_t *)pReplyData = -EINVAL; } @@ -470,24 +552,12 @@ int Visualizer_command(effect_handle_t self, uint32_t cmdCode, uint32_t cmdSize, } if (pContext->mState == VISUALIZER_STATE_ACTIVE) { int32_t latencyMs = pContext->mLatency; - uint32_t deltaMs = 0; - if (pContext->mBufferUpdateTime.tv_sec != 0) { - struct timespec ts; - if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) { - time_t secs = ts.tv_sec - pContext->mBufferUpdateTime.tv_sec; - long nsec = ts.tv_nsec - pContext->mBufferUpdateTime.tv_nsec; - if (nsec < 0) { - --secs; - nsec += 1000000000; - } - deltaMs = secs * 1000 + nsec / 1000000; - latencyMs -= deltaMs; - if (latencyMs < 0) { - latencyMs = 0; - } - } + const uint32_t deltaMs = Visualizer_getDeltaTimeMsFromUpdatedTime(pContext); + latencyMs -= deltaMs; + if (latencyMs < 0) { + latencyMs = 0; } - uint32_t deltaSmpl = pContext->mConfig.inputCfg.samplingRate * latencyMs / 1000; + const uint32_t deltaSmpl = pContext->mConfig.inputCfg.samplingRate * latencyMs / 1000; int32_t capturePoint = pContext->mCaptureIdx - pContext->mCaptureSize - deltaSmpl; int32_t captureSize = pContext->mCaptureSize; @@ -499,7 +569,7 @@ int Visualizer_command(effect_handle_t self, uint32_t cmdCode, uint32_t cmdSize, memcpy(pReplyData, pContext->mCaptureBuf + CAPTURE_BUF_SIZE + capturePoint, size); - pReplyData += size; + pReplyData = (char *)pReplyData + size; captureSize -= size; capturePoint = 0; } @@ -525,6 +595,54 @@ int Visualizer_command(effect_handle_t self, uint32_t cmdCode, uint32_t cmdSize, break; + case VISUALIZER_CMD_MEASURE: { + uint16_t peakU16 = 0; + float sumRmsSquared = 0.0f; + uint8_t nbValidMeasurements = 0; + // reset measurements if last measurement was too long ago (which implies stored + // measurements aren't relevant anymore and shouldn't bias the new one) + const int32_t delayMs = Visualizer_getDeltaTimeMsFromUpdatedTime(pContext); + if (delayMs > DISCARD_MEASUREMENTS_TIME_MS) { + ALOGV("Discarding measurements, last measurement is %dms old", delayMs); + for (uint32_t i=0 ; i<pContext->mMeasurementWindowSizeInBuffers ; i++) { + pContext->mPastMeasurements[i].mIsValid = false; + pContext->mPastMeasurements[i].mPeakU16 = 0; + pContext->mPastMeasurements[i].mRmsSquared = 0; + } + pContext->mMeasurementBufferIdx = 0; + } else { + // only use actual measurements, otherwise the first RMS measure happening before + // MEASUREMENT_WINDOW_MAX_SIZE_IN_BUFFERS have been played will always be artificially + // low + for (uint32_t i=0 ; i < pContext->mMeasurementWindowSizeInBuffers ; i++) { + if (pContext->mPastMeasurements[i].mIsValid) { + if (pContext->mPastMeasurements[i].mPeakU16 > peakU16) { + peakU16 = pContext->mPastMeasurements[i].mPeakU16; + } + sumRmsSquared += pContext->mPastMeasurements[i].mRmsSquared; + nbValidMeasurements++; + } + } + } + float rms = nbValidMeasurements == 0 ? 0.0f : sqrtf(sumRmsSquared / nbValidMeasurements); + int32_t* pIntReplyData = (int32_t*)pReplyData; + // convert from I16 sample values to mB and write results + if (rms < 0.000016f) { + pIntReplyData[MEASUREMENT_IDX_RMS] = -9600; //-96dB + } else { + pIntReplyData[MEASUREMENT_IDX_RMS] = (int32_t) (2000 * log10(rms / 32767.0f)); + } + if (peakU16 == 0) { + pIntReplyData[MEASUREMENT_IDX_PEAK] = -9600; //-96dB + } else { + pIntReplyData[MEASUREMENT_IDX_PEAK] = (int32_t) (2000 * log10(peakU16 / 32767.0f)); + } + ALOGV("VISUALIZER_CMD_MEASURE peak=%d (%dmB), rms=%.1f (%dmB)", + peakU16, pIntReplyData[MEASUREMENT_IDX_PEAK], + rms, pIntReplyData[MEASUREMENT_IDX_RMS]); + } + break; + default: ALOGW("Visualizer_command invalid command %d",cmdCode); return -EINVAL; diff --git a/media/libmedia/Android.mk b/media/libmedia/Android.mk index 2c0c3a5..56e7787 100644 --- a/media/libmedia/Android.mk +++ b/media/libmedia/Android.mk @@ -53,7 +53,8 @@ LOCAL_SRC_FILES:= \ Visualizer.cpp \ MemoryLeakTrackUtil.cpp \ SoundPool.cpp \ - SoundPoolThread.cpp + SoundPoolThread.cpp \ + StringArray.cpp LOCAL_SRC_FILES += ../libnbaio/roundup.c @@ -61,6 +62,7 @@ LOCAL_SRC_FILES += ../libnbaio/roundup.c LOCAL_CFLAGS += -DANDROID_SMP=$(if $(findstring true,$(TARGET_CPU_SMP)),1,0) LOCAL_SRC_FILES += SingleStateQueue.cpp LOCAL_CFLAGS += -DSINGLE_STATE_QUEUE_INSTANTIATIONS='"SingleStateQueueInstantiations.cpp"' +# Consider a separate a library for SingleStateQueueInstantiations. LOCAL_SHARED_LIBRARIES := \ libui liblog libcutils libutils libbinder libsonivox libicuuc libexpat \ diff --git a/media/libmedia/AudioRecord.cpp b/media/libmedia/AudioRecord.cpp index 40ff1bf..666fafa 100644 --- a/media/libmedia/AudioRecord.cpp +++ b/media/libmedia/AudioRecord.cpp @@ -19,17 +19,13 @@ #define LOG_TAG "AudioRecord" #include <sys/resource.h> -#include <sys/types.h> - #include <binder/IPCThreadState.h> -#include <cutils/atomic.h> -#include <cutils/compiler.h> #include <media/AudioRecord.h> -#include <media/AudioSystem.h> -#include <system/audio.h> #include <utils/Log.h> - #include <private/media/AudioTrackShared.h> +#include <media/IAudioFlinger.h> + +#define WAIT_PERIOD_MS 10 namespace android { // --------------------------------------------------------------------------- @@ -41,7 +37,9 @@ status_t AudioRecord::getMinFrameCount( audio_format_t format, audio_channel_mask_t channelMask) { - if (frameCount == NULL) return BAD_VALUE; + if (frameCount == NULL) { + return BAD_VALUE; + } // default to 0 in case of error *frameCount = 0; @@ -62,10 +60,9 @@ status_t AudioRecord::getMinFrameCount( // We double the size of input buffer for ping pong use of record buffer. size <<= 1; - if (audio_is_linear_pcm(format)) { - uint32_t channelCount = popcount(channelMask); - size /= channelCount * audio_bytes_per_sample(format); - } + // Assumes audio_is_linear_pcm(format) + uint32_t channelCount = popcount(channelMask); + size /= channelCount * audio_bytes_per_sample(format); *frameCount = size; return NO_ERROR; @@ -75,8 +72,7 @@ status_t AudioRecord::getMinFrameCount( AudioRecord::AudioRecord() : mStatus(NO_INIT), mSessionId(0), - mPreviousPriority(ANDROID_PRIORITY_NORMAL), mPreviousSchedulingGroup(SP_DEFAULT), - mProxy(NULL) + mPreviousPriority(ANDROID_PRIORITY_NORMAL), mPreviousSchedulingGroup(SP_DEFAULT) { } @@ -89,14 +85,16 @@ AudioRecord::AudioRecord( callback_t cbf, void* user, int notificationFrames, - int sessionId) + int sessionId, + transfer_type transferType, + audio_input_flags_t flags) : mStatus(NO_INIT), mSessionId(0), mPreviousPriority(ANDROID_PRIORITY_NORMAL), mPreviousSchedulingGroup(SP_DEFAULT), mProxy(NULL) { - mStatus = set(inputSource, sampleRate, format, channelMask, - frameCount, cbf, user, notificationFrames, false /*threadCanCallJava*/, sessionId); + mStatus = set(inputSource, sampleRate, format, channelMask, frameCount, cbf, user, + notificationFrames, false /*threadCanCallJava*/, sessionId, transferType); } AudioRecord::~AudioRecord() @@ -107,15 +105,18 @@ AudioRecord::~AudioRecord() // Otherwise the callback thread will never exit. stop(); if (mAudioRecordThread != 0) { + mProxy->interrupt(); mAudioRecordThread->requestExit(); // see comment in AudioRecord.h mAudioRecordThread->requestExitAndWait(); mAudioRecordThread.clear(); } - mAudioRecord.clear(); + if (mAudioRecord != 0) { + mAudioRecord->asBinder()->unlinkToDeath(mDeathNotifier, this); + mAudioRecord.clear(); + } IPCThreadState::self()->flushCommands(); AudioSystem::releaseAudioSessionId(mSessionId); } - delete mProxy; } status_t AudioRecord::set( @@ -128,8 +129,33 @@ status_t AudioRecord::set( void* user, int notificationFrames, bool threadCanCallJava, - int sessionId) + int sessionId, + transfer_type transferType, + audio_input_flags_t flags) { + switch (transferType) { + case TRANSFER_DEFAULT: + if (cbf == NULL || threadCanCallJava) { + transferType = TRANSFER_SYNC; + } else { + transferType = TRANSFER_CALLBACK; + } + break; + case TRANSFER_CALLBACK: + if (cbf == NULL) { + ALOGE("Transfer type TRANSFER_CALLBACK but cbf == NULL"); + return BAD_VALUE; + } + break; + case TRANSFER_OBTAIN: + case TRANSFER_SYNC: + break; + default: + ALOGE("Invalid transfer type %d", transferType); + return BAD_VALUE; + } + mTransfer = transferType; + // FIXME "int" here is legacy and will be replaced by size_t later if (frameCountInt < 0) { ALOGE("Invalid frame count %d", frameCountInt); @@ -143,15 +169,18 @@ status_t AudioRecord::set( AutoMutex lock(mLock); if (mAudioRecord != 0) { + ALOGE("Track already in use"); return INVALID_OPERATION; } if (inputSource == AUDIO_SOURCE_DEFAULT) { inputSource = AUDIO_SOURCE_MIC; } + mInputSource = inputSource; if (sampleRate == 0) { - sampleRate = DEFAULT_SAMPLE_RATE; + ALOGE("Invalid sample rate %u", sampleRate); + return BAD_VALUE; } mSampleRate = sampleRate; @@ -159,47 +188,36 @@ status_t AudioRecord::set( if (format == AUDIO_FORMAT_DEFAULT) { format = AUDIO_FORMAT_PCM_16_BIT; } + // validate parameters if (!audio_is_valid_format(format)) { - ALOGE("Invalid format"); + ALOGE("Invalid format %d", format); + return BAD_VALUE; + } + // Temporary restriction: AudioFlinger currently supports 16-bit PCM only + if (format != AUDIO_FORMAT_PCM_16_BIT) { + ALOGE("Format %d is not supported", format); return BAD_VALUE; } mFormat = format; if (!audio_is_input_channel(channelMask)) { + ALOGE("Invalid channel mask %#x", channelMask); return BAD_VALUE; } mChannelMask = channelMask; uint32_t channelCount = popcount(channelMask); mChannelCount = channelCount; - if (audio_is_linear_pcm(format)) { - mFrameSize = channelCount * audio_bytes_per_sample(format); - } else { - mFrameSize = sizeof(uint8_t); - } - - if (sessionId == 0 ) { - mSessionId = AudioSystem::newAudioSessionId(); - } else { - mSessionId = sessionId; - } - ALOGV("set(): mSessionId %d", mSessionId); - - audio_io_handle_t input = AudioSystem::getInput(inputSource, - sampleRate, - format, - channelMask, - mSessionId); - if (input == 0) { - ALOGE("Could not get audio input for record source %d", inputSource); - return BAD_VALUE; - } + // Assumes audio_is_linear_pcm(format), else sizeof(uint8_t) + mFrameSize = channelCount * audio_bytes_per_sample(format); // validate framecount size_t minFrameCount = 0; - status_t status = getMinFrameCount(&minFrameCount, sampleRate, format, channelMask); + status_t status = AudioRecord::getMinFrameCount(&minFrameCount, + sampleRate, format, channelMask); if (status != NO_ERROR) { + ALOGE("getMinFrameCount() failed; status %d", status); return status; } ALOGV("AudioRecord::set() minFrameCount = %d", minFrameCount); @@ -207,16 +225,26 @@ status_t AudioRecord::set( if (frameCount == 0) { frameCount = minFrameCount; } else if (frameCount < minFrameCount) { + ALOGE("frameCount %u < minFrameCount %u", frameCount, minFrameCount); return BAD_VALUE; } + mFrameCount = frameCount; - if (notificationFrames == 0) { - notificationFrames = frameCount/2; + mNotificationFramesReq = notificationFrames; + mNotificationFramesAct = 0; + + if (sessionId == 0 ) { + mSessionId = AudioSystem::newAudioSessionId(); + } else { + mSessionId = sessionId; } + ALOGV("set(): mSessionId %d", mSessionId); + + mFlags = flags; // create the IAudioRecord - status = openRecord_l(sampleRate, format, frameCount, input); - if (status != NO_ERROR) { + status = openRecord_l(0 /*epoch*/); + if (status) { return status; } @@ -232,8 +260,7 @@ status_t AudioRecord::set( mActive = false; mCbf = cbf; - mNotificationFrames = notificationFrames; - mRemainingFrames = notificationFrames; + mRefreshRemaining = true; mUserData = user; // TODO: add audio hardware input latency here mLatency = (1000*mFrameCount) / sampleRate; @@ -241,120 +268,79 @@ status_t AudioRecord::set( mMarkerReached = false; mNewPosition = 0; mUpdatePeriod = 0; - mInputSource = inputSource; - mInput = input; AudioSystem::acquireAudioSessionId(mSessionId); + mSequence = 1; + mObservedSequence = mSequence; + mInOverrun = false; return NO_ERROR; } -status_t AudioRecord::initCheck() const -{ - return mStatus; -} - -// ------------------------------------------------------------------------- - -uint32_t AudioRecord::latency() const -{ - return mLatency; -} - -audio_format_t AudioRecord::format() const -{ - return mFormat; -} - -uint32_t AudioRecord::channelCount() const -{ - return mChannelCount; -} - -size_t AudioRecord::frameCount() const -{ - return mFrameCount; -} - -audio_source_t AudioRecord::inputSource() const -{ - return mInputSource; -} - // ------------------------------------------------------------------------- status_t AudioRecord::start(AudioSystem::sync_event_t event, int triggerSession) { - status_t ret = NO_ERROR; - sp<AudioRecordThread> t = mAudioRecordThread; - ALOGV("start, sync event %d trigger session %d", event, triggerSession); AutoMutex lock(mLock); - // acquire a strong reference on the IAudioRecord and IMemory so that they cannot be destroyed - // while we are accessing the cblk - sp<IAudioRecord> audioRecord = mAudioRecord; - sp<IMemory> iMem = mCblkMemory; - audio_track_cblk_t* cblk = mCblk; + if (mActive) { + return NO_ERROR; + } - if (!mActive) { - mActive = true; + // reset current position as seen by client to 0 + mProxy->setEpoch(mProxy->getEpoch() - mProxy->getPosition()); - cblk->lock.lock(); - if (!(cblk->flags & CBLK_INVALID)) { - cblk->lock.unlock(); - ALOGV("mAudioRecord->start()"); - ret = mAudioRecord->start(event, triggerSession); - cblk->lock.lock(); - if (ret == DEAD_OBJECT) { - android_atomic_or(CBLK_INVALID, &cblk->flags); - } - } - if (cblk->flags & CBLK_INVALID) { - audio_track_cblk_t* temp = cblk; - ret = restoreRecord_l(temp); - cblk = temp; + mNewPosition = mProxy->getPosition() + mUpdatePeriod; + int32_t flags = android_atomic_acquire_load(&mCblk->mFlags); + + status_t status = NO_ERROR; + if (!(flags & CBLK_INVALID)) { + ALOGV("mAudioRecord->start()"); + status = mAudioRecord->start(event, triggerSession); + if (status == DEAD_OBJECT) { + flags |= CBLK_INVALID; } - cblk->lock.unlock(); - if (ret == NO_ERROR) { - mNewPosition = cblk->user + mUpdatePeriod; - cblk->bufferTimeoutMs = (event == AudioSystem::SYNC_EVENT_NONE) ? MAX_RUN_TIMEOUT_MS : - AudioSystem::kSyncRecordStartTimeOutMs; - cblk->waitTimeMs = 0; - if (t != 0) { - t->resume(); - } else { - mPreviousPriority = getpriority(PRIO_PROCESS, 0); - get_sched_policy(0, &mPreviousSchedulingGroup); - androidSetThreadPriority(0, ANDROID_PRIORITY_AUDIO); - } + } + if (flags & CBLK_INVALID) { + status = restoreRecord_l("start"); + } + + if (status != NO_ERROR) { + ALOGE("start() status %d", status); + } else { + mActive = true; + sp<AudioRecordThread> t = mAudioRecordThread; + if (t != 0) { + t->resume(); } else { - mActive = false; + mPreviousPriority = getpriority(PRIO_PROCESS, 0); + get_sched_policy(0, &mPreviousSchedulingGroup); + androidSetThreadPriority(0, ANDROID_PRIORITY_AUDIO); } } - return ret; + return status; } void AudioRecord::stop() { - sp<AudioRecordThread> t = mAudioRecordThread; - - ALOGV("stop"); - AutoMutex lock(mLock); - if (mActive) { - mActive = false; - mCblk->cv.signal(); - mAudioRecord->stop(); - // the record head position will reset to 0, so if a marker is set, we need - // to activate it again - mMarkerReached = false; - if (t != 0) { - t->pause(); - } else { - setpriority(PRIO_PROCESS, 0, mPreviousPriority); - set_sched_policy(0, mPreviousSchedulingGroup); - } + if (!mActive) { + return; + } + + mActive = false; + mProxy->interrupt(); + mAudioRecord->stop(); + // the record head position will reset to 0, so if a marker is set, we need + // to activate it again + mMarkerReached = false; + sp<AudioRecordThread> t = mAudioRecordThread; + if (t != 0) { + t->pause(); + } else { + setpriority(PRIO_PROCESS, 0, mPreviousPriority); + set_sched_policy(0, mPreviousSchedulingGroup); } } @@ -364,14 +350,11 @@ bool AudioRecord::stopped() const return !mActive; } -uint32_t AudioRecord::getSampleRate() const -{ - return mSampleRate; -} - status_t AudioRecord::setMarkerPosition(uint32_t marker) { - if (mCbf == NULL) return INVALID_OPERATION; + if (mCbf == NULL) { + return INVALID_OPERATION; + } AutoMutex lock(mLock); mMarkerPosition = marker; @@ -382,7 +365,9 @@ status_t AudioRecord::setMarkerPosition(uint32_t marker) status_t AudioRecord::getMarkerPosition(uint32_t *marker) const { - if (marker == NULL) return BAD_VALUE; + if (marker == NULL) { + return BAD_VALUE; + } AutoMutex lock(mLock); *marker = mMarkerPosition; @@ -392,13 +377,12 @@ status_t AudioRecord::getMarkerPosition(uint32_t *marker) const status_t AudioRecord::setPositionUpdatePeriod(uint32_t updatePeriod) { - if (mCbf == NULL) return INVALID_OPERATION; - - uint32_t curPosition; - getPosition(&curPosition); + if (mCbf == NULL) { + return INVALID_OPERATION; + } AutoMutex lock(mLock); - mNewPosition = curPosition + updatePeriod; + mNewPosition = mProxy->getPosition() + updatePeriod; mUpdatePeriod = updatePeriod; return NO_ERROR; @@ -406,7 +390,9 @@ status_t AudioRecord::setPositionUpdatePeriod(uint32_t updatePeriod) status_t AudioRecord::getPositionUpdatePeriod(uint32_t *updatePeriod) const { - if (updatePeriod == NULL) return BAD_VALUE; + if (updatePeriod == NULL) { + return BAD_VALUE; + } AutoMutex lock(mLock); *updatePeriod = mUpdatePeriod; @@ -416,10 +402,12 @@ status_t AudioRecord::getPositionUpdatePeriod(uint32_t *updatePeriod) const status_t AudioRecord::getPosition(uint32_t *position) const { - if (position == NULL) return BAD_VALUE; + if (position == NULL) { + return BAD_VALUE; + } AutoMutex lock(mLock); - *position = mCblk->user; + *position = mProxy->getPosition(); return NO_ERROR; } @@ -427,17 +415,13 @@ status_t AudioRecord::getPosition(uint32_t *position) const unsigned int AudioRecord::getInputFramesLost() const { // no need to check mActive, because if inactive this will return 0, which is what we want - return AudioSystem::getInputFramesLost(mInput); + return AudioSystem::getInputFramesLost(getInput()); } // ------------------------------------------------------------------------- // must be called with mLock held -status_t AudioRecord::openRecord_l( - uint32_t sampleRate, - audio_format_t format, - size_t frameCount, - audio_io_handle_t input) +status_t AudioRecord::openRecord_l(size_t epoch) { status_t status; const sp<IAudioFlinger>& audioFlinger = AudioSystem::get_audio_flinger(); @@ -446,23 +430,53 @@ status_t AudioRecord::openRecord_l( return NO_INIT; } + IAudioFlinger::track_flags_t trackFlags = IAudioFlinger::TRACK_DEFAULT; pid_t tid = -1; - // FIXME see similar logic at AudioTrack + + // Client can only express a preference for FAST. Server will perform additional tests. + // The only supported use case for FAST is callback transfer mode. + if (mFlags & AUDIO_INPUT_FLAG_FAST) { + if ((mTransfer != TRANSFER_CALLBACK) || (mAudioRecordThread == 0)) { + ALOGW("AUDIO_INPUT_FLAG_FAST denied by client"); + // once denied, do not request again if IAudioRecord is re-created + mFlags = (audio_input_flags_t) (mFlags & ~AUDIO_INPUT_FLAG_FAST); + } else { + trackFlags |= IAudioFlinger::TRACK_FAST; + tid = mAudioRecordThread->getTid(); + } + } + + mNotificationFramesAct = mNotificationFramesReq; + + if (!(mFlags & AUDIO_INPUT_FLAG_FAST)) { + // Make sure that application is notified with sufficient margin before overrun + if (mNotificationFramesAct == 0 || mNotificationFramesAct > mFrameCount/2) { + mNotificationFramesAct = mFrameCount/2; + } + } + + audio_io_handle_t input = AudioSystem::getInput(mInputSource, mSampleRate, mFormat, + mChannelMask, mSessionId); + if (input == 0) { + ALOGE("Could not get audio input for record source %d", mInputSource); + return BAD_VALUE; + } int originalSessionId = mSessionId; sp<IAudioRecord> record = audioFlinger->openRecord(input, - sampleRate, format, + mSampleRate, mFormat, mChannelMask, - frameCount, - IAudioFlinger::TRACK_DEFAULT, + mFrameCount, + &trackFlags, tid, &mSessionId, &status); ALOGE_IF(originalSessionId != 0 && mSessionId != originalSessionId, "session ID changed from %d to %d", originalSessionId, mSessionId); - if (record == 0) { + if (record == 0 || status != NO_ERROR) { ALOGE("AudioFlinger could not create record track, status: %d", status); + AudioSystem::releaseInput(input); return status; } sp<IMemory> iMem = record->getCblk(); @@ -470,133 +484,163 @@ status_t AudioRecord::openRecord_l( ALOGE("Could not get control block"); return NO_INIT; } - mAudioRecord.clear(); + void *iMemPointer = iMem->pointer(); + if (iMemPointer == NULL) { + ALOGE("Could not get control block pointer"); + return NO_INIT; + } + if (mAudioRecord != 0) { + mAudioRecord->asBinder()->unlinkToDeath(mDeathNotifier, this); + mDeathNotifier.clear(); + } + mInput = input; mAudioRecord = record; - mCblkMemory.clear(); mCblkMemory = iMem; - audio_track_cblk_t* cblk = static_cast<audio_track_cblk_t*>(iMem->pointer()); + audio_track_cblk_t* cblk = static_cast<audio_track_cblk_t*>(iMemPointer); mCblk = cblk; - mBuffers = (char*)cblk + sizeof(audio_track_cblk_t); - cblk->bufferTimeoutMs = MAX_RUN_TIMEOUT_MS; - cblk->waitTimeMs = 0; + // FIXME missing fast track frameCount logic + mAwaitBoost = false; + if (mFlags & AUDIO_INPUT_FLAG_FAST) { + if (trackFlags & IAudioFlinger::TRACK_FAST) { + ALOGV("AUDIO_INPUT_FLAG_FAST successful; frameCount %u", mFrameCount); + mAwaitBoost = true; + // double-buffering is not required for fast tracks, due to tighter scheduling + if (mNotificationFramesAct == 0 || mNotificationFramesAct > mFrameCount) { + mNotificationFramesAct = mFrameCount; + } + } else { + ALOGV("AUDIO_INPUT_FLAG_FAST denied by server; frameCount %u", mFrameCount); + // once denied, do not request again if IAudioRecord is re-created + mFlags = (audio_input_flags_t) (mFlags & ~AUDIO_INPUT_FLAG_FAST); + if (mNotificationFramesAct == 0 || mNotificationFramesAct > mFrameCount/2) { + mNotificationFramesAct = mFrameCount/2; + } + } + } + + // starting address of buffers in shared memory + void *buffers = (char*)cblk + sizeof(audio_track_cblk_t); // update proxy - delete mProxy; - mProxy = new AudioRecordClientProxy(cblk, mBuffers, frameCount, mFrameSize); + mProxy = new AudioRecordClientProxy(cblk, buffers, mFrameCount, mFrameSize); + mProxy->setEpoch(epoch); + mProxy->setMinimum(mNotificationFramesAct); + + mDeathNotifier = new DeathNotifier(this); + mAudioRecord->asBinder()->linkToDeath(mDeathNotifier, this); return NO_ERROR; } status_t AudioRecord::obtainBuffer(Buffer* audioBuffer, int32_t waitCount) { - ALOG_ASSERT(mStatus == NO_ERROR && mProxy != NULL); + if (audioBuffer == NULL) { + return BAD_VALUE; + } + if (mTransfer != TRANSFER_OBTAIN) { + audioBuffer->frameCount = 0; + audioBuffer->size = 0; + audioBuffer->raw = NULL; + return INVALID_OPERATION; + } - AutoMutex lock(mLock); - bool active; - status_t result = NO_ERROR; - audio_track_cblk_t* cblk = mCblk; - uint32_t framesReq = audioBuffer->frameCount; - uint32_t waitTimeMs = (waitCount < 0) ? cblk->bufferTimeoutMs : WAIT_PERIOD_MS; - - audioBuffer->frameCount = 0; - audioBuffer->size = 0; - - size_t framesReady = mProxy->framesReady(); - - if (framesReady == 0) { - cblk->lock.lock(); - goto start_loop_here; - while (framesReady == 0) { - active = mActive; - if (CC_UNLIKELY(!active)) { - cblk->lock.unlock(); - return NO_MORE_BUFFERS; - } - if (CC_UNLIKELY(!waitCount)) { - cblk->lock.unlock(); - return WOULD_BLOCK; - } - if (!(cblk->flags & CBLK_INVALID)) { - mLock.unlock(); - // this condition is in shared memory, so if IAudioRecord and control block - // are replaced due to mediaserver death or IAudioRecord invalidation then - // cv won't be signalled, but fortunately the timeout will limit the wait - result = cblk->cv.waitRelative(cblk->lock, milliseconds(waitTimeMs)); - cblk->lock.unlock(); - mLock.lock(); - if (!mActive) { - return status_t(STOPPED); - } - // IAudioRecord may have been re-created while mLock was unlocked - cblk = mCblk; - cblk->lock.lock(); - } - if (cblk->flags & CBLK_INVALID) { - goto create_new_record; - } - if (CC_UNLIKELY(result != NO_ERROR)) { - cblk->waitTimeMs += waitTimeMs; - if (cblk->waitTimeMs >= cblk->bufferTimeoutMs) { - ALOGW( "obtainBuffer timed out (is the CPU pegged?) " - "user=%08x, server=%08x", cblk->user, cblk->server); - cblk->lock.unlock(); - // callback thread or sync event hasn't changed - result = mAudioRecord->start(AudioSystem::SYNC_EVENT_SAME, 0); - cblk->lock.lock(); - if (result == DEAD_OBJECT) { - android_atomic_or(CBLK_INVALID, &cblk->flags); -create_new_record: - audio_track_cblk_t* temp = cblk; - result = AudioRecord::restoreRecord_l(temp); - cblk = temp; - } - if (result != NO_ERROR) { - ALOGW("obtainBuffer create Track error %d", result); - cblk->lock.unlock(); - return result; + const struct timespec *requested; + if (waitCount == -1) { + requested = &ClientProxy::kForever; + } else if (waitCount == 0) { + requested = &ClientProxy::kNonBlocking; + } else if (waitCount > 0) { + long long ms = WAIT_PERIOD_MS * (long long) waitCount; + struct timespec timeout; + timeout.tv_sec = ms / 1000; + timeout.tv_nsec = (int) (ms % 1000) * 1000000; + requested = &timeout; + } else { + ALOGE("%s invalid waitCount %d", __func__, waitCount); + requested = NULL; + } + return obtainBuffer(audioBuffer, requested); +} + +status_t AudioRecord::obtainBuffer(Buffer* audioBuffer, const struct timespec *requested, + struct timespec *elapsed, size_t *nonContig) +{ + // previous and new IAudioRecord sequence numbers are used to detect track re-creation + uint32_t oldSequence = 0; + uint32_t newSequence; + + Proxy::Buffer buffer; + status_t status = NO_ERROR; + + static const int32_t kMaxTries = 5; + int32_t tryCounter = kMaxTries; + + do { + // obtainBuffer() is called with mutex unlocked, so keep extra references to these fields to + // keep them from going away if another thread re-creates the track during obtainBuffer() + sp<AudioRecordClientProxy> proxy; + sp<IMemory> iMem; + { + // start of lock scope + AutoMutex lock(mLock); + + newSequence = mSequence; + // did previous obtainBuffer() fail due to media server death or voluntary invalidation? + if (status == DEAD_OBJECT) { + // re-create track, unless someone else has already done so + if (newSequence == oldSequence) { + status = restoreRecord_l("obtainBuffer"); + if (status != NO_ERROR) { + break; } - cblk->waitTimeMs = 0; - } - if (--waitCount == 0) { - cblk->lock.unlock(); - return TIMED_OUT; } } - // read the server count again - start_loop_here: - framesReady = mProxy->framesReady(); - } - cblk->lock.unlock(); - } + oldSequence = newSequence; - cblk->waitTimeMs = 0; - // reset time out to running value after obtaining a buffer - cblk->bufferTimeoutMs = MAX_RUN_TIMEOUT_MS; + // Keep the extra references + proxy = mProxy; + iMem = mCblkMemory; - if (framesReq > framesReady) { - framesReq = framesReady; - } + // Non-blocking if track is stopped + if (!mActive) { + requested = &ClientProxy::kNonBlocking; + } - uint32_t u = cblk->user; - uint32_t bufferEnd = cblk->userBase + mFrameCount; + } // end of lock scope - if (framesReq > bufferEnd - u) { - framesReq = bufferEnd - u; - } + buffer.mFrameCount = audioBuffer->frameCount; + // FIXME starts the requested timeout and elapsed over from scratch + status = proxy->obtainBuffer(&buffer, requested, elapsed); + + } while ((status == DEAD_OBJECT) && (tryCounter-- > 0)); - audioBuffer->frameCount = framesReq; - audioBuffer->size = framesReq * mFrameSize; - audioBuffer->raw = mProxy->buffer(u); - active = mActive; - return active ? status_t(NO_ERROR) : status_t(STOPPED); + audioBuffer->frameCount = buffer.mFrameCount; + audioBuffer->size = buffer.mFrameCount * mFrameSize; + audioBuffer->raw = buffer.mRaw; + if (nonContig != NULL) { + *nonContig = buffer.mNonContig; + } + return status; } void AudioRecord::releaseBuffer(Buffer* audioBuffer) { - ALOG_ASSERT(mStatus == NO_ERROR && mProxy != NULL); + // all TRANSFER_* are valid + + size_t stepCount = audioBuffer->size / mFrameSize; + if (stepCount == 0) { + return; + } + + Proxy::Buffer buffer; + buffer.mFrameCount = stepCount; + buffer.mRaw = audioBuffer->raw; AutoMutex lock(mLock); - (void) mProxy->stepUser(audioBuffer->frameCount); + mInOverrun = false; + mProxy->releaseBuffer(&buffer); + + // the server does not automatically disable recorder on overrun, so no need to restart } audio_io_handle_t AudioRecord::getInput() const @@ -605,226 +649,324 @@ audio_io_handle_t AudioRecord::getInput() const return mInput; } -// must be called with mLock held -audio_io_handle_t AudioRecord::getInput_l() -{ - mInput = AudioSystem::getInput(mInputSource, - mSampleRate, - mFormat, - mChannelMask, - mSessionId); - return mInput; -} - -int AudioRecord::getSessionId() const -{ - // no lock needed because session ID doesn't change after first set() - return mSessionId; -} - // ------------------------------------------------------------------------- ssize_t AudioRecord::read(void* buffer, size_t userSize) { - ssize_t read = 0; - Buffer audioBuffer; - int8_t *dst = static_cast<int8_t*>(buffer); + if (mTransfer != TRANSFER_SYNC) { + return INVALID_OPERATION; + } - if (ssize_t(userSize) < 0) { - // sanity-check. user is most-likely passing an error code. - ALOGE("AudioRecord::read(buffer=%p, size=%u (%d)", - buffer, userSize, userSize); + if (ssize_t(userSize) < 0 || (buffer == NULL && userSize != 0)) { + // sanity-check. user is most-likely passing an error code, and it would + // make the return value ambiguous (actualSize vs error). + ALOGE("AudioRecord::read(buffer=%p, size=%u (%d)", buffer, userSize, userSize); return BAD_VALUE; } - mLock.lock(); - // acquire a strong reference on the IAudioRecord and IMemory so that they cannot be destroyed - // while we are accessing the cblk - sp<IAudioRecord> audioRecord = mAudioRecord; - sp<IMemory> iMem = mCblkMemory; - mLock.unlock(); - - do { + ssize_t read = 0; + Buffer audioBuffer; - audioBuffer.frameCount = userSize/frameSize(); + while (userSize >= mFrameSize) { + audioBuffer.frameCount = userSize / mFrameSize; - // By using a wait count corresponding to twice the timeout period in - // obtainBuffer() we give a chance to recover once for a read timeout - // (if media_server crashed for instance) before returning a length of - // 0 bytes read to the client - status_t err = obtainBuffer(&audioBuffer, ((2 * MAX_RUN_TIMEOUT_MS) / WAIT_PERIOD_MS)); + status_t err = obtainBuffer(&audioBuffer, &ClientProxy::kForever); if (err < 0) { - // out of buffers, return #bytes written - if (err == status_t(NO_MORE_BUFFERS)) { + if (read > 0) { break; } - if (err == status_t(TIMED_OUT)) { - // return partial transfer count - return read; - } return ssize_t(err); } size_t bytesRead = audioBuffer.size; - memcpy(dst, audioBuffer.i8, bytesRead); - - dst += bytesRead; + memcpy(buffer, audioBuffer.i8, bytesRead); + buffer = ((char *) buffer) + bytesRead; userSize -= bytesRead; read += bytesRead; releaseBuffer(&audioBuffer); - } while (userSize); + } return read; } // ------------------------------------------------------------------------- -bool AudioRecord::processAudioBuffer(const sp<AudioRecordThread>& thread) +nsecs_t AudioRecord::processAudioBuffer(const sp<AudioRecordThread>& thread) { - Buffer audioBuffer; - uint32_t frames = mRemainingFrames; - size_t readSize; - mLock.lock(); - // acquire a strong reference on the IAudioRecord and IMemory so that they cannot be destroyed - // while we are accessing the cblk - sp<IAudioRecord> audioRecord = mAudioRecord; - sp<IMemory> iMem = mCblkMemory; - audio_track_cblk_t* cblk = mCblk; + if (mAwaitBoost) { + mAwaitBoost = false; + mLock.unlock(); + static const int32_t kMaxTries = 5; + int32_t tryCounter = kMaxTries; + uint32_t pollUs = 10000; + do { + int policy = sched_getscheduler(0); + if (policy == SCHED_FIFO || policy == SCHED_RR) { + break; + } + usleep(pollUs); + pollUs <<= 1; + } while (tryCounter-- > 0); + if (tryCounter < 0) { + ALOGE("did not receive expected priority boost on time"); + } + // Run again immediately + return 0; + } + + // Can only reference mCblk while locked + int32_t flags = android_atomic_and(~CBLK_OVERRUN, &mCblk->mFlags); + + // Check for track invalidation + if (flags & CBLK_INVALID) { + (void) restoreRecord_l("processAudioBuffer"); + mLock.unlock(); + // Run again immediately, but with a new IAudioRecord + return 0; + } + bool active = mActive; - uint32_t markerPosition = mMarkerPosition; - uint32_t newPosition = mNewPosition; - uint32_t user = cblk->user; - // determine whether a marker callback will be needed, while locked - bool needMarker = !mMarkerReached && (mMarkerPosition > 0) && (user >= mMarkerPosition); - if (needMarker) { - mMarkerReached = true; - } - // determine the number of new position callback(s) that will be needed, while locked + + // Manage overrun callback, must be done under lock to avoid race with releaseBuffer() + bool newOverrun = false; + if (flags & CBLK_OVERRUN) { + if (!mInOverrun) { + mInOverrun = true; + newOverrun = true; + } + } + + // Get current position of server + size_t position = mProxy->getPosition(); + + // Manage marker callback + bool markerReached = false; + size_t markerPosition = mMarkerPosition; + // FIXME fails for wraparound, need 64 bits + if (!mMarkerReached && (markerPosition > 0) && (position >= markerPosition)) { + mMarkerReached = markerReached = true; + } + + // Determine the number of new position callback(s) that will be needed, while locked + size_t newPosCount = 0; + size_t newPosition = mNewPosition; uint32_t updatePeriod = mUpdatePeriod; - uint32_t needNewPos = updatePeriod > 0 && user >= newPosition ? - ((user - newPosition) / updatePeriod) + 1 : 0; - mNewPosition = newPosition + updatePeriod * needNewPos; + // FIXME fails for wraparound, need 64 bits + if (updatePeriod > 0 && position >= newPosition) { + newPosCount = ((position - newPosition) / updatePeriod) + 1; + mNewPosition += updatePeriod * newPosCount; + } + + // Cache other fields that will be needed soon + size_t notificationFrames = mNotificationFramesAct; + if (mRefreshRemaining) { + mRefreshRemaining = false; + mRemainingFrames = notificationFrames; + mRetryOnPartialBuffer = false; + } + size_t misalignment = mProxy->getMisalignment(); + int32_t sequence = mSequence; + + // These fields don't need to be cached, because they are assigned only by set(): + // mTransfer, mCbf, mUserData, mSampleRate + mLock.unlock(); - // perform marker callback, while unlocked - if (needMarker) { + // perform callbacks while unlocked + if (newOverrun) { + mCbf(EVENT_OVERRUN, mUserData, NULL); + } + if (markerReached) { mCbf(EVENT_MARKER, mUserData, &markerPosition); } - - // perform new position callback(s), while unlocked - for (; needNewPos > 0; --needNewPos) { - uint32_t temp = newPosition; + while (newPosCount > 0) { + size_t temp = newPosition; mCbf(EVENT_NEW_POS, mUserData, &temp); newPosition += updatePeriod; + newPosCount--; + } + if (mObservedSequence != sequence) { + mObservedSequence = sequence; + mCbf(EVENT_NEW_IAUDIORECORD, mUserData, NULL); } - do { - audioBuffer.frameCount = frames; - // Calling obtainBuffer() with a wait count of 1 - // limits wait time to WAIT_PERIOD_MS. This prevents from being - // stuck here not being able to handle timed events (position, markers). - status_t err = obtainBuffer(&audioBuffer, 1); - if (err < NO_ERROR) { - if (err != TIMED_OUT) { - ALOGE_IF(err != status_t(NO_MORE_BUFFERS), - "Error obtaining an audio buffer, giving up."); - return false; + // if inactive, then don't run me again until re-started + if (!active) { + return NS_INACTIVE; + } + + // Compute the estimated time until the next timed event (position, markers) + uint32_t minFrames = ~0; + if (!markerReached && position < markerPosition) { + minFrames = markerPosition - position; + } + if (updatePeriod > 0 && updatePeriod < minFrames) { + minFrames = updatePeriod; + } + + // If > 0, poll periodically to recover from a stuck server. A good value is 2. + static const uint32_t kPoll = 0; + if (kPoll > 0 && mTransfer == TRANSFER_CALLBACK && kPoll * notificationFrames < minFrames) { + minFrames = kPoll * notificationFrames; + } + + // Convert frame units to time units + nsecs_t ns = NS_WHENEVER; + if (minFrames != (uint32_t) ~0) { + // This "fudge factor" avoids soaking CPU, and compensates for late progress by server + static const nsecs_t kFudgeNs = 10000000LL; // 10 ms + ns = ((minFrames * 1000000000LL) / mSampleRate) + kFudgeNs; + } + + // If not supplying data by EVENT_MORE_DATA, then we're done + if (mTransfer != TRANSFER_CALLBACK) { + return ns; + } + + struct timespec timeout; + const struct timespec *requested = &ClientProxy::kForever; + if (ns != NS_WHENEVER) { + timeout.tv_sec = ns / 1000000000LL; + timeout.tv_nsec = ns % 1000000000LL; + ALOGV("timeout %ld.%03d", timeout.tv_sec, (int) timeout.tv_nsec / 1000000); + requested = &timeout; + } + + while (mRemainingFrames > 0) { + + Buffer audioBuffer; + audioBuffer.frameCount = mRemainingFrames; + size_t nonContig; + status_t err = obtainBuffer(&audioBuffer, requested, NULL, &nonContig); + LOG_ALWAYS_FATAL_IF((err != NO_ERROR) != (audioBuffer.frameCount == 0), + "obtainBuffer() err=%d frameCount=%u", err, audioBuffer.frameCount); + requested = &ClientProxy::kNonBlocking; + size_t avail = audioBuffer.frameCount + nonContig; + ALOGV("obtainBuffer(%u) returned %u = %u + %u", + mRemainingFrames, avail, audioBuffer.frameCount, nonContig); + if (err != NO_ERROR) { + if (err == TIMED_OUT || err == WOULD_BLOCK || err == -EINTR) { + break; + } + ALOGE("Error %d obtaining an audio buffer, giving up.", err); + return NS_NEVER; + } + + if (mRetryOnPartialBuffer) { + mRetryOnPartialBuffer = false; + if (avail < mRemainingFrames) { + int64_t myns = ((mRemainingFrames - avail) * + 1100000000LL) / mSampleRate; + if (ns < 0 || myns < ns) { + ns = myns; + } + return ns; } - break; } - if (err == status_t(STOPPED)) return false; size_t reqSize = audioBuffer.size; mCbf(EVENT_MORE_DATA, mUserData, &audioBuffer); - readSize = audioBuffer.size; + size_t readSize = audioBuffer.size; // Sanity check on returned size - if (ssize_t(readSize) <= 0) { - // The callback is done filling buffers + if (ssize_t(readSize) < 0 || readSize > reqSize) { + ALOGE("EVENT_MORE_DATA requested %u bytes but callback returned %d bytes", + reqSize, (int) readSize); + return NS_NEVER; + } + + if (readSize == 0) { + // The callback is done consuming buffers // Keep this thread going to handle timed events and - // still try to get more data in intervals of WAIT_PERIOD_MS + // still try to provide more data in intervals of WAIT_PERIOD_MS // but don't just loop and block the CPU, so wait - usleep(WAIT_PERIOD_MS*1000); - break; + return WAIT_PERIOD_MS * 1000000LL; } - if (readSize > reqSize) readSize = reqSize; - audioBuffer.size = readSize; - audioBuffer.frameCount = readSize/frameSize(); - frames -= audioBuffer.frameCount; + size_t releasedFrames = readSize / mFrameSize; + audioBuffer.frameCount = releasedFrames; + mRemainingFrames -= releasedFrames; + if (misalignment >= releasedFrames) { + misalignment -= releasedFrames; + } else { + misalignment = 0; + } releaseBuffer(&audioBuffer); - } while (frames); + // FIXME here is where we would repeat EVENT_MORE_DATA again on same advanced buffer + // if callback doesn't like to accept the full chunk + if (readSize < reqSize) { + continue; + } + // There could be enough non-contiguous frames available to satisfy the remaining request + if (mRemainingFrames <= nonContig) { + continue; + } - // Manage overrun callback - if (active && (mProxy->framesAvailable() == 0)) { - // The value of active is stale, but we are almost sure to be active here because - // otherwise we would have exited when obtainBuffer returned STOPPED earlier. - ALOGV("Overrun user: %x, server: %x, flags %04x", cblk->user, cblk->server, cblk->flags); - if (!(android_atomic_or(CBLK_UNDERRUN, &cblk->flags) & CBLK_UNDERRUN)) { - mCbf(EVENT_OVERRUN, mUserData, NULL); +#if 0 + // This heuristic tries to collapse a series of EVENT_MORE_DATA that would total to a + // sum <= notificationFrames. It replaces that series by at most two EVENT_MORE_DATA + // that total to a sum == notificationFrames. + if (0 < misalignment && misalignment <= mRemainingFrames) { + mRemainingFrames = misalignment; + return (mRemainingFrames * 1100000000LL) / mSampleRate; } - } +#endif - if (frames == 0) { - mRemainingFrames = mNotificationFrames; - } else { - mRemainingFrames = frames; } - return true; + mRemainingFrames = notificationFrames; + mRetryOnPartialBuffer = true; + + // A lot has transpired since ns was calculated, so run again immediately and re-calculate + return 0; } -// must be called with mLock and cblk.lock held. Callers must also hold strong references on -// the IAudioRecord and IMemory in case they are recreated here. -// If the IAudioRecord is successfully restored, the cblk pointer is updated -status_t AudioRecord::restoreRecord_l(audio_track_cblk_t*& refCblk) +status_t AudioRecord::restoreRecord_l(const char *from) { + ALOGW("dead IAudioRecord, creating a new one from %s()", from); + ++mSequence; status_t result; - audio_track_cblk_t* cblk = refCblk; - audio_track_cblk_t* newCblk = cblk; - ALOGW("dead IAudioRecord, creating a new one"); - - // signal old cblk condition so that other threads waiting for available buffers stop - // waiting now - cblk->cv.broadcast(); - cblk->lock.unlock(); - // if the new IAudioRecord is created, openRecord_l() will modify the // following member variables: mAudioRecord, mCblkMemory and mCblk. // It will also delete the strong references on previous IAudioRecord and IMemory - result = openRecord_l(mSampleRate, mFormat, mFrameCount, getInput_l()); + size_t position = mProxy->getPosition(); + mNewPosition = position + mUpdatePeriod; + result = openRecord_l(position); if (result == NO_ERROR) { - newCblk = mCblk; - // callback thread or sync event hasn't changed - result = mAudioRecord->start(AudioSystem::SYNC_EVENT_SAME, 0); + if (mActive) { + // callback thread or sync event hasn't changed + // FIXME this fails if we have a new AudioFlinger instance + result = mAudioRecord->start(AudioSystem::SYNC_EVENT_SAME, 0); + } } if (result != NO_ERROR) { + ALOGW("restoreRecord_l() failed status %d", result); mActive = false; } - ALOGV("restoreRecord_l() status %d mActive %d cblk %p, old cblk %p flags %08x old flags %08x", - result, mActive, newCblk, cblk, newCblk->flags, cblk->flags); - - if (result == NO_ERROR) { - // from now on we switch to the newly created cblk - refCblk = newCblk; - } - newCblk->lock.lock(); + return result; +} - ALOGW_IF(result != NO_ERROR, "restoreRecord_l() error %d", result); +// ========================================================================= - return result; +void AudioRecord::DeathNotifier::binderDied(const wp<IBinder>& who) +{ + sp<AudioRecord> audioRecord = mAudioRecord.promote(); + if (audioRecord != 0) { + AutoMutex lock(audioRecord->mLock); + audioRecord->mProxy->binderDied(); + } } // ========================================================================= AudioRecord::AudioRecordThread::AudioRecordThread(AudioRecord& receiver, bool bCanCallJava) - : Thread(bCanCallJava), mReceiver(receiver), mPaused(true) + : Thread(bCanCallJava), mReceiver(receiver), mPaused(true), mPausedInt(false), mPausedNs(0LL) { } @@ -841,18 +983,46 @@ bool AudioRecord::AudioRecordThread::threadLoop() // caller will check for exitPending() return true; } + if (mPausedInt) { + if (mPausedNs > 0) { + (void) mMyCond.waitRelative(mMyLock, mPausedNs); + } else { + mMyCond.wait(mMyLock); + } + mPausedInt = false; + return true; + } } - if (!mReceiver.processAudioBuffer(this)) { - pause(); + nsecs_t ns = mReceiver.processAudioBuffer(this); + switch (ns) { + case 0: + return true; + case NS_INACTIVE: + pauseInternal(); + return true; + case NS_NEVER: + return false; + case NS_WHENEVER: + // FIXME increase poll interval, or make event-driven + ns = 1000000000LL; + // fall through + default: + LOG_ALWAYS_FATAL_IF(ns < 0, "processAudioBuffer() returned %lld", ns); + pauseInternal(ns); + return true; } - return true; } void AudioRecord::AudioRecordThread::requestExit() { // must be in this order to avoid a race condition Thread::requestExit(); - resume(); + AutoMutex _l(mMyLock); + if (mPaused || mPausedInt) { + mPaused = false; + mPausedInt = false; + mMyCond.signal(); + } } void AudioRecord::AudioRecordThread::pause() @@ -864,12 +1034,20 @@ void AudioRecord::AudioRecordThread::pause() void AudioRecord::AudioRecordThread::resume() { AutoMutex _l(mMyLock); - if (mPaused) { + if (mPaused || mPausedInt) { mPaused = false; + mPausedInt = false; mMyCond.signal(); } } +void AudioRecord::AudioRecordThread::pauseInternal(nsecs_t ns) +{ + AutoMutex _l(mMyLock); + mPausedInt = true; + mPausedNs = ns; +} + // ------------------------------------------------------------------------- }; // namespace android diff --git a/media/libmedia/AudioSystem.cpp b/media/libmedia/AudioSystem.cpp index 693df60..8033c2c 100644 --- a/media/libmedia/AudioSystem.cpp +++ b/media/libmedia/AudioSystem.cpp @@ -20,6 +20,7 @@ #include <utils/Log.h> #include <binder/IServiceManager.h> #include <media/AudioSystem.h> +#include <media/IAudioFlinger.h> #include <media/IAudioPolicyService.h> #include <math.h> @@ -75,6 +76,14 @@ const sp<IAudioFlinger>& AudioSystem::get_audio_flinger() return gAudioFlinger; } +/* static */ status_t AudioSystem::checkAudioFlinger() +{ + if (defaultServiceManager()->checkService(String16("media.audio_flinger")) != 0) { + return NO_ERROR; + } + return DEAD_OBJECT; +} + status_t AudioSystem::muteMicrophone(bool state) { const sp<IAudioFlinger>& af = AudioSystem::get_audio_flinger(); if (af == 0) return PERMISSION_DENIED; @@ -361,8 +370,8 @@ status_t AudioSystem::setVoiceVolume(float value) return af->setVoiceVolume(value); } -status_t AudioSystem::getRenderPosition(size_t *halFrames, size_t *dspFrames, - audio_stream_type_t stream) +status_t AudioSystem::getRenderPosition(audio_io_handle_t output, size_t *halFrames, + size_t *dspFrames, audio_stream_type_t stream) { const sp<IAudioFlinger>& af = AudioSystem::get_audio_flinger(); if (af == 0) return PERMISSION_DENIED; @@ -371,7 +380,11 @@ status_t AudioSystem::getRenderPosition(size_t *halFrames, size_t *dspFrames, stream = AUDIO_STREAM_MUSIC; } - return af->getRenderPosition(halFrames, dspFrames, getOutput(stream)); + if (output == 0) { + output = getOutput(stream); + } + + return af->getRenderPosition(halFrames, dspFrames, output); } size_t AudioSystem::getInputFramesLost(audio_io_handle_t ioHandle) { @@ -442,14 +455,14 @@ void AudioSystem::AudioFlingerClient::ioConfigChanged(int event, audio_io_handle OutputDescriptor *outputDesc = new OutputDescriptor(*desc); gOutputs.add(ioHandle, outputDesc); - ALOGV("ioConfigChanged() new output samplingRate %u, format %d channels %#x frameCount %u " + ALOGV("ioConfigChanged() new output samplingRate %u, format %d channel mask %#x frameCount %u " "latency %d", - outputDesc->samplingRate, outputDesc->format, outputDesc->channels, + outputDesc->samplingRate, outputDesc->format, outputDesc->channelMask, outputDesc->frameCount, outputDesc->latency); } break; case OUTPUT_CLOSED: { if (gOutputs.indexOfKey(ioHandle) < 0) { - ALOGW("ioConfigChanged() closing unknow output! %d", ioHandle); + ALOGW("ioConfigChanged() closing unknown output! %d", ioHandle); break; } ALOGV("ioConfigChanged() output %d closed", ioHandle); @@ -460,16 +473,16 @@ void AudioSystem::AudioFlingerClient::ioConfigChanged(int event, audio_io_handle case OUTPUT_CONFIG_CHANGED: { int index = gOutputs.indexOfKey(ioHandle); if (index < 0) { - ALOGW("ioConfigChanged() modifying unknow output! %d", ioHandle); + ALOGW("ioConfigChanged() modifying unknown output! %d", ioHandle); break; } if (param2 == NULL) break; desc = (const OutputDescriptor *)param2; - ALOGV("ioConfigChanged() new config for output %d samplingRate %u, format %d channels %#x " + ALOGV("ioConfigChanged() new config for output %d samplingRate %u, format %d channel mask %#x " "frameCount %d latency %d", ioHandle, desc->samplingRate, desc->format, - desc->channels, desc->frameCount, desc->latency); + desc->channelMask, desc->frameCount, desc->latency); OutputDescriptor *outputDesc = gOutputs.valueAt(index); delete outputDesc; outputDesc = new OutputDescriptor(*desc); @@ -532,6 +545,8 @@ const sp<IAudioPolicyService>& AudioSystem::get_audio_policy_service() return gAudioPolicyService; } +// --------------------------------------------------------------------------- + status_t AudioSystem::setDeviceConnectionState(audio_devices_t device, audio_policy_dev_state_t state, const char *device_address) @@ -585,11 +600,12 @@ audio_io_handle_t AudioSystem::getOutput(audio_stream_type_t stream, uint32_t samplingRate, audio_format_t format, audio_channel_mask_t channelMask, - audio_output_flags_t flags) + audio_output_flags_t flags, + const audio_offload_info_t *offloadInfo) { const sp<IAudioPolicyService>& aps = AudioSystem::get_audio_policy_service(); if (aps == 0) return 0; - return aps->getOutput(stream, samplingRate, format, channelMask, flags); + return aps->getOutput(stream, samplingRate, format, channelMask, flags, offloadInfo); } status_t AudioSystem::startOutput(audio_io_handle_t output, @@ -764,6 +780,13 @@ size_t AudioSystem::getPrimaryOutputFrameCount() return af->getPrimaryOutputFrameCount(); } +status_t AudioSystem::setLowRamDevice(bool isLowRamDevice) +{ + const sp<IAudioFlinger>& af = AudioSystem::get_audio_flinger(); + if (af == 0) return PERMISSION_DENIED; + return af->setLowRamDevice(isLowRamDevice); +} + void AudioSystem::clearAudioConfigCache() { Mutex::Autolock _l(gLock); @@ -771,6 +794,14 @@ void AudioSystem::clearAudioConfigCache() gOutputs.clear(); } +bool AudioSystem::isOffloadSupported(const audio_offload_info_t& info) +{ + ALOGV("isOffloadSupported()"); + const sp<IAudioPolicyService>& aps = AudioSystem::get_audio_policy_service(); + if (aps == 0) return false; + return aps->isOffloadSupported(info); +} + // --------------------------------------------------------------------------- void AudioSystem::AudioPolicyServiceClient::binderDied(const wp<IBinder>& who) { diff --git a/media/libmedia/AudioTrack.cpp b/media/libmedia/AudioTrack.cpp index 7eeb4f8..fe5cd9e 100644 --- a/media/libmedia/AudioTrack.cpp +++ b/media/libmedia/AudioTrack.cpp @@ -19,31 +19,17 @@ //#define LOG_NDEBUG 0 #define LOG_TAG "AudioTrack" -#include <stdint.h> -#include <sys/types.h> -#include <limits.h> - -#include <sched.h> #include <sys/resource.h> - -#include <private/media/AudioTrackShared.h> - -#include <media/AudioSystem.h> +#include <audio_utils/primitives.h> +#include <binder/IPCThreadState.h> #include <media/AudioTrack.h> - #include <utils/Log.h> -#include <binder/Parcel.h> -#include <binder/IPCThreadState.h> -#include <utils/Timers.h> -#include <utils/Atomic.h> - -#include <cutils/bitops.h> -#include <cutils/compiler.h> +#include <private/media/AudioTrackShared.h> +#include <media/IAudioFlinger.h> -#include <system/audio.h> -#include <system/audio_policy.h> +#define WAIT_PERIOD_MS 10 +#define WAIT_STREAM_END_TIMEOUT_SEC 120 -#include <audio_utils/primitives.h> namespace android { // --------------------------------------------------------------------------- @@ -82,7 +68,9 @@ status_t AudioTrack::getMinFrameCount( // Ensure that buffer depth covers at least audio hardware latency uint32_t minBufCount = afLatency / ((1000 * afFrameCount) / afSampleRate); - if (minBufCount < 2) minBufCount = 2; + if (minBufCount < 2) { + minBufCount = 2; + } *frameCount = (sampleRate == 0) ? afFrameCount * minBufCount : afFrameCount * minBufCount * sampleRate / afSampleRate; @@ -97,8 +85,7 @@ AudioTrack::AudioTrack() : mStatus(NO_INIT), mIsTimed(false), mPreviousPriority(ANDROID_PRIORITY_NORMAL), - mPreviousSchedulingGroup(SP_DEFAULT), - mProxy(NULL) + mPreviousSchedulingGroup(SP_DEFAULT) { } @@ -112,16 +99,17 @@ AudioTrack::AudioTrack( callback_t cbf, void* user, int notificationFrames, - int sessionId) + int sessionId, + transfer_type transferType, + const audio_offload_info_t *offloadInfo) : mStatus(NO_INIT), mIsTimed(false), mPreviousPriority(ANDROID_PRIORITY_NORMAL), - mPreviousSchedulingGroup(SP_DEFAULT), - mProxy(NULL) + mPreviousSchedulingGroup(SP_DEFAULT) { mStatus = set(streamType, sampleRate, format, channelMask, frameCount, flags, cbf, user, notificationFrames, - 0 /*sharedBuffer*/, false /*threadCanCallJava*/, sessionId); + 0 /*sharedBuffer*/, false /*threadCanCallJava*/, sessionId, transferType, offloadInfo); } AudioTrack::AudioTrack( @@ -134,42 +122,37 @@ AudioTrack::AudioTrack( callback_t cbf, void* user, int notificationFrames, - int sessionId) + int sessionId, + transfer_type transferType, + const audio_offload_info_t *offloadInfo) : mStatus(NO_INIT), mIsTimed(false), mPreviousPriority(ANDROID_PRIORITY_NORMAL), - mPreviousSchedulingGroup(SP_DEFAULT), - mProxy(NULL) + mPreviousSchedulingGroup(SP_DEFAULT) { - if (sharedBuffer == 0) { - ALOGE("sharedBuffer must be non-0"); - mStatus = BAD_VALUE; - return; - } mStatus = set(streamType, sampleRate, format, channelMask, 0 /*frameCount*/, flags, cbf, user, notificationFrames, - sharedBuffer, false /*threadCanCallJava*/, sessionId); + sharedBuffer, false /*threadCanCallJava*/, sessionId, transferType, offloadInfo); } AudioTrack::~AudioTrack() { - ALOGV_IF(mSharedBuffer != 0, "Destructor sharedBuffer: %p", mSharedBuffer->pointer()); - if (mStatus == NO_ERROR) { // Make sure that callback function exits in the case where // it is looping on buffer full condition in obtainBuffer(). // Otherwise the callback thread will never exit. stop(); if (mAudioTrackThread != 0) { + mProxy->interrupt(); mAudioTrackThread->requestExit(); // see comment in AudioTrack.h mAudioTrackThread->requestExitAndWait(); mAudioTrackThread.clear(); } + mAudioTrack->asBinder()->unlinkToDeath(mDeathNotifier, this); mAudioTrack.clear(); IPCThreadState::self()->flushCommands(); AudioSystem::releaseAudioSessionId(mSessionId); } - delete mProxy; } status_t AudioTrack::set( @@ -184,8 +167,45 @@ status_t AudioTrack::set( int notificationFrames, const sp<IMemory>& sharedBuffer, bool threadCanCallJava, - int sessionId) + int sessionId, + transfer_type transferType, + const audio_offload_info_t *offloadInfo) { + switch (transferType) { + case TRANSFER_DEFAULT: + if (sharedBuffer != 0) { + transferType = TRANSFER_SHARED; + } else if (cbf == NULL || threadCanCallJava) { + transferType = TRANSFER_SYNC; + } else { + transferType = TRANSFER_CALLBACK; + } + break; + case TRANSFER_CALLBACK: + if (cbf == NULL || sharedBuffer != 0) { + ALOGE("Transfer type TRANSFER_CALLBACK but cbf == NULL || sharedBuffer != 0"); + return BAD_VALUE; + } + break; + case TRANSFER_OBTAIN: + case TRANSFER_SYNC: + if (sharedBuffer != 0) { + ALOGE("Transfer type TRANSFER_OBTAIN but sharedBuffer != 0"); + return BAD_VALUE; + } + break; + case TRANSFER_SHARED: + if (sharedBuffer == 0) { + ALOGE("Transfer type TRANSFER_SHARED but sharedBuffer == 0"); + return BAD_VALUE; + } + break; + default: + ALOGE("Invalid transfer type %d", transferType); + return BAD_VALUE; + } + mTransfer = transferType; + // FIXME "int" here is legacy and will be replaced by size_t later if (frameCountInt < 0) { ALOGE("Invalid frame count %d", frameCountInt); @@ -199,11 +219,15 @@ status_t AudioTrack::set( ALOGV("set() streamType %d frameCount %u flags %04x", streamType, frameCount, flags); AutoMutex lock(mLock); + + // invariant that mAudioTrack != 0 is true only after set() returns successfully if (mAudioTrack != 0) { ALOGE("Track already in use"); return INVALID_OPERATION; } + mOutput = 0; + // handle default values first. if (streamType == AUDIO_STREAM_DEFAULT) { streamType = AUDIO_STREAM_MUSIC; @@ -228,7 +252,7 @@ status_t AudioTrack::set( // validate parameters if (!audio_is_valid_format(format)) { - ALOGE("Invalid format"); + ALOGE("Invalid format %d", format); return BAD_VALUE; } @@ -239,7 +263,12 @@ status_t AudioTrack::set( } // force direct flag if format is not linear PCM - if (!audio_is_linear_pcm(format)) { + // or offload was requested + if ((flags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) + || !audio_is_linear_pcm(format)) { + ALOGV( (flags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) + ? "Offload request, forcing to Direct Output" + : "Not linear PCM, forcing to Direct Output"); flags = (audio_output_flags_t) // FIXME why can't we allow direct AND fast? ((flags | AUDIO_OUTPUT_FLAG_DIRECT) & ~AUDIO_OUTPUT_FLAG_FAST); @@ -268,7 +297,8 @@ status_t AudioTrack::set( audio_io_handle_t output = AudioSystem::getOutput( streamType, sampleRate, format, channelMask, - flags); + flags, + offloadInfo); if (output == 0) { ALOGE("Could not get audio output for stream type %d", streamType); @@ -281,6 +311,7 @@ status_t AudioTrack::set( mFrameCount = frameCount; mReqFrameCount = frameCount; mNotificationFramesReq = notificationFrames; + mNotificationFramesAct = 0; mSessionId = sessionId; mAuxEffectId = 0; mFlags = flags; @@ -298,178 +329,201 @@ status_t AudioTrack::set( frameCount, flags, sharedBuffer, - output); + output, + 0 /*epoch*/); if (status != NO_ERROR) { if (mAudioTrackThread != 0) { - mAudioTrackThread->requestExit(); + mAudioTrackThread->requestExit(); // see comment in AudioTrack.h + mAudioTrackThread->requestExitAndWait(); mAudioTrackThread.clear(); } + //Use of direct and offloaded output streams is ref counted by audio policy manager. + // As getOutput was called above and resulted in an output stream to be opened, + // we need to release it. + AudioSystem::releaseOutput(output); return status; } mStatus = NO_ERROR; - mStreamType = streamType; mFormat = format; - mSharedBuffer = sharedBuffer; - mActive = false; + mState = STATE_STOPPED; mUserData = user; - mLoopCount = 0; + mLoopPeriod = 0; mMarkerPosition = 0; mMarkerReached = false; mNewPosition = 0; mUpdatePeriod = 0; - mFlushed = false; AudioSystem::acquireAudioSessionId(mSessionId); + mSequence = 1; + mObservedSequence = mSequence; + mInUnderrun = false; + mOutput = output; + return NO_ERROR; } // ------------------------------------------------------------------------- -void AudioTrack::start() +status_t AudioTrack::start() { - sp<AudioTrackThread> t = mAudioTrackThread; + AutoMutex lock(mLock); - ALOGV("start %p", this); + if (mState == STATE_ACTIVE) { + return INVALID_OPERATION; + } - AutoMutex lock(mLock); - // acquire a strong reference on the IMemory and IAudioTrack so that they cannot be destroyed - // while we are accessing the cblk - sp<IAudioTrack> audioTrack = mAudioTrack; - sp<IMemory> iMem = mCblkMemory; - audio_track_cblk_t* cblk = mCblk; + mInUnderrun = true; - if (!mActive) { - mFlushed = false; - mActive = true; - mNewPosition = cblk->server + mUpdatePeriod; - cblk->lock.lock(); - cblk->bufferTimeoutMs = MAX_STARTUP_TIMEOUT_MS; - cblk->waitTimeMs = 0; - android_atomic_and(~CBLK_DISABLED, &cblk->flags); - if (t != 0) { - t->resume(); + State previousState = mState; + if (previousState == STATE_PAUSED_STOPPING) { + mState = STATE_STOPPING; + } else { + mState = STATE_ACTIVE; + } + if (previousState == STATE_STOPPED || previousState == STATE_FLUSHED) { + // reset current position as seen by client to 0 + mProxy->setEpoch(mProxy->getEpoch() - mProxy->getPosition()); + // force refresh of remaining frames by processAudioBuffer() as last + // write before stop could be partial. + mRefreshRemaining = true; + } + mNewPosition = mProxy->getPosition() + mUpdatePeriod; + int32_t flags = android_atomic_and(~CBLK_DISABLED, &mCblk->mFlags); + + sp<AudioTrackThread> t = mAudioTrackThread; + if (t != 0) { + if (previousState == STATE_STOPPING) { + mProxy->interrupt(); } else { - mPreviousPriority = getpriority(PRIO_PROCESS, 0); - get_sched_policy(0, &mPreviousSchedulingGroup); - androidSetThreadPriority(0, ANDROID_PRIORITY_AUDIO); + t->resume(); } + } else { + mPreviousPriority = getpriority(PRIO_PROCESS, 0); + get_sched_policy(0, &mPreviousSchedulingGroup); + androidSetThreadPriority(0, ANDROID_PRIORITY_AUDIO); + } - ALOGV("start %p before lock cblk %p", this, cblk); - status_t status = NO_ERROR; - if (!(cblk->flags & CBLK_INVALID)) { - cblk->lock.unlock(); - ALOGV("mAudioTrack->start()"); - status = mAudioTrack->start(); - cblk->lock.lock(); - if (status == DEAD_OBJECT) { - android_atomic_or(CBLK_INVALID, &cblk->flags); - } - } - if (cblk->flags & CBLK_INVALID) { - audio_track_cblk_t* temp = cblk; - status = restoreTrack_l(temp, true /*fromStart*/); - cblk = temp; + status_t status = NO_ERROR; + if (!(flags & CBLK_INVALID)) { + status = mAudioTrack->start(); + if (status == DEAD_OBJECT) { + flags |= CBLK_INVALID; } - cblk->lock.unlock(); - if (status != NO_ERROR) { - ALOGV("start() failed"); - mActive = false; - if (t != 0) { + } + if (flags & CBLK_INVALID) { + status = restoreTrack_l("start"); + } + + if (status != NO_ERROR) { + ALOGE("start() status %d", status); + mState = previousState; + if (t != 0) { + if (previousState != STATE_STOPPING) { t->pause(); - } else { - setpriority(PRIO_PROCESS, 0, mPreviousPriority); - set_sched_policy(0, mPreviousSchedulingGroup); } + } else { + setpriority(PRIO_PROCESS, 0, mPreviousPriority); + set_sched_policy(0, mPreviousSchedulingGroup); } } + return status; } void AudioTrack::stop() { - sp<AudioTrackThread> t = mAudioTrackThread; + AutoMutex lock(mLock); + // FIXME pause then stop should not be a nop + if (mState != STATE_ACTIVE) { + return; + } - ALOGV("stop %p", this); + if (isOffloaded()) { + mState = STATE_STOPPING; + } else { + mState = STATE_STOPPED; + } - AutoMutex lock(mLock); - if (mActive) { - mActive = false; - mCblk->cv.signal(); - mAudioTrack->stop(); - // Cancel loops (If we are in the middle of a loop, playback - // would not stop until loopCount reaches 0). - setLoop_l(0, 0, 0); - // the playback head position will reset to 0, so if a marker is set, we need - // to activate it again - mMarkerReached = false; - // Force flush if a shared buffer is used otherwise audioflinger - // will not stop before end of buffer is reached. - // It may be needed to make sure that we stop playback, likely in case looping is on. - if (mSharedBuffer != 0) { - flush_l(); - } - if (t != 0) { + mProxy->interrupt(); + mAudioTrack->stop(); + // the playback head position will reset to 0, so if a marker is set, we need + // to activate it again + mMarkerReached = false; +#if 0 + // Force flush if a shared buffer is used otherwise audioflinger + // will not stop before end of buffer is reached. + // It may be needed to make sure that we stop playback, likely in case looping is on. + if (mSharedBuffer != 0) { + flush_l(); + } +#endif + + sp<AudioTrackThread> t = mAudioTrackThread; + if (t != 0) { + if (!isOffloaded()) { t->pause(); - } else { - setpriority(PRIO_PROCESS, 0, mPreviousPriority); - set_sched_policy(0, mPreviousSchedulingGroup); } + } else { + setpriority(PRIO_PROCESS, 0, mPreviousPriority); + set_sched_policy(0, mPreviousSchedulingGroup); } - } bool AudioTrack::stopped() const { AutoMutex lock(mLock); - return stopped_l(); + return mState != STATE_ACTIVE; } void AudioTrack::flush() { + if (mSharedBuffer != 0) { + return; + } AutoMutex lock(mLock); - if (!mActive && mSharedBuffer == 0) { - flush_l(); + if (mState == STATE_ACTIVE || mState == STATE_FLUSHED) { + return; } + flush_l(); } void AudioTrack::flush_l() { - ALOGV("flush"); - ALOG_ASSERT(!mActive); + ALOG_ASSERT(mState != STATE_ACTIVE); // clear playback marker and periodic update counter mMarkerPosition = 0; mMarkerReached = false; mUpdatePeriod = 0; + mRefreshRemaining = true; - mFlushed = true; + mState = STATE_FLUSHED; + if (isOffloaded()) { + mProxy->interrupt(); + } + mProxy->flush(); mAudioTrack->flush(); - // Release AudioTrack callback thread in case it was waiting for new buffers - // in AudioTrack::obtainBuffer() - mCblk->cv.signal(); } void AudioTrack::pause() { - ALOGV("pause"); AutoMutex lock(mLock); - if (mActive) { - mActive = false; - mCblk->cv.signal(); - mAudioTrack->pause(); + if (mState == STATE_ACTIVE) { + mState = STATE_PAUSED; + } else if (mState == STATE_STOPPING) { + mState = STATE_PAUSED_STOPPING; + } else { + return; } + mProxy->interrupt(); + mAudioTrack->pause(); } status_t AudioTrack::setVolume(float left, float right) { - if (mStatus != NO_ERROR) { - return mStatus; - } - ALOG_ASSERT(mProxy != NULL); - if (left < 0.0f || left > 1.0f || right < 0.0f || right > 1.0f) { return BAD_VALUE; } @@ -480,6 +534,9 @@ status_t AudioTrack::setVolume(float left, float right) mProxy->setVolumeLR((uint32_t(uint16_t(right * 0x1000)) << 16) | uint16_t(left * 0x1000)); + if (isOffloaded()) { + mAudioTrack->signal(); + } return NO_ERROR; } @@ -490,18 +547,11 @@ status_t AudioTrack::setVolume(float volume) status_t AudioTrack::setAuxEffectSendLevel(float level) { - ALOGV("setAuxEffectSendLevel(%f)", level); - - if (mStatus != NO_ERROR) { - return mStatus; - } - ALOG_ASSERT(mProxy != NULL); - if (level < 0.0f || level > 1.0f) { return BAD_VALUE; } - AutoMutex lock(mLock); + AutoMutex lock(mLock); mSendLevel = level; mProxy->setSendLevel(level); @@ -511,18 +561,17 @@ status_t AudioTrack::setAuxEffectSendLevel(float level) void AudioTrack::getAuxEffectSendLevel(float* level) const { if (level != NULL) { - *level = mSendLevel; + *level = mSendLevel; } } status_t AudioTrack::setSampleRate(uint32_t rate) { - uint32_t afSamplingRate; - - if (mIsTimed) { + if (mIsTimed || isOffloaded()) { return INVALID_OPERATION; } + uint32_t afSamplingRate; if (AudioSystem::getOutputSamplingRate(&afSamplingRate, mStreamType) != NO_ERROR) { return NO_INIT; } @@ -550,58 +599,45 @@ uint32_t AudioTrack::getSampleRate() const status_t AudioTrack::setLoop(uint32_t loopStart, uint32_t loopEnd, int loopCount) { - AutoMutex lock(mLock); - return setLoop_l(loopStart, loopEnd, loopCount); -} - -// must be called with mLock held -status_t AudioTrack::setLoop_l(uint32_t loopStart, uint32_t loopEnd, int loopCount) -{ - if (mSharedBuffer == 0 || mIsTimed) { + if (mSharedBuffer == 0 || mIsTimed || isOffloaded()) { return INVALID_OPERATION; } - audio_track_cblk_t* cblk = mCblk; - - Mutex::Autolock _l(cblk->lock); - if (loopCount == 0) { - cblk->loopStart = UINT_MAX; - cblk->loopEnd = UINT_MAX; - cblk->loopCount = 0; - mLoopCount = 0; - return NO_ERROR; - } - - if (loopStart >= loopEnd || - loopEnd - loopStart > mFrameCount || - cblk->server > loopStart) { - ALOGE("setLoop invalid value: loopStart %d, loopEnd %d, loopCount %d, framecount %d, " - "user %d", loopStart, loopEnd, loopCount, mFrameCount, cblk->user); + ; + } else if (loopCount >= -1 && loopStart < loopEnd && loopEnd <= mFrameCount && + loopEnd - loopStart >= MIN_LOOP) { + ; + } else { return BAD_VALUE; } - if ((mSharedBuffer != 0) && (loopEnd > mFrameCount)) { - ALOGE("setLoop invalid value: loop markers beyond data: loopStart %d, loopEnd %d, " - "framecount %d", - loopStart, loopEnd, mFrameCount); - return BAD_VALUE; + AutoMutex lock(mLock); + // See setPosition() regarding setting parameters such as loop points or position while active + if (mState == STATE_ACTIVE) { + return INVALID_OPERATION; } - - cblk->loopStart = loopStart; - cblk->loopEnd = loopEnd; - cblk->loopCount = loopCount; - mLoopCount = loopCount; - + setLoop_l(loopStart, loopEnd, loopCount); return NO_ERROR; } +void AudioTrack::setLoop_l(uint32_t loopStart, uint32_t loopEnd, int loopCount) +{ + // FIXME If setting a loop also sets position to start of loop, then + // this is correct. Otherwise it should be removed. + mNewPosition = mProxy->getPosition() + mUpdatePeriod; + mLoopPeriod = loopCount != 0 ? loopEnd - loopStart : 0; + mStaticProxy->setLoop(loopStart, loopEnd, loopCount); +} + status_t AudioTrack::setMarkerPosition(uint32_t marker) { - if (mCbf == NULL) { + // The only purpose of setting marker position is to get a callback + if (mCbf == NULL || isOffloaded()) { return INVALID_OPERATION; } + AutoMutex lock(mLock); mMarkerPosition = marker; mMarkerReached = false; @@ -610,10 +646,14 @@ status_t AudioTrack::setMarkerPosition(uint32_t marker) status_t AudioTrack::getMarkerPosition(uint32_t *marker) const { + if (isOffloaded()) { + return INVALID_OPERATION; + } if (marker == NULL) { return BAD_VALUE; } + AutoMutex lock(mLock); *marker = mMarkerPosition; return NO_ERROR; @@ -621,24 +661,27 @@ status_t AudioTrack::getMarkerPosition(uint32_t *marker) const status_t AudioTrack::setPositionUpdatePeriod(uint32_t updatePeriod) { - if (mCbf == NULL) { + // The only purpose of setting position update period is to get a callback + if (mCbf == NULL || isOffloaded()) { return INVALID_OPERATION; } - uint32_t curPosition; - getPosition(&curPosition); - mNewPosition = curPosition + updatePeriod; + AutoMutex lock(mLock); + mNewPosition = mProxy->getPosition() + updatePeriod; mUpdatePeriod = updatePeriod; - return NO_ERROR; } status_t AudioTrack::getPositionUpdatePeriod(uint32_t *updatePeriod) const { + if (isOffloaded()) { + return INVALID_OPERATION; + } if (updatePeriod == NULL) { return BAD_VALUE; } + AutoMutex lock(mLock); *updatePeriod = mUpdatePeriod; return NO_ERROR; @@ -646,80 +689,108 @@ status_t AudioTrack::getPositionUpdatePeriod(uint32_t *updatePeriod) const status_t AudioTrack::setPosition(uint32_t position) { - if (mSharedBuffer == 0 || mIsTimed) { + if (mSharedBuffer == 0 || mIsTimed || isOffloaded()) { return INVALID_OPERATION; } + if (position > mFrameCount) { + return BAD_VALUE; + } AutoMutex lock(mLock); - - if (!stopped_l()) { + // Currently we require that the player is inactive before setting parameters such as position + // or loop points. Otherwise, there could be a race condition: the application could read the + // current position, compute a new position or loop parameters, and then set that position or + // loop parameters but it would do the "wrong" thing since the position has continued to advance + // in the mean time. If we ever provide a sequencer in server, we could allow a way for the app + // to specify how it wants to handle such scenarios. + if (mState == STATE_ACTIVE) { return INVALID_OPERATION; } + mNewPosition = mProxy->getPosition() + mUpdatePeriod; + mLoopPeriod = 0; + // FIXME Check whether loops and setting position are incompatible in old code. + // If we use setLoop for both purposes we lose the capability to set the position while looping. + mStaticProxy->setLoop(position, mFrameCount, 0); - audio_track_cblk_t* cblk = mCblk; - Mutex::Autolock _l(cblk->lock); + return NO_ERROR; +} - if (position > cblk->user) { +status_t AudioTrack::getPosition(uint32_t *position) const +{ + if (position == NULL) { return BAD_VALUE; } - cblk->server = position; - android_atomic_or(CBLK_FORCEREADY, &cblk->flags); + AutoMutex lock(mLock); + if (isOffloaded()) { + uint32_t dspFrames = 0; + if (mOutput != 0) { + uint32_t halFrames; + AudioSystem::getRenderPosition(mOutput, &halFrames, &dspFrames); + } + *position = dspFrames; + } else { + // IAudioTrack::stop() isn't synchronous; we don't know when presentation completes + *position = (mState == STATE_STOPPED || mState == STATE_FLUSHED) ? 0 : + mProxy->getPosition(); + } return NO_ERROR; } -status_t AudioTrack::getPosition(uint32_t *position) +status_t AudioTrack::getBufferPosition(size_t *position) { + if (mSharedBuffer == 0 || mIsTimed) { + return INVALID_OPERATION; + } if (position == NULL) { return BAD_VALUE; } - AutoMutex lock(mLock); - *position = mFlushed ? 0 : mCblk->server; + AutoMutex lock(mLock); + *position = mStaticProxy->getBufferPosition(); return NO_ERROR; } status_t AudioTrack::reload() { - if (mStatus != NO_ERROR) { - return mStatus; - } - ALOG_ASSERT(mProxy != NULL); - - if (mSharedBuffer == 0 || mIsTimed) { + if (mSharedBuffer == 0 || mIsTimed || isOffloaded()) { return INVALID_OPERATION; } AutoMutex lock(mLock); - - if (!stopped_l()) { + // See setPosition() regarding setting parameters such as loop points or position while active + if (mState == STATE_ACTIVE) { return INVALID_OPERATION; } - - flush_l(); - - (void) mProxy->stepUser(mFrameCount); - + mNewPosition = mUpdatePeriod; + mLoopPeriod = 0; + // FIXME The new code cannot reload while keeping a loop specified. + // Need to check how the old code handled this, and whether it's a significant change. + mStaticProxy->setLoop(0, mFrameCount, 0); return NO_ERROR; } audio_io_handle_t AudioTrack::getOutput() { AutoMutex lock(mLock); - return getOutput_l(); + return mOutput; } // must be called with mLock held audio_io_handle_t AudioTrack::getOutput_l() { - return AudioSystem::getOutput(mStreamType, - mSampleRate, mFormat, mChannelMask, mFlags); + if (mOutput) { + return mOutput; + } else { + return AudioSystem::getOutput(mStreamType, + mSampleRate, mFormat, mChannelMask, mFlags); + } } status_t AudioTrack::attachAuxEffect(int effectId) { - ALOGV("attachAuxEffect(%d)", effectId); + AutoMutex lock(mLock); status_t status = mAudioTrack->attachAuxEffect(effectId); if (status == NO_ERROR) { mAuxEffectId = effectId; @@ -737,7 +808,8 @@ status_t AudioTrack::createTrack_l( size_t frameCount, audio_output_flags_t flags, const sp<IMemory>& sharedBuffer, - audio_io_handle_t output) + audio_io_handle_t output, + size_t epoch) { status_t status; const sp<IAudioFlinger>& audioFlinger = AudioSystem::get_audio_flinger(); @@ -746,8 +818,26 @@ status_t AudioTrack::createTrack_l( return NO_INIT; } + // Not all of these values are needed under all conditions, but it is easier to get them all + uint32_t afLatency; - if (AudioSystem::getLatency(output, streamType, &afLatency) != NO_ERROR) { + status = AudioSystem::getLatency(output, streamType, &afLatency); + if (status != NO_ERROR) { + ALOGE("getLatency(%d) failed status %d", output, status); + return NO_INIT; + } + + size_t afFrameCount; + status = AudioSystem::getFrameCount(output, streamType, &afFrameCount); + if (status != NO_ERROR) { + ALOGE("getFrameCount(output=%d, streamType=%d) status %d", output, streamType, status); + return NO_INIT; + } + + uint32_t afSampleRate; + status = AudioSystem::getSamplingRate(output, streamType, &afSampleRate); + if (status != NO_ERROR) { + ALOGE("getSamplingRate(output=%d, streamType=%d) status %d", output, streamType, status); return NO_INIT; } @@ -766,6 +856,14 @@ status_t AudioTrack::createTrack_l( } ALOGV("createTrack_l() output %d afLatency %d", output, afLatency); + // The client's AudioTrack buffer is divided into n parts for purpose of wakeup by server, where + // n = 1 fast track; nBuffering is ignored + // n = 2 normal track, no sample rate conversion + // n = 3 normal track, with sample rate conversion + // (pessimistic; some non-1:1 conversion ratios don't actually need triple-buffering) + // n > 3 very high latency or very small notification interval; nBuffering is ignored + const uint32_t nBuffering = (sampleRate == afSampleRate) ? 2 : 3; + mNotificationFramesAct = mNotificationFramesReq; if (!audio_is_linear_pcm(format)) { @@ -774,13 +872,11 @@ status_t AudioTrack::createTrack_l( // Same comment as below about ignoring frameCount parameter for set() frameCount = sharedBuffer->size(); } else if (frameCount == 0) { - size_t afFrameCount; - if (AudioSystem::getFrameCount(output, streamType, &afFrameCount) != NO_ERROR) { - return NO_INIT; - } frameCount = afFrameCount; } - + if (mNotificationFramesAct != frameCount) { + mNotificationFramesAct = frameCount; + } } else if (sharedBuffer != 0) { // Ensure that buffer alignment matches channel count @@ -805,18 +901,14 @@ status_t AudioTrack::createTrack_l( } else if (!(flags & AUDIO_OUTPUT_FLAG_FAST)) { // FIXME move these calculations and associated checks to server - uint32_t afSampleRate; - if (AudioSystem::getSamplingRate(output, streamType, &afSampleRate) != NO_ERROR) { - return NO_INIT; - } - size_t afFrameCount; - if (AudioSystem::getFrameCount(output, streamType, &afFrameCount) != NO_ERROR) { - return NO_INIT; - } // Ensure that buffer depth covers at least audio hardware latency uint32_t minBufCount = afLatency / ((1000 * afFrameCount)/afSampleRate); - if (minBufCount < 2) minBufCount = 2; + ALOGV("afFrameCount=%d, minBufCount=%d, afSampleRate=%u, afLatency=%d", + afFrameCount, minBufCount, afSampleRate, afLatency); + if (minBufCount <= nBuffering) { + minBufCount = nBuffering; + } size_t minFrameCount = (afFrameCount*sampleRate*minBufCount)/afSampleRate; ALOGV("minFrameCount: %u, afFrameCount=%d, minBufCount=%d, sampleRate=%u, afSampleRate=%u" @@ -825,21 +917,16 @@ status_t AudioTrack::createTrack_l( if (frameCount == 0) { frameCount = minFrameCount; - } - if (mNotificationFramesAct == 0) { - mNotificationFramesAct = frameCount/2; - } - // Make sure that application is notified with sufficient margin - // before underrun - if (mNotificationFramesAct > frameCount/2) { - mNotificationFramesAct = frameCount/2; - } - if (frameCount < minFrameCount) { + } else if (frameCount < minFrameCount) { // not ALOGW because it happens all the time when playing key clicks over A2DP ALOGV("Minimum buffer size corrected from %d to %d", frameCount, minFrameCount); frameCount = minFrameCount; } + // Make sure that application is notified with sufficient margin before underrun + if (mNotificationFramesAct == 0 || mNotificationFramesAct > frameCount/nBuffering) { + mNotificationFramesAct = frameCount/nBuffering; + } } else { // For fast tracks, the frame count calculations and checks are done by server @@ -858,6 +945,10 @@ status_t AudioTrack::createTrack_l( } } + if (flags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) { + trackFlags |= IAudioFlinger::TRACK_OFFLOAD; + } + sp<IAudioTrack> track = audioFlinger->createTrack(streamType, sampleRate, // AudioFlinger only sees 16-bit PCM @@ -870,6 +961,7 @@ status_t AudioTrack::createTrack_l( output, tid, &mSessionId, + mName, &status); if (track == 0) { @@ -881,6 +973,11 @@ status_t AudioTrack::createTrack_l( ALOGE("Could not get control block"); return NO_INIT; } + // invariant that mAudioTrack != 0 is true only after set() returns successfully + if (mAudioTrack != 0) { + mAudioTrack->asBinder()->unlinkToDeath(mDeathNotifier, this); + mDeathNotifier.clear(); + } mAudioTrack = track; mCblkMemory = iMem; audio_track_cblk_t* cblk = static_cast<audio_track_cblk_t*>(iMem->pointer()); @@ -898,26 +995,49 @@ status_t AudioTrack::createTrack_l( if (trackFlags & IAudioFlinger::TRACK_FAST) { ALOGV("AUDIO_OUTPUT_FLAG_FAST successful; frameCount %u", frameCount); mAwaitBoost = true; + if (sharedBuffer == 0) { + // double-buffering is not required for fast tracks, due to tighter scheduling + if (mNotificationFramesAct == 0 || mNotificationFramesAct > frameCount) { + mNotificationFramesAct = frameCount; + } + } } else { ALOGV("AUDIO_OUTPUT_FLAG_FAST denied by server; frameCount %u", frameCount); // once denied, do not request again if IAudioTrack is re-created flags = (audio_output_flags_t) (flags & ~AUDIO_OUTPUT_FLAG_FAST); mFlags = flags; + if (sharedBuffer == 0) { + if (mNotificationFramesAct == 0 || mNotificationFramesAct > frameCount/nBuffering) { + mNotificationFramesAct = frameCount/nBuffering; + } + } } - if (sharedBuffer == 0) { - mNotificationFramesAct = frameCount/2; + } + if (flags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) { + if (trackFlags & IAudioFlinger::TRACK_OFFLOAD) { + ALOGV("AUDIO_OUTPUT_FLAG_OFFLOAD successful"); + } else { + ALOGW("AUDIO_OUTPUT_FLAG_OFFLOAD denied by server"); + flags = (audio_output_flags_t) (flags & ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD); + mFlags = flags; + return NO_INIT; } } + + mRefreshRemaining = true; + + // Starting address of buffers in shared memory. If there is a shared buffer, buffers + // is the value of pointer() for the shared buffer, otherwise buffers points + // immediately after the control block. This address is for the mapping within client + // address space. AudioFlinger::TrackBase::mBuffer is for the server address space. + void* buffers; if (sharedBuffer == 0) { - mBuffers = (char*)cblk + sizeof(audio_track_cblk_t); + buffers = (char*)cblk + sizeof(audio_track_cblk_t); } else { - mBuffers = sharedBuffer->pointer(); + buffers = sharedBuffer->pointer(); } mAudioTrack->attachAuxEffect(mAuxEffectId); - cblk->bufferTimeoutMs = MAX_STARTUP_TIMEOUT_MS; - cblk->waitTimeMs = 0; - mRemainingFrames = mNotificationFramesAct; // FIXME don't believe this lie mLatency = afLatency + (1000*frameCount) / sampleRate; mFrameCount = frameCount; @@ -928,147 +1048,154 @@ status_t AudioTrack::createTrack_l( } // update proxy - delete mProxy; - mProxy = new AudioTrackClientProxy(cblk, mBuffers, frameCount, mFrameSizeAF); + if (sharedBuffer == 0) { + mStaticProxy.clear(); + mProxy = new AudioTrackClientProxy(cblk, buffers, frameCount, mFrameSizeAF); + } else { + mStaticProxy = new StaticAudioTrackClientProxy(cblk, buffers, frameCount, mFrameSizeAF); + mProxy = mStaticProxy; + } mProxy->setVolumeLR((uint32_t(uint16_t(mVolume[RIGHT] * 0x1000)) << 16) | uint16_t(mVolume[LEFT] * 0x1000)); mProxy->setSendLevel(mSendLevel); mProxy->setSampleRate(mSampleRate); - if (sharedBuffer != 0) { - // Force buffer full condition as data is already present in shared memory - mProxy->stepUser(frameCount); - } + mProxy->setEpoch(epoch); + mProxy->setMinimum(mNotificationFramesAct); + + mDeathNotifier = new DeathNotifier(this); + mAudioTrack->asBinder()->linkToDeath(mDeathNotifier, this); return NO_ERROR; } status_t AudioTrack::obtainBuffer(Buffer* audioBuffer, int32_t waitCount) { - ALOG_ASSERT(mStatus == NO_ERROR && mProxy != NULL); + if (audioBuffer == NULL) { + return BAD_VALUE; + } + if (mTransfer != TRANSFER_OBTAIN) { + audioBuffer->frameCount = 0; + audioBuffer->size = 0; + audioBuffer->raw = NULL; + return INVALID_OPERATION; + } - AutoMutex lock(mLock); - bool active; - status_t result = NO_ERROR; - audio_track_cblk_t* cblk = mCblk; - uint32_t framesReq = audioBuffer->frameCount; - uint32_t waitTimeMs = (waitCount < 0) ? cblk->bufferTimeoutMs : WAIT_PERIOD_MS; + const struct timespec *requested; + if (waitCount == -1) { + requested = &ClientProxy::kForever; + } else if (waitCount == 0) { + requested = &ClientProxy::kNonBlocking; + } else if (waitCount > 0) { + long long ms = WAIT_PERIOD_MS * (long long) waitCount; + struct timespec timeout; + timeout.tv_sec = ms / 1000; + timeout.tv_nsec = (int) (ms % 1000) * 1000000; + requested = &timeout; + } else { + ALOGE("%s invalid waitCount %d", __func__, waitCount); + requested = NULL; + } + return obtainBuffer(audioBuffer, requested); +} + +status_t AudioTrack::obtainBuffer(Buffer* audioBuffer, const struct timespec *requested, + struct timespec *elapsed, size_t *nonContig) +{ + // previous and new IAudioTrack sequence numbers are used to detect track re-creation + uint32_t oldSequence = 0; + uint32_t newSequence; - audioBuffer->frameCount = 0; - audioBuffer->size = 0; + Proxy::Buffer buffer; + status_t status = NO_ERROR; - size_t framesAvail = mProxy->framesAvailable(); + static const int32_t kMaxTries = 5; + int32_t tryCounter = kMaxTries; - cblk->lock.lock(); - if (cblk->flags & CBLK_INVALID) { - goto create_new_track; - } - cblk->lock.unlock(); + do { + // obtainBuffer() is called with mutex unlocked, so keep extra references to these fields to + // keep them from going away if another thread re-creates the track during obtainBuffer() + sp<AudioTrackClientProxy> proxy; + sp<IMemory> iMem; - if (framesAvail == 0) { - cblk->lock.lock(); - goto start_loop_here; - while (framesAvail == 0) { - active = mActive; - if (CC_UNLIKELY(!active)) { - ALOGV("Not active and NO_MORE_BUFFERS"); - cblk->lock.unlock(); - return NO_MORE_BUFFERS; - } - if (CC_UNLIKELY(!waitCount)) { - cblk->lock.unlock(); - return WOULD_BLOCK; - } - if (!(cblk->flags & CBLK_INVALID)) { - mLock.unlock(); - // this condition is in shared memory, so if IAudioTrack and control block - // are replaced due to mediaserver death or IAudioTrack invalidation then - // cv won't be signalled, but fortunately the timeout will limit the wait - result = cblk->cv.waitRelative(cblk->lock, milliseconds(waitTimeMs)); - cblk->lock.unlock(); - mLock.lock(); - if (!mActive) { - return status_t(STOPPED); + { // start of lock scope + AutoMutex lock(mLock); + + newSequence = mSequence; + // did previous obtainBuffer() fail due to media server death or voluntary invalidation? + if (status == DEAD_OBJECT) { + // re-create track, unless someone else has already done so + if (newSequence == oldSequence) { + status = restoreTrack_l("obtainBuffer"); + if (status != NO_ERROR) { + buffer.mFrameCount = 0; + buffer.mRaw = NULL; + buffer.mNonContig = 0; + break; + } } - // IAudioTrack may have been re-created while mLock was unlocked - cblk = mCblk; - cblk->lock.lock(); } + oldSequence = newSequence; + + // Keep the extra references + proxy = mProxy; + iMem = mCblkMemory; - if (cblk->flags & CBLK_INVALID) { - goto create_new_track; + if (mState == STATE_STOPPING) { + status = -EINTR; + buffer.mFrameCount = 0; + buffer.mRaw = NULL; + buffer.mNonContig = 0; + break; } - if (CC_UNLIKELY(result != NO_ERROR)) { - cblk->waitTimeMs += waitTimeMs; - if (cblk->waitTimeMs >= cblk->bufferTimeoutMs) { - // timing out when a loop has been set and we have already written upto loop end - // is a normal condition: no need to wake AudioFlinger up. - if (cblk->user < cblk->loopEnd) { - ALOGW("obtainBuffer timed out (is the CPU pegged?) %p name=%#x user=%08x, " - "server=%08x", this, cblk->mName, cblk->user, cblk->server); - //unlock cblk mutex before calling mAudioTrack->start() (see issue #1617140) - cblk->lock.unlock(); - result = mAudioTrack->start(); - cblk->lock.lock(); - if (result == DEAD_OBJECT) { - android_atomic_or(CBLK_INVALID, &cblk->flags); -create_new_track: - audio_track_cblk_t* temp = cblk; - result = restoreTrack_l(temp, false /*fromStart*/); - cblk = temp; - } - if (result != NO_ERROR) { - ALOGW("obtainBuffer create Track error %d", result); - cblk->lock.unlock(); - return result; - } - } - cblk->waitTimeMs = 0; - } - if (--waitCount == 0) { - cblk->lock.unlock(); - return TIMED_OUT; - } + // Non-blocking if track is stopped or paused + if (mState != STATE_ACTIVE) { + requested = &ClientProxy::kNonBlocking; } - // read the server count again - start_loop_here: - framesAvail = mProxy->framesAvailable_l(); - } - cblk->lock.unlock(); - } - cblk->waitTimeMs = 0; + } // end of lock scope - if (framesReq > framesAvail) { - framesReq = framesAvail; - } + buffer.mFrameCount = audioBuffer->frameCount; + // FIXME starts the requested timeout and elapsed over from scratch + status = proxy->obtainBuffer(&buffer, requested, elapsed); - uint32_t u = cblk->user; - uint32_t bufferEnd = cblk->userBase + mFrameCount; + } while ((status == DEAD_OBJECT) && (tryCounter-- > 0)); - if (framesReq > bufferEnd - u) { - framesReq = bufferEnd - u; + audioBuffer->frameCount = buffer.mFrameCount; + audioBuffer->size = buffer.mFrameCount * mFrameSizeAF; + audioBuffer->raw = buffer.mRaw; + if (nonContig != NULL) { + *nonContig = buffer.mNonContig; } - - audioBuffer->frameCount = framesReq; - audioBuffer->size = framesReq * mFrameSizeAF; - audioBuffer->raw = mProxy->buffer(u); - active = mActive; - return active ? status_t(NO_ERROR) : status_t(STOPPED); + return status; } void AudioTrack::releaseBuffer(Buffer* audioBuffer) { - ALOG_ASSERT(mStatus == NO_ERROR && mProxy != NULL); + if (mTransfer == TRANSFER_SHARED) { + return; + } + + size_t stepCount = audioBuffer->size / mFrameSizeAF; + if (stepCount == 0) { + return; + } + + Proxy::Buffer buffer; + buffer.mFrameCount = stepCount; + buffer.mRaw = audioBuffer->raw; AutoMutex lock(mLock); - audio_track_cblk_t* cblk = mCblk; - (void) mProxy->stepUser(audioBuffer->frameCount); - if (audioBuffer->frameCount > 0) { - // restart track if it was disabled by audioflinger due to previous underrun - if (mActive && (cblk->flags & CBLK_DISABLED)) { - android_atomic_and(~CBLK_DISABLED, &cblk->flags); - ALOGW("releaseBuffer() track %p name=%#x disabled, restarting", this, cblk->mName); + mInUnderrun = false; + mProxy->releaseBuffer(&buffer); + + // restart track if it was disabled by audioflinger due to previous underrun + if (mState == STATE_ACTIVE) { + audio_track_cblk_t* cblk = mCblk; + if (android_atomic_and(~CBLK_DISABLED, &cblk->mFlags) & CBLK_DISABLED) { + ALOGW("releaseBuffer() track %p name=%s disabled due to previous underrun, restarting", + this, mName.string()); + // FIXME ignoring status mAudioTrack->start(); } } @@ -1078,68 +1205,46 @@ void AudioTrack::releaseBuffer(Buffer* audioBuffer) ssize_t AudioTrack::write(const void* buffer, size_t userSize) { - - if (mSharedBuffer != 0 || mIsTimed) { + if (mTransfer != TRANSFER_SYNC || mIsTimed) { return INVALID_OPERATION; } - if (ssize_t(userSize) < 0) { + if (ssize_t(userSize) < 0 || (buffer == NULL && userSize != 0)) { // Sanity-check: user is most-likely passing an error code, and it would // make the return value ambiguous (actualSize vs error). - ALOGE("AudioTrack::write(buffer=%p, size=%u (%d)", - buffer, userSize, userSize); + ALOGE("AudioTrack::write(buffer=%p, size=%u (%d)", buffer, userSize, userSize); return BAD_VALUE; } - ALOGV("write %p: %d bytes, mActive=%d", this, userSize, mActive); - - if (userSize == 0) { - return 0; - } - - // acquire a strong reference on the IMemory and IAudioTrack so that they cannot be destroyed - // while we are accessing the cblk - mLock.lock(); - sp<IAudioTrack> audioTrack = mAudioTrack; - sp<IMemory> iMem = mCblkMemory; - mLock.unlock(); - - // since mLock is unlocked the IAudioTrack and shared memory may be re-created, - // so all cblk references might still refer to old shared memory, but that should be benign - - ssize_t written = 0; - const int8_t *src = (const int8_t *)buffer; + size_t written = 0; Buffer audioBuffer; - size_t frameSz = frameSize(); - do { - audioBuffer.frameCount = userSize/frameSz; + while (userSize >= mFrameSize) { + audioBuffer.frameCount = userSize / mFrameSize; - status_t err = obtainBuffer(&audioBuffer, -1); + status_t err = obtainBuffer(&audioBuffer, &ClientProxy::kForever); if (err < 0) { - // out of buffers, return #bytes written - if (err == status_t(NO_MORE_BUFFERS)) { + if (written > 0) { break; } return ssize_t(err); } size_t toWrite; - if (mFormat == AUDIO_FORMAT_PCM_8_BIT && !(mFlags & AUDIO_OUTPUT_FLAG_DIRECT)) { // Divide capacity by 2 to take expansion into account - toWrite = audioBuffer.size>>1; - memcpy_to_i16_from_u8(audioBuffer.i16, (const uint8_t *) src, toWrite); + toWrite = audioBuffer.size >> 1; + memcpy_to_i16_from_u8(audioBuffer.i16, (const uint8_t *) buffer, toWrite); } else { toWrite = audioBuffer.size; - memcpy(audioBuffer.i8, src, toWrite); + memcpy(audioBuffer.i8, buffer, toWrite); } - src += toWrite; + buffer = ((const char *) buffer) + toWrite; userSize -= toWrite; written += toWrite; releaseBuffer(&audioBuffer); - } while (userSize >= frameSz); + } return written; } @@ -1155,32 +1260,30 @@ status_t TimedAudioTrack::allocateTimedBuffer(size_t size, sp<IMemory>* buffer) AutoMutex lock(mLock); status_t result = UNKNOWN_ERROR; +#if 1 // acquire a strong reference on the IMemory and IAudioTrack so that they cannot be destroyed // while we are accessing the cblk sp<IAudioTrack> audioTrack = mAudioTrack; sp<IMemory> iMem = mCblkMemory; +#endif // If the track is not invalid already, try to allocate a buffer. alloc // fails indicating that the server is dead, flag the track as invalid so // we can attempt to restore in just a bit. audio_track_cblk_t* cblk = mCblk; - if (!(cblk->flags & CBLK_INVALID)) { + if (!(cblk->mFlags & CBLK_INVALID)) { result = mAudioTrack->allocateTimedBuffer(size, buffer); if (result == DEAD_OBJECT) { - android_atomic_or(CBLK_INVALID, &cblk->flags); + android_atomic_or(CBLK_INVALID, &cblk->mFlags); } } // If the track is invalid at this point, attempt to restore it. and try the // allocation one more time. - if (cblk->flags & CBLK_INVALID) { - cblk->lock.lock(); - audio_track_cblk_t* temp = cblk; - result = restoreTrack_l(temp, false /*fromStart*/); - cblk = temp; - cblk->lock.unlock(); - - if (result == OK) { + if (cblk->mFlags & CBLK_INVALID) { + result = restoreTrack_l("allocateTimedBuffer"); + + if (result == NO_ERROR) { result = mAudioTrack->allocateTimedBuffer(size, buffer); } } @@ -1197,9 +1300,10 @@ status_t TimedAudioTrack::queueTimedBuffer(const sp<IMemory>& buffer, audio_track_cblk_t* cblk = mCblk; // restart track if it was disabled by audioflinger due to previous underrun if (buffer->size() != 0 && status == NO_ERROR && - mActive && (cblk->flags & CBLK_DISABLED)) { - android_atomic_and(~CBLK_DISABLED, &cblk->flags); + (mState == STATE_ACTIVE) && (cblk->mFlags & CBLK_DISABLED)) { + android_atomic_and(~CBLK_DISABLED, &cblk->mFlags); ALOGW("queueTimedBuffer() track %p disabled, restarting", this); + // FIXME ignoring status mAudioTrack->start(); } } @@ -1214,11 +1318,12 @@ status_t TimedAudioTrack::setMediaTimeTransform(const LinearTransform& xform, // ------------------------------------------------------------------------- -bool AudioTrack::processAudioBuffer(const sp<AudioTrackThread>& thread) +nsecs_t AudioTrack::processAudioBuffer(const sp<AudioTrackThread>& thread) { - Buffer audioBuffer; - uint32_t frames; - size_t writtenSize; + // Currently the AudioTrack thread is not created if there are no callbacks. + // Would it ever make sense to run the thread, even without callbacks? + // If so, then replace this by checks at each use for mCbf != NULL. + LOG_ALWAYS_FATAL_IF(mCblk == NULL); mLock.lock(); if (mAwaitBoost) { @@ -1238,88 +1343,228 @@ bool AudioTrack::processAudioBuffer(const sp<AudioTrackThread>& thread) if (tryCounter < 0) { ALOGE("did not receive expected priority boost on time"); } - return true; + // Run again immediately + return 0; } - // acquire a strong reference on the IMemory and IAudioTrack so that they cannot be destroyed - // while we are accessing the cblk - sp<IAudioTrack> audioTrack = mAudioTrack; - sp<IMemory> iMem = mCblkMemory; - audio_track_cblk_t* cblk = mCblk; - bool active = mActive; + + // Can only reference mCblk while locked + int32_t flags = android_atomic_and( + ~(CBLK_UNDERRUN | CBLK_LOOP_CYCLE | CBLK_LOOP_FINAL | CBLK_BUFFER_END), &mCblk->mFlags); + + // Check for track invalidation + if (flags & CBLK_INVALID) { + // for offloaded tracks restoreTrack_l() will just update the sequence and clear + // AudioSystem cache. We should not exit here but after calling the callback so + // that the upper layers can recreate the track + if (!isOffloaded() || (mSequence == mObservedSequence)) { + status_t status = restoreTrack_l("processAudioBuffer"); + mLock.unlock(); + // Run again immediately, but with a new IAudioTrack + return 0; + } + } + + bool waitStreamEnd = mState == STATE_STOPPING; + bool active = mState == STATE_ACTIVE; + + // Manage underrun callback, must be done under lock to avoid race with releaseBuffer() + bool newUnderrun = false; + if (flags & CBLK_UNDERRUN) { +#if 0 + // Currently in shared buffer mode, when the server reaches the end of buffer, + // the track stays active in continuous underrun state. It's up to the application + // to pause or stop the track, or set the position to a new offset within buffer. + // This was some experimental code to auto-pause on underrun. Keeping it here + // in "if 0" so we can re-visit this if we add a real sequencer for shared memory content. + if (mTransfer == TRANSFER_SHARED) { + mState = STATE_PAUSED; + active = false; + } +#endif + if (!mInUnderrun) { + mInUnderrun = true; + newUnderrun = true; + } + } + + // Get current position of server + size_t position = mProxy->getPosition(); + + // Manage marker callback + bool markerReached = false; + size_t markerPosition = mMarkerPosition; + // FIXME fails for wraparound, need 64 bits + if (!mMarkerReached && (markerPosition > 0) && (position >= markerPosition)) { + mMarkerReached = markerReached = true; + } + + // Determine number of new position callback(s) that will be needed, while locked + size_t newPosCount = 0; + size_t newPosition = mNewPosition; + size_t updatePeriod = mUpdatePeriod; + // FIXME fails for wraparound, need 64 bits + if (updatePeriod > 0 && position >= newPosition) { + newPosCount = ((position - newPosition) / updatePeriod) + 1; + mNewPosition += updatePeriod * newPosCount; + } + + // Cache other fields that will be needed soon + uint32_t loopPeriod = mLoopPeriod; + uint32_t sampleRate = mSampleRate; + size_t notificationFrames = mNotificationFramesAct; + if (mRefreshRemaining) { + mRefreshRemaining = false; + mRemainingFrames = notificationFrames; + mRetryOnPartialBuffer = false; + } + size_t misalignment = mProxy->getMisalignment(); + uint32_t sequence = mSequence; + + // These fields don't need to be cached, because they are assigned only by set(): + // mTransfer, mCbf, mUserData, mFormat, mFrameSize, mFrameSizeAF, mFlags + // mFlags is also assigned by createTrack_l(), but not the bit we care about. + mLock.unlock(); - // since mLock is unlocked the IAudioTrack and shared memory may be re-created, - // so all cblk references might still refer to old shared memory, but that should be benign + if (waitStreamEnd) { + AutoMutex lock(mLock); - // Manage underrun callback - if (active && (mProxy->framesAvailable() == mFrameCount)) { - ALOGV("Underrun user: %x, server: %x, flags %04x", cblk->user, cblk->server, cblk->flags); - if (!(android_atomic_or(CBLK_UNDERRUN, &cblk->flags) & CBLK_UNDERRUN)) { - mCbf(EVENT_UNDERRUN, mUserData, 0); - if (cblk->server == mFrameCount) { - mCbf(EVENT_BUFFER_END, mUserData, 0); - } - if (mSharedBuffer != 0) { - return false; + sp<AudioTrackClientProxy> proxy = mProxy; + sp<IMemory> iMem = mCblkMemory; + + struct timespec timeout; + timeout.tv_sec = WAIT_STREAM_END_TIMEOUT_SEC; + timeout.tv_nsec = 0; + + mLock.unlock(); + status_t status = mProxy->waitStreamEndDone(&timeout); + mLock.lock(); + switch (status) { + case NO_ERROR: + case DEAD_OBJECT: + case TIMED_OUT: + mLock.unlock(); + mCbf(EVENT_STREAM_END, mUserData, NULL); + mLock.lock(); + if (mState == STATE_STOPPING) { + mState = STATE_STOPPED; + if (status != DEAD_OBJECT) { + return NS_INACTIVE; + } } + return 0; + default: + return 0; } } - // Manage loop end callback - while (mLoopCount > cblk->loopCount) { - int loopCount = -1; - mLoopCount--; - if (mLoopCount >= 0) loopCount = mLoopCount; - - mCbf(EVENT_LOOP_END, mUserData, (void *)&loopCount); + // perform callbacks while unlocked + if (newUnderrun) { + mCbf(EVENT_UNDERRUN, mUserData, NULL); + } + // FIXME we will miss loops if loop cycle was signaled several times since last call + // to processAudioBuffer() + if (flags & (CBLK_LOOP_CYCLE | CBLK_LOOP_FINAL)) { + mCbf(EVENT_LOOP_END, mUserData, NULL); + } + if (flags & CBLK_BUFFER_END) { + mCbf(EVENT_BUFFER_END, mUserData, NULL); + } + if (markerReached) { + mCbf(EVENT_MARKER, mUserData, &markerPosition); + } + while (newPosCount > 0) { + size_t temp = newPosition; + mCbf(EVENT_NEW_POS, mUserData, &temp); + newPosition += updatePeriod; + newPosCount--; } - // Manage marker callback - if (!mMarkerReached && (mMarkerPosition > 0)) { - if (cblk->server >= mMarkerPosition) { - mCbf(EVENT_MARKER, mUserData, (void *)&mMarkerPosition); - mMarkerReached = true; + if (mObservedSequence != sequence) { + mObservedSequence = sequence; + mCbf(EVENT_NEW_IAUDIOTRACK, mUserData, NULL); + // for offloaded tracks, just wait for the upper layers to recreate the track + if (isOffloaded()) { + return NS_INACTIVE; } } - // Manage new position callback - if (mUpdatePeriod > 0) { - while (cblk->server >= mNewPosition) { - mCbf(EVENT_NEW_POS, mUserData, (void *)&mNewPosition); - mNewPosition += mUpdatePeriod; - } + // if inactive, then don't run me again until re-started + if (!active) { + return NS_INACTIVE; } - // If Shared buffer is used, no data is requested from client. - if (mSharedBuffer != 0) { - frames = 0; - } else { - frames = mRemainingFrames; + // Compute the estimated time until the next timed event (position, markers, loops) + // FIXME only for non-compressed audio + uint32_t minFrames = ~0; + if (!markerReached && position < markerPosition) { + minFrames = markerPosition - position; + } + if (loopPeriod > 0 && loopPeriod < minFrames) { + minFrames = loopPeriod; + } + if (updatePeriod > 0 && updatePeriod < minFrames) { + minFrames = updatePeriod; } - // See description of waitCount parameter at declaration of obtainBuffer(). - // The logic below prevents us from being stuck below at obtainBuffer() - // not being able to handle timed events (position, markers, loops). - int32_t waitCount = -1; - if (mUpdatePeriod || (!mMarkerReached && mMarkerPosition) || mLoopCount) { - waitCount = 1; + // If > 0, poll periodically to recover from a stuck server. A good value is 2. + static const uint32_t kPoll = 0; + if (kPoll > 0 && mTransfer == TRANSFER_CALLBACK && kPoll * notificationFrames < minFrames) { + minFrames = kPoll * notificationFrames; } - do { + // Convert frame units to time units + nsecs_t ns = NS_WHENEVER; + if (minFrames != (uint32_t) ~0) { + // This "fudge factor" avoids soaking CPU, and compensates for late progress by server + static const nsecs_t kFudgeNs = 10000000LL; // 10 ms + ns = ((minFrames * 1000000000LL) / sampleRate) + kFudgeNs; + } + + // If not supplying data by EVENT_MORE_DATA, then we're done + if (mTransfer != TRANSFER_CALLBACK) { + return ns; + } + + struct timespec timeout; + const struct timespec *requested = &ClientProxy::kForever; + if (ns != NS_WHENEVER) { + timeout.tv_sec = ns / 1000000000LL; + timeout.tv_nsec = ns % 1000000000LL; + ALOGV("timeout %ld.%03d", timeout.tv_sec, (int) timeout.tv_nsec / 1000000); + requested = &timeout; + } - audioBuffer.frameCount = frames; + while (mRemainingFrames > 0) { - status_t err = obtainBuffer(&audioBuffer, waitCount); - if (err < NO_ERROR) { - if (err != TIMED_OUT) { - ALOGE_IF(err != status_t(NO_MORE_BUFFERS), - "Error obtaining an audio buffer, giving up."); - return false; + Buffer audioBuffer; + audioBuffer.frameCount = mRemainingFrames; + size_t nonContig; + status_t err = obtainBuffer(&audioBuffer, requested, NULL, &nonContig); + LOG_ALWAYS_FATAL_IF((err != NO_ERROR) != (audioBuffer.frameCount == 0), + "obtainBuffer() err=%d frameCount=%u", err, audioBuffer.frameCount); + requested = &ClientProxy::kNonBlocking; + size_t avail = audioBuffer.frameCount + nonContig; + ALOGV("obtainBuffer(%u) returned %u = %u + %u err %d", + mRemainingFrames, avail, audioBuffer.frameCount, nonContig, err); + if (err != NO_ERROR) { + if (err == TIMED_OUT || err == WOULD_BLOCK || err == -EINTR || + (isOffloaded() && (err == DEAD_OBJECT))) { + return 0; } - break; + ALOGE("Error %d obtaining an audio buffer, giving up.", err); + return NS_NEVER; } - if (err == status_t(STOPPED)) { - return false; + + if (mRetryOnPartialBuffer && !isOffloaded()) { + mRetryOnPartialBuffer = false; + if (avail < mRemainingFrames) { + int64_t myns = ((mRemainingFrames - avail) * 1100000000LL) / sampleRate; + if (ns < 0 || myns < ns) { + ns = myns; + } + return ns; + } } // Divide buffer size by 2 to take into account the expansion @@ -1331,139 +1576,175 @@ bool AudioTrack::processAudioBuffer(const sp<AudioTrackThread>& thread) size_t reqSize = audioBuffer.size; mCbf(EVENT_MORE_DATA, mUserData, &audioBuffer); - writtenSize = audioBuffer.size; + size_t writtenSize = audioBuffer.size; + size_t writtenFrames = writtenSize / mFrameSize; // Sanity check on returned size - if (ssize_t(writtenSize) <= 0) { + if (ssize_t(writtenSize) < 0 || writtenSize > reqSize) { + ALOGE("EVENT_MORE_DATA requested %u bytes but callback returned %d bytes", + reqSize, (int) writtenSize); + return NS_NEVER; + } + + if (writtenSize == 0) { // The callback is done filling buffers // Keep this thread going to handle timed events and // still try to get more data in intervals of WAIT_PERIOD_MS // but don't just loop and block the CPU, so wait - usleep(WAIT_PERIOD_MS*1000); - break; - } - - if (writtenSize > reqSize) { - writtenSize = reqSize; + return WAIT_PERIOD_MS * 1000000LL; } if (mFormat == AUDIO_FORMAT_PCM_8_BIT && !(mFlags & AUDIO_OUTPUT_FLAG_DIRECT)) { // 8 to 16 bit conversion, note that source and destination are the same address memcpy_to_i16_from_u8(audioBuffer.i16, (const uint8_t *) audioBuffer.i8, writtenSize); - writtenSize <<= 1; + audioBuffer.size <<= 1; } - audioBuffer.size = writtenSize; - // NOTE: cblk->frameSize is not equal to AudioTrack::frameSize() for - // 8 bit PCM data: in this case, cblk->frameSize is based on a sample size of - // 16 bit. - audioBuffer.frameCount = writtenSize / mFrameSizeAF; - - frames -= audioBuffer.frameCount; + size_t releasedFrames = audioBuffer.size / mFrameSizeAF; + audioBuffer.frameCount = releasedFrames; + mRemainingFrames -= releasedFrames; + if (misalignment >= releasedFrames) { + misalignment -= releasedFrames; + } else { + misalignment = 0; + } releaseBuffer(&audioBuffer); - } - while (frames); - if (frames == 0) { - mRemainingFrames = mNotificationFramesAct; - } else { - mRemainingFrames = frames; + // FIXME here is where we would repeat EVENT_MORE_DATA again on same advanced buffer + // if callback doesn't like to accept the full chunk + if (writtenSize < reqSize) { + continue; + } + + // There could be enough non-contiguous frames available to satisfy the remaining request + if (mRemainingFrames <= nonContig) { + continue; + } + +#if 0 + // This heuristic tries to collapse a series of EVENT_MORE_DATA that would total to a + // sum <= notificationFrames. It replaces that series by at most two EVENT_MORE_DATA + // that total to a sum == notificationFrames. + if (0 < misalignment && misalignment <= mRemainingFrames) { + mRemainingFrames = misalignment; + return (mRemainingFrames * 1100000000LL) / sampleRate; + } +#endif + } - return true; + mRemainingFrames = notificationFrames; + mRetryOnPartialBuffer = true; + + // A lot has transpired since ns was calculated, so run again immediately and re-calculate + return 0; } -// must be called with mLock and refCblk.lock held. Callers must also hold strong references on -// the IAudioTrack and IMemory in case they are recreated here. -// If the IAudioTrack is successfully restored, the refCblk pointer is updated -// FIXME Don't depend on caller to hold strong references. -status_t AudioTrack::restoreTrack_l(audio_track_cblk_t*& refCblk, bool fromStart) +status_t AudioTrack::restoreTrack_l(const char *from) { + ALOGW("dead IAudioTrack, %s, creating a new one from %s()", + isOffloaded() ? "Offloaded" : "PCM", from); + ++mSequence; status_t result; - audio_track_cblk_t* cblk = refCblk; - audio_track_cblk_t* newCblk = cblk; - ALOGW("dead IAudioTrack, creating a new one from %s", - fromStart ? "start()" : "obtainBuffer()"); - - // signal old cblk condition so that other threads waiting for available buffers stop - // waiting now - cblk->cv.broadcast(); - cblk->lock.unlock(); - // refresh the audio configuration cache in this process to make sure we get new // output parameters in getOutput_l() and createTrack_l() AudioSystem::clearAudioConfigCache(); + if (isOffloaded()) { + return DEAD_OBJECT; + } + + // force new output query from audio policy manager; + mOutput = 0; + audio_io_handle_t output = getOutput_l(); + // if the new IAudioTrack is created, createTrack_l() will modify the // following member variables: mAudioTrack, mCblkMemory and mCblk. // It will also delete the strong references on previous IAudioTrack and IMemory + + // take the frames that will be lost by track recreation into account in saved position + size_t position = mProxy->getPosition() + mProxy->getFramesFilled(); + mNewPosition = position + mUpdatePeriod; + size_t bufferPosition = mStaticProxy != NULL ? mStaticProxy->getBufferPosition() : 0; result = createTrack_l(mStreamType, mSampleRate, mFormat, mReqFrameCount, // so that frame count never goes down mFlags, mSharedBuffer, - getOutput_l()); + output, + position /*epoch*/); if (result == NO_ERROR) { - uint32_t user = cblk->user; - uint32_t server = cblk->server; + // continue playback from last known position, but + // don't attempt to restore loop after invalidation; it's difficult and not worthwhile + if (mStaticProxy != NULL) { + mLoopPeriod = 0; + mStaticProxy->setLoop(bufferPosition, mFrameCount, 0); + } + // FIXME How do we simulate the fact that all frames present in the buffer at the time of + // track destruction have been played? This is critical for SoundPool implementation + // This must be broken, and needs to be tested/debugged. +#if 0 // restore write index and set other indexes to reflect empty buffer status - newCblk = mCblk; - newCblk->user = user; - newCblk->server = user; - newCblk->userBase = user; - newCblk->serverBase = user; - // restore loop: this is not guaranteed to succeed if new frame count is not - // compatible with loop length - setLoop_l(cblk->loopStart, cblk->loopEnd, cblk->loopCount); - size_t frames = 0; - if (!fromStart) { - newCblk->bufferTimeoutMs = MAX_RUN_TIMEOUT_MS; + if (!strcmp(from, "start")) { // Make sure that a client relying on callback events indicating underrun or // the actual amount of audio frames played (e.g SoundPool) receives them. if (mSharedBuffer == 0) { - if (user > server) { - frames = ((user - server) > mFrameCount) ? - mFrameCount : (user - server); - memset(mBuffers, 0, frames * mFrameSizeAF); - } // restart playback even if buffer is not completely filled. - android_atomic_or(CBLK_FORCEREADY, &newCblk->flags); + android_atomic_or(CBLK_FORCEREADY, &mCblk->mFlags); } } - if (mSharedBuffer != 0) { - frames = mFrameCount; - } - if (frames > 0) { - // stepUser() clears CBLK_UNDERRUN flag enabling underrun callbacks to - // the client - mProxy->stepUser(frames); - } - if (mActive) { +#endif + if (mState == STATE_ACTIVE) { result = mAudioTrack->start(); - ALOGW_IF(result != NO_ERROR, "restoreTrack_l() start() failed status %d", result); - } - if (fromStart && result == NO_ERROR) { - mNewPosition = newCblk->server + mUpdatePeriod; } } - ALOGW_IF(result != NO_ERROR, "restoreTrack_l() failed status %d", result); - ALOGV("restoreTrack_l() status %d mActive %d cblk %p, old cblk %p flags %08x old flags %08x", - result, mActive, newCblk, cblk, newCblk->flags, cblk->flags); - - if (result == NO_ERROR) { - // from now on we switch to the newly created cblk - refCblk = newCblk; + if (result != NO_ERROR) { + //Use of direct and offloaded output streams is ref counted by audio policy manager. + // As getOutput was called above and resulted in an output stream to be opened, + // we need to release it. + AudioSystem::releaseOutput(output); + ALOGW("restoreTrack_l() failed status %d", result); + mState = STATE_STOPPED; } - newCblk->lock.lock(); - - ALOGW_IF(result != NO_ERROR, "restoreTrack_l() error %d", result); return result; } +status_t AudioTrack::setParameters(const String8& keyValuePairs) +{ + AutoMutex lock(mLock); + return mAudioTrack->setParameters(keyValuePairs); +} + +status_t AudioTrack::getTimestamp(AudioTimestamp& timestamp) +{ + AutoMutex lock(mLock); + // FIXME not implemented for fast tracks; should use proxy and SSQ + if (mFlags & AUDIO_OUTPUT_FLAG_FAST) { + return INVALID_OPERATION; + } + if (mState != STATE_ACTIVE && mState != STATE_PAUSED) { + return INVALID_OPERATION; + } + status_t status = mAudioTrack->getTimestamp(timestamp); + if (status == NO_ERROR) { + timestamp.mPosition += mProxy->getEpoch(); + } + return status; +} + +String8 AudioTrack::getParameters(const String8& keys) +{ + if (mOutput) { + return AudioSystem::getParameters(mOutput, keys); + } else { + return String8::empty(); + } +} + status_t AudioTrack::dump(int fd, const Vector<String16>& args) const { @@ -1480,16 +1761,34 @@ status_t AudioTrack::dump(int fd, const Vector<String16>& args) const result.append(buffer); snprintf(buffer, 255, " sample rate(%u), status(%d)\n", mSampleRate, mStatus); result.append(buffer); - snprintf(buffer, 255, " active(%d), latency (%d)\n", mActive, mLatency); + snprintf(buffer, 255, " state(%d), latency (%d)\n", mState, mLatency); result.append(buffer); ::write(fd, result.string(), result.size()); return NO_ERROR; } +uint32_t AudioTrack::getUnderrunFrames() const +{ + AutoMutex lock(mLock); + return mProxy->getUnderrunFrames(); +} + +// ========================================================================= + +void AudioTrack::DeathNotifier::binderDied(const wp<IBinder>& who) +{ + sp<AudioTrack> audioTrack = mAudioTrack.promote(); + if (audioTrack != 0) { + AutoMutex lock(audioTrack->mLock); + audioTrack->mProxy->binderDied(); + } +} + // ========================================================================= AudioTrack::AudioTrackThread::AudioTrackThread(AudioTrack& receiver, bool bCanCallJava) - : Thread(bCanCallJava), mReceiver(receiver), mPaused(true) + : Thread(bCanCallJava), mReceiver(receiver), mPaused(true), mPausedInt(false), mPausedNs(0LL), + mIgnoreNextPausedInt(false) { } @@ -1506,11 +1805,38 @@ bool AudioTrack::AudioTrackThread::threadLoop() // caller will check for exitPending() return true; } + if (mIgnoreNextPausedInt) { + mIgnoreNextPausedInt = false; + mPausedInt = false; + } + if (mPausedInt) { + if (mPausedNs > 0) { + (void) mMyCond.waitRelative(mMyLock, mPausedNs); + } else { + mMyCond.wait(mMyLock); + } + mPausedInt = false; + return true; + } } - if (!mReceiver.processAudioBuffer(this)) { - pause(); + nsecs_t ns = mReceiver.processAudioBuffer(this); + switch (ns) { + case 0: + return true; + case NS_INACTIVE: + pauseInternal(); + return true; + case NS_NEVER: + return false; + case NS_WHENEVER: + // FIXME increase poll interval, or make event-driven + ns = 1000000000LL; + // fall through + default: + LOG_ALWAYS_FATAL_IF(ns < 0, "processAudioBuffer() returned %lld", ns); + pauseInternal(ns); + return true; } - return true; } void AudioTrack::AudioTrackThread::requestExit() @@ -1529,10 +1855,19 @@ void AudioTrack::AudioTrackThread::pause() void AudioTrack::AudioTrackThread::resume() { AutoMutex _l(mMyLock); - if (mPaused) { + mIgnoreNextPausedInt = true; + if (mPaused || mPausedInt) { mPaused = false; + mPausedInt = false; mMyCond.signal(); } } +void AudioTrack::AudioTrackThread::pauseInternal(nsecs_t ns) +{ + AutoMutex _l(mMyLock); + mPausedInt = true; + mPausedNs = ns; +} + }; // namespace android diff --git a/media/libmedia/AudioTrackShared.cpp b/media/libmedia/AudioTrackShared.cpp index 13d47c9..da73d65 100644 --- a/media/libmedia/AudioTrackShared.cpp +++ b/media/libmedia/AudioTrackShared.cpp @@ -19,178 +19,841 @@ #include <private/media/AudioTrackShared.h> #include <utils/Log.h> +extern "C" { +#include "../private/bionic_futex.h" +} namespace android { audio_track_cblk_t::audio_track_cblk_t() - : lock(Mutex::SHARED), cv(Condition::SHARED), user(0), server(0), - userBase(0), serverBase(0), frameCount_(0), - loopStart(UINT_MAX), loopEnd(UINT_MAX), loopCount(0), mVolumeLR(0x10001000), - mSampleRate(0), mSendLevel(0), flags(0) + : mServer(0), frameCount_(0), mFutex(0), mMinimum(0), + mVolumeLR(0x10001000), mSampleRate(0), mSendLevel(0), mFlags(0) +{ + memset(&u, 0, sizeof(u)); +} + +// --------------------------------------------------------------------------- + +Proxy::Proxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, size_t frameSize, + bool isOut, bool clientInServer) + : mCblk(cblk), mBuffers(buffers), mFrameCount(frameCount), mFrameSize(frameSize), + mFrameCountP2(roundup(frameCount)), mIsOut(isOut), mClientInServer(clientInServer), + mIsShutdown(false), mUnreleased(0) { } -uint32_t audio_track_cblk_t::stepUser(size_t stepCount, size_t frameCount, bool isOut) +// --------------------------------------------------------------------------- + +ClientProxy::ClientProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, + size_t frameSize, bool isOut, bool clientInServer) + : Proxy(cblk, buffers, frameCount, frameSize, isOut, clientInServer), mEpoch(0) +{ +} + +const struct timespec ClientProxy::kForever = {INT_MAX /*tv_sec*/, 0 /*tv_nsec*/}; +const struct timespec ClientProxy::kNonBlocking = {0 /*tv_sec*/, 0 /*tv_nsec*/}; + +#define MEASURE_NS 10000000 // attempt to provide accurate timeouts if requested >= MEASURE_NS + +// To facilitate quicker recovery from server failure, this value limits the timeout per each futex +// wait. However it does not protect infinite timeouts. If defined to be zero, there is no limit. +// FIXME May not be compatible with audio tunneling requirements where timeout should be in the +// order of minutes. +#define MAX_SEC 5 + +status_t ClientProxy::obtainBuffer(Buffer* buffer, const struct timespec *requested, + struct timespec *elapsed) { - ALOGV("stepuser %08x %08x %d", user, server, stepCount); + LOG_ALWAYS_FATAL_IF(buffer == NULL || buffer->mFrameCount == 0); + struct timespec total; // total elapsed time spent waiting + total.tv_sec = 0; + total.tv_nsec = 0; + bool measure = elapsed != NULL; // whether to measure total elapsed time spent waiting - uint32_t u = user; - u += stepCount; - // Ensure that user is never ahead of server for AudioRecord - if (isOut) { - // If stepServer() has been called once, switch to normal obtainBuffer() timeout period - if (bufferTimeoutMs == MAX_STARTUP_TIMEOUT_MS-1) { - bufferTimeoutMs = MAX_RUN_TIMEOUT_MS; + status_t status; + enum { + TIMEOUT_ZERO, // requested == NULL || *requested == 0 + TIMEOUT_INFINITE, // *requested == infinity + TIMEOUT_FINITE, // 0 < *requested < infinity + TIMEOUT_CONTINUE, // additional chances after TIMEOUT_FINITE + } timeout; + if (requested == NULL) { + timeout = TIMEOUT_ZERO; + } else if (requested->tv_sec == 0 && requested->tv_nsec == 0) { + timeout = TIMEOUT_ZERO; + } else if (requested->tv_sec == INT_MAX) { + timeout = TIMEOUT_INFINITE; + } else { + timeout = TIMEOUT_FINITE; + if (requested->tv_sec > 0 || requested->tv_nsec >= MEASURE_NS) { + measure = true; } - } else if (u > server) { - ALOGW("stepUser occurred after track reset"); - u = server; } - - if (u >= frameCount) { - // common case, user didn't just wrap - if (u - frameCount >= userBase ) { - userBase += frameCount; + struct timespec before; + bool beforeIsValid = false; + audio_track_cblk_t* cblk = mCblk; + bool ignoreInitialPendingInterrupt = true; + // check for shared memory corruption + if (mIsShutdown) { + status = NO_INIT; + goto end; + } + for (;;) { + int32_t flags = android_atomic_and(~CBLK_INTERRUPT, &cblk->mFlags); + // check for track invalidation by server, or server death detection + if (flags & CBLK_INVALID) { + ALOGV("Track invalidated"); + status = DEAD_OBJECT; + goto end; + } + // check for obtainBuffer interrupted by client + if (!ignoreInitialPendingInterrupt && (flags & CBLK_INTERRUPT)) { + ALOGV("obtainBuffer() interrupted by client"); + status = -EINTR; + goto end; + } + ignoreInitialPendingInterrupt = false; + // compute number of frames available to write (AudioTrack) or read (AudioRecord) + int32_t front; + int32_t rear; + if (mIsOut) { + // The barrier following the read of mFront is probably redundant. + // We're about to perform a conditional branch based on 'filled', + // which will force the processor to observe the read of mFront + // prior to allowing data writes starting at mRaw. + // However, the processor may support speculative execution, + // and be unable to undo speculative writes into shared memory. + // The barrier will prevent such speculative execution. + front = android_atomic_acquire_load(&cblk->u.mStreaming.mFront); + rear = cblk->u.mStreaming.mRear; + } else { + // On the other hand, this barrier is required. + rear = android_atomic_acquire_load(&cblk->u.mStreaming.mRear); + front = cblk->u.mStreaming.mFront; + } + ssize_t filled = rear - front; + // pipe should not be overfull + if (!(0 <= filled && (size_t) filled <= mFrameCount)) { + ALOGE("Shared memory control block is corrupt (filled=%d); shutting down", filled); + mIsShutdown = true; + status = NO_INIT; + goto end; + } + // don't allow filling pipe beyond the nominal size + size_t avail = mIsOut ? mFrameCount - filled : filled; + if (avail > 0) { + // 'avail' may be non-contiguous, so return only the first contiguous chunk + size_t part1; + if (mIsOut) { + rear &= mFrameCountP2 - 1; + part1 = mFrameCountP2 - rear; + } else { + front &= mFrameCountP2 - 1; + part1 = mFrameCountP2 - front; + } + if (part1 > avail) { + part1 = avail; + } + if (part1 > buffer->mFrameCount) { + part1 = buffer->mFrameCount; + } + buffer->mFrameCount = part1; + buffer->mRaw = part1 > 0 ? + &((char *) mBuffers)[(mIsOut ? rear : front) * mFrameSize] : NULL; + buffer->mNonContig = avail - part1; + mUnreleased = part1; + status = NO_ERROR; + break; + } + struct timespec remaining; + const struct timespec *ts; + switch (timeout) { + case TIMEOUT_ZERO: + status = WOULD_BLOCK; + goto end; + case TIMEOUT_INFINITE: + ts = NULL; + break; + case TIMEOUT_FINITE: + timeout = TIMEOUT_CONTINUE; + if (MAX_SEC == 0) { + ts = requested; + break; + } + // fall through + case TIMEOUT_CONTINUE: + // FIXME we do not retry if requested < 10ms? needs documentation on this state machine + if (!measure || requested->tv_sec < total.tv_sec || + (requested->tv_sec == total.tv_sec && requested->tv_nsec <= total.tv_nsec)) { + status = TIMED_OUT; + goto end; + } + remaining.tv_sec = requested->tv_sec - total.tv_sec; + if ((remaining.tv_nsec = requested->tv_nsec - total.tv_nsec) < 0) { + remaining.tv_nsec += 1000000000; + remaining.tv_sec++; + } + if (0 < MAX_SEC && MAX_SEC < remaining.tv_sec) { + remaining.tv_sec = MAX_SEC; + remaining.tv_nsec = 0; + } + ts = &remaining; + break; + default: + LOG_FATAL("obtainBuffer() timeout=%d", timeout); + ts = NULL; + break; + } + int32_t old = android_atomic_and(~CBLK_FUTEX_WAKE, &cblk->mFutex); + if (!(old & CBLK_FUTEX_WAKE)) { + int rc; + if (measure && !beforeIsValid) { + clock_gettime(CLOCK_MONOTONIC, &before); + beforeIsValid = true; + } + int ret = __futex_syscall4(&cblk->mFutex, + mClientInServer ? FUTEX_WAIT_PRIVATE : FUTEX_WAIT, old & ~CBLK_FUTEX_WAKE, ts); + // update total elapsed time spent waiting + if (measure) { + struct timespec after; + clock_gettime(CLOCK_MONOTONIC, &after); + total.tv_sec += after.tv_sec - before.tv_sec; + long deltaNs = after.tv_nsec - before.tv_nsec; + if (deltaNs < 0) { + deltaNs += 1000000000; + total.tv_sec--; + } + if ((total.tv_nsec += deltaNs) >= 1000000000) { + total.tv_nsec -= 1000000000; + total.tv_sec++; + } + before = after; + beforeIsValid = true; + } + switch (ret) { + case 0: // normal wakeup by server, or by binderDied() + case -EWOULDBLOCK: // benign race condition with server + case -EINTR: // wait was interrupted by signal or other spurious wakeup + case -ETIMEDOUT: // time-out expired + // FIXME these error/non-0 status are being dropped + break; + default: + ALOGE("%s unexpected error %d", __func__, ret); + status = -ret; + goto end; + } } - } else if (u >= userBase + frameCount) { - // user just wrapped - userBase += frameCount; } - user = u; +end: + if (status != NO_ERROR) { + buffer->mFrameCount = 0; + buffer->mRaw = NULL; + buffer->mNonContig = 0; + mUnreleased = 0; + } + if (elapsed != NULL) { + *elapsed = total; + } + if (requested == NULL) { + requested = &kNonBlocking; + } + if (measure) { + ALOGV("requested %ld.%03ld elapsed %ld.%03ld", + requested->tv_sec, requested->tv_nsec / 1000000, + total.tv_sec, total.tv_nsec / 1000000); + } + return status; +} + +void ClientProxy::releaseBuffer(Buffer* buffer) +{ + LOG_ALWAYS_FATAL_IF(buffer == NULL); + size_t stepCount = buffer->mFrameCount; + if (stepCount == 0 || mIsShutdown) { + // prevent accidental re-use of buffer + buffer->mFrameCount = 0; + buffer->mRaw = NULL; + buffer->mNonContig = 0; + return; + } + LOG_ALWAYS_FATAL_IF(!(stepCount <= mUnreleased && mUnreleased <= mFrameCount)); + mUnreleased -= stepCount; + audio_track_cblk_t* cblk = mCblk; + // Both of these barriers are required + if (mIsOut) { + int32_t rear = cblk->u.mStreaming.mRear; + android_atomic_release_store(stepCount + rear, &cblk->u.mStreaming.mRear); + } else { + int32_t front = cblk->u.mStreaming.mFront; + android_atomic_release_store(stepCount + front, &cblk->u.mStreaming.mFront); + } +} - // Clear flow control error condition as new data has been written/read to/from buffer. - if (flags & CBLK_UNDERRUN) { - android_atomic_and(~CBLK_UNDERRUN, &flags); +void ClientProxy::binderDied() +{ + audio_track_cblk_t* cblk = mCblk; + if (!(android_atomic_or(CBLK_INVALID, &cblk->mFlags) & CBLK_INVALID)) { + // it seems that a FUTEX_WAKE_PRIVATE will not wake a FUTEX_WAIT, even within same process + (void) __futex_syscall3(&cblk->mFutex, mClientInServer ? FUTEX_WAKE_PRIVATE : FUTEX_WAKE, + 1); } +} - return u; +void ClientProxy::interrupt() +{ + audio_track_cblk_t* cblk = mCblk; + if (!(android_atomic_or(CBLK_INTERRUPT, &cblk->mFlags) & CBLK_INTERRUPT)) { + (void) __futex_syscall3(&cblk->mFutex, mClientInServer ? FUTEX_WAKE_PRIVATE : FUTEX_WAKE, + 1); + } } -bool audio_track_cblk_t::stepServer(size_t stepCount, size_t frameCount, bool isOut) +size_t ClientProxy::getMisalignment() { - ALOGV("stepserver %08x %08x %d", user, server, stepCount); + audio_track_cblk_t* cblk = mCblk; + return (mFrameCountP2 - (mIsOut ? cblk->u.mStreaming.mRear : cblk->u.mStreaming.mFront)) & + (mFrameCountP2 - 1); +} + +size_t ClientProxy::getFramesFilled() { + audio_track_cblk_t* cblk = mCblk; + int32_t front; + int32_t rear; - if (!tryLock()) { - ALOGW("stepServer() could not lock cblk"); - return false; + if (mIsOut) { + front = android_atomic_acquire_load(&cblk->u.mStreaming.mFront); + rear = cblk->u.mStreaming.mRear; + } else { + rear = android_atomic_acquire_load(&cblk->u.mStreaming.mRear); + front = cblk->u.mStreaming.mFront; + } + ssize_t filled = rear - front; + // pipe should not be overfull + if (!(0 <= filled && (size_t) filled <= mFrameCount)) { + ALOGE("Shared memory control block is corrupt (filled=%d); shutting down", filled); + return 0; } + return (size_t)filled; +} + +// --------------------------------------------------------------------------- + +void AudioTrackClientProxy::flush() +{ + mCblk->u.mStreaming.mFlush++; +} - uint32_t s = server; - bool flushed = (s == user); +bool AudioTrackClientProxy::clearStreamEndDone() { + return (android_atomic_and(~CBLK_STREAM_END_DONE, &mCblk->mFlags) & CBLK_STREAM_END_DONE) != 0; +} + +bool AudioTrackClientProxy::getStreamEndDone() const { + return (mCblk->mFlags & CBLK_STREAM_END_DONE) != 0; +} - s += stepCount; - if (isOut) { - // Mark that we have read the first buffer so that next time stepUser() is called - // we switch to normal obtainBuffer() timeout period - if (bufferTimeoutMs == MAX_STARTUP_TIMEOUT_MS) { - bufferTimeoutMs = MAX_STARTUP_TIMEOUT_MS - 1; +status_t AudioTrackClientProxy::waitStreamEndDone(const struct timespec *requested) +{ + struct timespec total; // total elapsed time spent waiting + total.tv_sec = 0; + total.tv_nsec = 0; + audio_track_cblk_t* cblk = mCblk; + status_t status; + enum { + TIMEOUT_ZERO, // requested == NULL || *requested == 0 + TIMEOUT_INFINITE, // *requested == infinity + TIMEOUT_FINITE, // 0 < *requested < infinity + TIMEOUT_CONTINUE, // additional chances after TIMEOUT_FINITE + } timeout; + if (requested == NULL) { + timeout = TIMEOUT_ZERO; + } else if (requested->tv_sec == 0 && requested->tv_nsec == 0) { + timeout = TIMEOUT_ZERO; + } else if (requested->tv_sec == INT_MAX) { + timeout = TIMEOUT_INFINITE; + } else { + timeout = TIMEOUT_FINITE; + } + for (;;) { + int32_t flags = android_atomic_and(~(CBLK_INTERRUPT|CBLK_STREAM_END_DONE), &cblk->mFlags); + // check for track invalidation by server, or server death detection + if (flags & CBLK_INVALID) { + ALOGV("Track invalidated"); + status = DEAD_OBJECT; + goto end; + } + if (flags & CBLK_STREAM_END_DONE) { + ALOGV("stream end received"); + status = NO_ERROR; + goto end; + } + // check for obtainBuffer interrupted by client + // check for obtainBuffer interrupted by client + if (flags & CBLK_INTERRUPT) { + ALOGV("waitStreamEndDone() interrupted by client"); + status = -EINTR; + goto end; } - // It is possible that we receive a flush() - // while the mixer is processing a block: in this case, - // stepServer() is called After the flush() has reset u & s and - // we have s > u - if (flushed) { - ALOGW("stepServer occurred after track reset"); - s = user; + struct timespec remaining; + const struct timespec *ts; + switch (timeout) { + case TIMEOUT_ZERO: + status = WOULD_BLOCK; + goto end; + case TIMEOUT_INFINITE: + ts = NULL; + break; + case TIMEOUT_FINITE: + timeout = TIMEOUT_CONTINUE; + if (MAX_SEC == 0) { + ts = requested; + break; + } + // fall through + case TIMEOUT_CONTINUE: + // FIXME we do not retry if requested < 10ms? needs documentation on this state machine + if (requested->tv_sec < total.tv_sec || + (requested->tv_sec == total.tv_sec && requested->tv_nsec <= total.tv_nsec)) { + status = TIMED_OUT; + goto end; + } + remaining.tv_sec = requested->tv_sec - total.tv_sec; + if ((remaining.tv_nsec = requested->tv_nsec - total.tv_nsec) < 0) { + remaining.tv_nsec += 1000000000; + remaining.tv_sec++; + } + if (0 < MAX_SEC && MAX_SEC < remaining.tv_sec) { + remaining.tv_sec = MAX_SEC; + remaining.tv_nsec = 0; + } + ts = &remaining; + break; + default: + LOG_FATAL("waitStreamEndDone() timeout=%d", timeout); + ts = NULL; + break; } + int32_t old = android_atomic_and(~CBLK_FUTEX_WAKE, &cblk->mFutex); + if (!(old & CBLK_FUTEX_WAKE)) { + int rc; + int ret = __futex_syscall4(&cblk->mFutex, + mClientInServer ? FUTEX_WAIT_PRIVATE : FUTEX_WAIT, old & ~CBLK_FUTEX_WAKE, ts); + switch (ret) { + case 0: // normal wakeup by server, or by binderDied() + case -EWOULDBLOCK: // benign race condition with server + case -EINTR: // wait was interrupted by signal or other spurious wakeup + case -ETIMEDOUT: // time-out expired + break; + default: + ALOGE("%s unexpected error %d", __func__, ret); + status = -ret; + goto end; + } + } + } + +end: + if (requested == NULL) { + requested = &kNonBlocking; } + return status; +} + +// --------------------------------------------------------------------------- - if (s >= loopEnd) { - ALOGW_IF(s > loopEnd, "stepServer: s %u > loopEnd %u", s, loopEnd); - s = loopStart; - if (--loopCount == 0) { - loopEnd = UINT_MAX; - loopStart = UINT_MAX; +StaticAudioTrackClientProxy::StaticAudioTrackClientProxy(audio_track_cblk_t* cblk, void *buffers, + size_t frameCount, size_t frameSize) + : AudioTrackClientProxy(cblk, buffers, frameCount, frameSize), + mMutator(&cblk->u.mStatic.mSingleStateQueue), mBufferPosition(0) +{ +} + +void StaticAudioTrackClientProxy::flush() +{ + LOG_FATAL("static flush"); +} + +void StaticAudioTrackClientProxy::setLoop(size_t loopStart, size_t loopEnd, int loopCount) +{ + StaticAudioTrackState newState; + newState.mLoopStart = loopStart; + newState.mLoopEnd = loopEnd; + newState.mLoopCount = loopCount; + mBufferPosition = loopStart; + (void) mMutator.push(newState); +} + +size_t StaticAudioTrackClientProxy::getBufferPosition() +{ + size_t bufferPosition; + if (mMutator.ack()) { + bufferPosition = mCblk->u.mStatic.mBufferPosition; + if (bufferPosition > mFrameCount) { + bufferPosition = mFrameCount; } + } else { + bufferPosition = mBufferPosition; } + return bufferPosition; +} + +// --------------------------------------------------------------------------- + +ServerProxy::ServerProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, + size_t frameSize, bool isOut, bool clientInServer) + : Proxy(cblk, buffers, frameCount, frameSize, isOut, clientInServer), + mAvailToClient(0), mFlush(0) +{ +} - if (s >= frameCount) { - // common case, server didn't just wrap - if (s - frameCount >= serverBase ) { - serverBase += frameCount; +status_t ServerProxy::obtainBuffer(Buffer* buffer) +{ + LOG_ALWAYS_FATAL_IF(buffer == NULL || buffer->mFrameCount == 0); + if (mIsShutdown) { + goto no_init; + } + { + audio_track_cblk_t* cblk = mCblk; + // compute number of frames available to write (AudioTrack) or read (AudioRecord), + // or use previous cached value from framesReady(), with added barrier if it omits. + int32_t front; + int32_t rear; + // See notes on barriers at ClientProxy::obtainBuffer() + if (mIsOut) { + int32_t flush = cblk->u.mStreaming.mFlush; + rear = android_atomic_acquire_load(&cblk->u.mStreaming.mRear); + front = cblk->u.mStreaming.mFront; + if (flush != mFlush) { + mFlush = flush; + // effectively obtain then release whatever is in the buffer + android_atomic_release_store(rear, &cblk->u.mStreaming.mFront); + if (front != rear) { + int32_t old = android_atomic_or(CBLK_FUTEX_WAKE, &cblk->mFutex); + if (!(old & CBLK_FUTEX_WAKE)) { + (void) __futex_syscall3(&cblk->mFutex, + mClientInServer ? FUTEX_WAKE_PRIVATE : FUTEX_WAKE, 1); + } + } + front = rear; } - } else if (s >= serverBase + frameCount) { - // server just wrapped - serverBase += frameCount; + } else { + front = android_atomic_acquire_load(&cblk->u.mStreaming.mFront); + rear = cblk->u.mStreaming.mRear; + } + ssize_t filled = rear - front; + // pipe should not already be overfull + if (!(0 <= filled && (size_t) filled <= mFrameCount)) { + ALOGE("Shared memory control block is corrupt (filled=%d); shutting down", filled); + mIsShutdown = true; + } + if (mIsShutdown) { + goto no_init; + } + // don't allow filling pipe beyond the nominal size + size_t availToServer; + if (mIsOut) { + availToServer = filled; + mAvailToClient = mFrameCount - filled; + } else { + availToServer = mFrameCount - filled; + mAvailToClient = filled; } + // 'availToServer' may be non-contiguous, so return only the first contiguous chunk + size_t part1; + if (mIsOut) { + front &= mFrameCountP2 - 1; + part1 = mFrameCountP2 - front; + } else { + rear &= mFrameCountP2 - 1; + part1 = mFrameCountP2 - rear; + } + if (part1 > availToServer) { + part1 = availToServer; + } + size_t ask = buffer->mFrameCount; + if (part1 > ask) { + part1 = ask; + } + // is assignment redundant in some cases? + buffer->mFrameCount = part1; + buffer->mRaw = part1 > 0 ? + &((char *) mBuffers)[(mIsOut ? front : rear) * mFrameSize] : NULL; + buffer->mNonContig = availToServer - part1; + mUnreleased = part1; + return part1 > 0 ? NO_ERROR : WOULD_BLOCK; + } +no_init: + buffer->mFrameCount = 0; + buffer->mRaw = NULL; + buffer->mNonContig = 0; + mUnreleased = 0; + return NO_INIT; +} - server = s; +void ServerProxy::releaseBuffer(Buffer* buffer) +{ + LOG_ALWAYS_FATAL_IF(buffer == NULL); + size_t stepCount = buffer->mFrameCount; + if (stepCount == 0 || mIsShutdown) { + // prevent accidental re-use of buffer + buffer->mFrameCount = 0; + buffer->mRaw = NULL; + buffer->mNonContig = 0; + return; + } + LOG_ALWAYS_FATAL_IF(!(stepCount <= mUnreleased && mUnreleased <= mFrameCount)); + mUnreleased -= stepCount; + audio_track_cblk_t* cblk = mCblk; + if (mIsOut) { + int32_t front = cblk->u.mStreaming.mFront; + android_atomic_release_store(stepCount + front, &cblk->u.mStreaming.mFront); + } else { + int32_t rear = cblk->u.mStreaming.mRear; + android_atomic_release_store(stepCount + rear, &cblk->u.mStreaming.mRear); + } - if (!(flags & CBLK_INVALID)) { - cv.signal(); + mCblk->mServer += stepCount; + + size_t half = mFrameCount / 2; + if (half == 0) { + half = 1; + } + size_t minimum = cblk->mMinimum; + if (minimum == 0) { + minimum = mIsOut ? half : 1; + } else if (minimum > half) { + minimum = half; } - lock.unlock(); - return true; + // FIXME AudioRecord wakeup needs to be optimized; it currently wakes up client every time + if (!mIsOut || (mAvailToClient + stepCount >= minimum)) { + ALOGV("mAvailToClient=%u stepCount=%u minimum=%u", mAvailToClient, stepCount, minimum); + int32_t old = android_atomic_or(CBLK_FUTEX_WAKE, &cblk->mFutex); + if (!(old & CBLK_FUTEX_WAKE)) { + (void) __futex_syscall3(&cblk->mFutex, + mClientInServer ? FUTEX_WAKE_PRIVATE : FUTEX_WAKE, 1); + } + } + + buffer->mFrameCount = 0; + buffer->mRaw = NULL; + buffer->mNonContig = 0; } -void* audio_track_cblk_t::buffer(void *buffers, size_t frameSize, uint32_t offset) const +// --------------------------------------------------------------------------- + +size_t AudioTrackServerProxy::framesReady() { - return (int8_t *)buffers + (offset - userBase) * frameSize; + LOG_ALWAYS_FATAL_IF(!mIsOut); + + if (mIsShutdown) { + return 0; + } + audio_track_cblk_t* cblk = mCblk; + + int32_t flush = cblk->u.mStreaming.mFlush; + if (flush != mFlush) { + return mFrameCount; + } + // the acquire might not be necessary since not doing a subsequent read + int32_t rear = android_atomic_acquire_load(&cblk->u.mStreaming.mRear); + ssize_t filled = rear - cblk->u.mStreaming.mFront; + // pipe should not already be overfull + if (!(0 <= filled && (size_t) filled <= mFrameCount)) { + ALOGE("Shared memory control block is corrupt (filled=%d); shutting down", filled); + mIsShutdown = true; + return 0; + } + // cache this value for later use by obtainBuffer(), with added barrier + // and racy if called by normal mixer thread + // ignores flush(), so framesReady() may report a larger mFrameCount than obtainBuffer() + return filled; +} + +bool AudioTrackServerProxy::setStreamEndDone() { + bool old = + (android_atomic_or(CBLK_STREAM_END_DONE, &mCblk->mFlags) & CBLK_STREAM_END_DONE) != 0; + if (!old) { + (void) __futex_syscall3(&mCblk->mFutex, mClientInServer ? FUTEX_WAKE_PRIVATE : FUTEX_WAKE, + 1); + } + return old; } -uint32_t audio_track_cblk_t::framesAvailable(size_t frameCount, bool isOut) +void AudioTrackServerProxy::tallyUnderrunFrames(uint32_t frameCount) { - Mutex::Autolock _l(lock); - return framesAvailable_l(frameCount, isOut); + mCblk->u.mStreaming.mUnderrunFrames += frameCount; + + // FIXME also wake futex so that underrun is noticed more quickly + (void) android_atomic_or(CBLK_UNDERRUN, &mCblk->mFlags); } -uint32_t audio_track_cblk_t::framesAvailable_l(size_t frameCount, bool isOut) +// --------------------------------------------------------------------------- + +StaticAudioTrackServerProxy::StaticAudioTrackServerProxy(audio_track_cblk_t* cblk, void *buffers, + size_t frameCount, size_t frameSize) + : AudioTrackServerProxy(cblk, buffers, frameCount, frameSize), + mObserver(&cblk->u.mStatic.mSingleStateQueue), mPosition(0), + mEnd(frameCount), mFramesReadyIsCalledByMultipleThreads(false) { - uint32_t u = user; - uint32_t s = server; + mState.mLoopStart = 0; + mState.mLoopEnd = 0; + mState.mLoopCount = 0; +} - if (isOut) { - uint32_t limit = (s < loopStart) ? s : loopStart; - return limit + frameCount - u; - } else { - return frameCount + u - s; - } +void StaticAudioTrackServerProxy::framesReadyIsCalledByMultipleThreads() +{ + mFramesReadyIsCalledByMultipleThreads = true; } -uint32_t audio_track_cblk_t::framesReady(bool isOut) +size_t StaticAudioTrackServerProxy::framesReady() { - uint32_t u = user; - uint32_t s = server; + // FIXME + // This is racy if called by normal mixer thread, + // as we're reading 2 independent variables without a lock. + // Can't call mObserver.poll(), as we might be called from wrong thread. + // If looping is enabled, should return a higher number (since includes non-contiguous). + size_t position = mPosition; + if (!mFramesReadyIsCalledByMultipleThreads) { + ssize_t positionOrStatus = pollPosition(); + if (positionOrStatus >= 0) { + position = (size_t) positionOrStatus; + } + } + size_t end = mEnd; + return position < end ? end - position : 0; +} - if (isOut) { - if (u < loopEnd) { - return u - s; - } else { - // do not block on mutex shared with client on AudioFlinger side - if (!tryLock()) { - ALOGW("framesReady() could not lock cblk"); - return 0; +ssize_t StaticAudioTrackServerProxy::pollPosition() +{ + size_t position = mPosition; + StaticAudioTrackState state; + if (mObserver.poll(state)) { + bool valid = false; + size_t loopStart = state.mLoopStart; + size_t loopEnd = state.mLoopEnd; + if (state.mLoopCount == 0) { + if (loopStart > mFrameCount) { + loopStart = mFrameCount; } - uint32_t frames = UINT_MAX; - if (loopCount >= 0) { - frames = (loopEnd - loopStart)*loopCount + u - s; + // ignore loopEnd + mPosition = position = loopStart; + mEnd = mFrameCount; + mState.mLoopCount = 0; + valid = true; + } else { + if (loopStart < loopEnd && loopEnd <= mFrameCount && + loopEnd - loopStart >= MIN_LOOP) { + if (!(loopStart <= position && position < loopEnd)) { + mPosition = position = loopStart; + } + mEnd = loopEnd; + mState = state; + valid = true; } - lock.unlock(); - return frames; } - } else { - return s - u; + if (!valid) { + ALOGE("%s client pushed an invalid state, shutting down", __func__); + mIsShutdown = true; + return (ssize_t) NO_INIT; + } + mCblk->u.mStatic.mBufferPosition = position; } + return (ssize_t) position; } -bool audio_track_cblk_t::tryLock() +status_t StaticAudioTrackServerProxy::obtainBuffer(Buffer* buffer) { - // the code below simulates lock-with-timeout - // we MUST do this to protect the AudioFlinger server - // as this lock is shared with the client. - status_t err; + if (mIsShutdown) { + buffer->mFrameCount = 0; + buffer->mRaw = NULL; + buffer->mNonContig = 0; + mUnreleased = 0; + return NO_INIT; + } + ssize_t positionOrStatus = pollPosition(); + if (positionOrStatus < 0) { + buffer->mFrameCount = 0; + buffer->mRaw = NULL; + buffer->mNonContig = 0; + mUnreleased = 0; + return (status_t) positionOrStatus; + } + size_t position = (size_t) positionOrStatus; + size_t avail; + if (position < mEnd) { + avail = mEnd - position; + size_t wanted = buffer->mFrameCount; + if (avail < wanted) { + buffer->mFrameCount = avail; + } else { + avail = wanted; + } + buffer->mRaw = &((char *) mBuffers)[position * mFrameSize]; + } else { + avail = 0; + buffer->mFrameCount = 0; + buffer->mRaw = NULL; + } + buffer->mNonContig = 0; // FIXME should be > 0 for looping + mUnreleased = avail; + return NO_ERROR; +} - err = lock.tryLock(); - if (err == -EBUSY) { // just wait a bit - usleep(1000); - err = lock.tryLock(); +void StaticAudioTrackServerProxy::releaseBuffer(Buffer* buffer) +{ + size_t stepCount = buffer->mFrameCount; + LOG_ALWAYS_FATAL_IF(!(stepCount <= mUnreleased)); + if (stepCount == 0) { + // prevent accidental re-use of buffer + buffer->mRaw = NULL; + buffer->mNonContig = 0; + return; } - if (err != NO_ERROR) { - // probably, the client just died. - return false; + mUnreleased -= stepCount; + audio_track_cblk_t* cblk = mCblk; + size_t position = mPosition; + size_t newPosition = position + stepCount; + int32_t setFlags = 0; + if (!(position <= newPosition && newPosition <= mFrameCount)) { + ALOGW("%s newPosition %u outside [%u, %u]", __func__, newPosition, position, mFrameCount); + newPosition = mFrameCount; + } else if (mState.mLoopCount != 0 && newPosition == mState.mLoopEnd) { + if (mState.mLoopCount == -1 || --mState.mLoopCount != 0) { + newPosition = mState.mLoopStart; + setFlags = CBLK_LOOP_CYCLE; + } else { + mEnd = mFrameCount; // this is what allows playback to continue after the loop + setFlags = CBLK_LOOP_FINAL; + } } - return true; + if (newPosition == mFrameCount) { + setFlags |= CBLK_BUFFER_END; + } + mPosition = newPosition; + + cblk->mServer += stepCount; + cblk->u.mStatic.mBufferPosition = newPosition; + if (setFlags != 0) { + (void) android_atomic_or(setFlags, &cblk->mFlags); + // this would be a good place to wake a futex + } + + buffer->mFrameCount = 0; + buffer->mRaw = NULL; + buffer->mNonContig = 0; } +void StaticAudioTrackServerProxy::tallyUnderrunFrames(uint32_t frameCount) +{ + // Unlike AudioTrackServerProxy::tallyUnderrunFrames() used for streaming tracks, + // we don't have a location to count underrun frames. The underrun frame counter + // only exists in AudioTrackSharedStreaming. Fortunately, underruns are not + // possible for static buffer tracks other than at end of buffer, so this is not a loss. + + // FIXME also wake futex so that underrun is noticed more quickly + (void) android_atomic_or(CBLK_UNDERRUN, &mCblk->mFlags); +} + +// --------------------------------------------------------------------------- + } // namespace android diff --git a/media/libmedia/IAudioFlinger.cpp b/media/libmedia/IAudioFlinger.cpp index 2f18680..448a82e 100644 --- a/media/libmedia/IAudioFlinger.cpp +++ b/media/libmedia/IAudioFlinger.cpp @@ -73,6 +73,7 @@ enum { LOAD_HW_MODULE, GET_PRIMARY_OUTPUT_SAMPLING_RATE, GET_PRIMARY_OUTPUT_FRAME_COUNT, + SET_LOW_RAM_DEVICE, }; class BpAudioFlinger : public BpInterface<IAudioFlinger> @@ -94,6 +95,7 @@ public: audio_io_handle_t output, pid_t tid, int *sessionId, + String8& name, status_t *status) { Parcel data, reply; @@ -106,7 +108,12 @@ public: data.writeInt32(frameCount); track_flags_t lFlags = flags != NULL ? *flags : (track_flags_t) TRACK_DEFAULT; data.writeInt32(lFlags); - data.writeStrongBinder(sharedBuffer->asBinder()); + if (sharedBuffer != 0) { + data.writeInt32(true); + data.writeStrongBinder(sharedBuffer->asBinder()); + } else { + data.writeInt32(false); + } data.writeInt32((int32_t) output); data.writeInt32((int32_t) tid); int lSessionId = 0; @@ -126,6 +133,7 @@ public: if (sessionId != NULL) { *sessionId = lSessionId; } + name = reply.readString8(); lStatus = reply.readInt32(); track = interface_cast<IAudioTrack>(reply.readStrongBinder()); } @@ -141,7 +149,7 @@ public: audio_format_t format, audio_channel_mask_t channelMask, size_t frameCount, - track_flags_t flags, + track_flags_t *flags, pid_t tid, int *sessionId, status_t *status) @@ -154,7 +162,8 @@ public: data.writeInt32(format); data.writeInt32(channelMask); data.writeInt32(frameCount); - data.writeInt32(flags); + track_flags_t lFlags = flags != NULL ? *flags : (track_flags_t) TRACK_DEFAULT; + data.writeInt32(lFlags); data.writeInt32((int32_t) tid); int lSessionId = 0; if (sessionId != NULL) { @@ -165,12 +174,27 @@ public: if (lStatus != NO_ERROR) { ALOGE("openRecord error: %s", strerror(-lStatus)); } else { + lFlags = reply.readInt32(); + if (flags != NULL) { + *flags = lFlags; + } lSessionId = reply.readInt32(); if (sessionId != NULL) { *sessionId = lSessionId; } lStatus = reply.readInt32(); record = interface_cast<IAudioRecord>(reply.readStrongBinder()); + if (lStatus == NO_ERROR) { + if (record == 0) { + ALOGE("openRecord should have returned an IAudioRecord"); + lStatus = UNKNOWN_ERROR; + } + } else { + if (record != 0) { + ALOGE("openRecord returned an IAudioRecord but with status %d", lStatus); + record.clear(); + } + } } if (status) { *status = lStatus; @@ -361,15 +385,16 @@ public: audio_format_t *pFormat, audio_channel_mask_t *pChannelMask, uint32_t *pLatencyMs, - audio_output_flags_t flags) + audio_output_flags_t flags, + const audio_offload_info_t *offloadInfo) { Parcel data, reply; - audio_devices_t devices = pDevices ? *pDevices : (audio_devices_t)0; - uint32_t samplingRate = pSamplingRate ? *pSamplingRate : 0; - audio_format_t format = pFormat ? *pFormat : AUDIO_FORMAT_DEFAULT; - audio_channel_mask_t channelMask = pChannelMask ? *pChannelMask : (audio_channel_mask_t)0; - uint32_t latency = pLatencyMs ? *pLatencyMs : 0; - + audio_devices_t devices = pDevices != NULL ? *pDevices : (audio_devices_t)0; + uint32_t samplingRate = pSamplingRate != NULL ? *pSamplingRate : 0; + audio_format_t format = pFormat != NULL ? *pFormat : AUDIO_FORMAT_DEFAULT; + audio_channel_mask_t channelMask = pChannelMask != NULL ? + *pChannelMask : (audio_channel_mask_t)0; + uint32_t latency = pLatencyMs != NULL ? *pLatencyMs : 0; data.writeInterfaceToken(IAudioFlinger::getInterfaceDescriptor()); data.writeInt32(module); data.writeInt32(devices); @@ -378,19 +403,25 @@ public: data.writeInt32(channelMask); data.writeInt32(latency); data.writeInt32((int32_t) flags); + if (offloadInfo == NULL) { + data.writeInt32(0); + } else { + data.writeInt32(1); + data.write(offloadInfo, sizeof(audio_offload_info_t)); + } remote()->transact(OPEN_OUTPUT, data, &reply); audio_io_handle_t output = (audio_io_handle_t) reply.readInt32(); ALOGV("openOutput() returned output, %d", output); devices = (audio_devices_t)reply.readInt32(); - if (pDevices) *pDevices = devices; + if (pDevices != NULL) *pDevices = devices; samplingRate = reply.readInt32(); - if (pSamplingRate) *pSamplingRate = samplingRate; + if (pSamplingRate != NULL) *pSamplingRate = samplingRate; format = (audio_format_t) reply.readInt32(); - if (pFormat) *pFormat = format; + if (pFormat != NULL) *pFormat = format; channelMask = (audio_channel_mask_t)reply.readInt32(); - if (pChannelMask) *pChannelMask = channelMask; + if (pChannelMask != NULL) *pChannelMask = channelMask; latency = reply.readInt32(); - if (pLatencyMs) *pLatencyMs = latency; + if (pLatencyMs != NULL) *pLatencyMs = latency; return output; } @@ -439,10 +470,11 @@ public: audio_channel_mask_t *pChannelMask) { Parcel data, reply; - audio_devices_t devices = pDevices ? *pDevices : (audio_devices_t)0; - uint32_t samplingRate = pSamplingRate ? *pSamplingRate : 0; - audio_format_t format = pFormat ? *pFormat : AUDIO_FORMAT_DEFAULT; - audio_channel_mask_t channelMask = pChannelMask ? *pChannelMask : (audio_channel_mask_t)0; + audio_devices_t devices = pDevices != NULL ? *pDevices : (audio_devices_t)0; + uint32_t samplingRate = pSamplingRate != NULL ? *pSamplingRate : 0; + audio_format_t format = pFormat != NULL ? *pFormat : AUDIO_FORMAT_DEFAULT; + audio_channel_mask_t channelMask = pChannelMask != NULL ? + *pChannelMask : (audio_channel_mask_t)0; data.writeInterfaceToken(IAudioFlinger::getInterfaceDescriptor()); data.writeInt32(module); @@ -453,13 +485,13 @@ public: remote()->transact(OPEN_INPUT, data, &reply); audio_io_handle_t input = (audio_io_handle_t) reply.readInt32(); devices = (audio_devices_t)reply.readInt32(); - if (pDevices) *pDevices = devices; + if (pDevices != NULL) *pDevices = devices; samplingRate = reply.readInt32(); - if (pSamplingRate) *pSamplingRate = samplingRate; + if (pSamplingRate != NULL) *pSamplingRate = samplingRate; format = (audio_format_t) reply.readInt32(); - if (pFormat) *pFormat = format; + if (pFormat != NULL) *pFormat = format; channelMask = (audio_channel_mask_t)reply.readInt32(); - if (pChannelMask) *pChannelMask = channelMask; + if (pChannelMask != NULL) *pChannelMask = channelMask; return input; } @@ -695,6 +727,15 @@ public: return reply.readInt32(); } + virtual status_t setLowRamDevice(bool isLowRamDevice) + { + Parcel data, reply; + data.writeInterfaceToken(IAudioFlinger::getInterfaceDescriptor()); + data.writeInt32((int) isLowRamDevice); + remote()->transact(SET_LOW_RAM_DEVICE, data, &reply); + return reply.readInt32(); + } + }; IMPLEMENT_META_INTERFACE(AudioFlinger, "android.media.IAudioFlinger"); @@ -713,16 +754,30 @@ status_t BnAudioFlinger::onTransact( audio_channel_mask_t channelMask = data.readInt32(); size_t frameCount = data.readInt32(); track_flags_t flags = (track_flags_t) data.readInt32(); - sp<IMemory> buffer = interface_cast<IMemory>(data.readStrongBinder()); + bool haveSharedBuffer = data.readInt32() != 0; + sp<IMemory> buffer; + if (haveSharedBuffer) { + buffer = interface_cast<IMemory>(data.readStrongBinder()); + } audio_io_handle_t output = (audio_io_handle_t) data.readInt32(); pid_t tid = (pid_t) data.readInt32(); int sessionId = data.readInt32(); + String8 name; status_t status; - sp<IAudioTrack> track = createTrack( - (audio_stream_type_t) streamType, sampleRate, format, - channelMask, frameCount, &flags, buffer, output, tid, &sessionId, &status); + sp<IAudioTrack> track; + if ((haveSharedBuffer && (buffer == 0)) || + ((buffer != 0) && (buffer->pointer() == NULL))) { + ALOGW("CREATE_TRACK: cannot retrieve shared memory"); + status = DEAD_OBJECT; + } else { + track = createTrack( + (audio_stream_type_t) streamType, sampleRate, format, + channelMask, frameCount, &flags, buffer, output, tid, + &sessionId, name, &status); + } reply->writeInt32(flags); reply->writeInt32(sessionId); + reply->writeString8(name); reply->writeInt32(status); reply->writeStrongBinder(track->asBinder()); return NO_ERROR; @@ -739,7 +794,9 @@ status_t BnAudioFlinger::onTransact( int sessionId = data.readInt32(); status_t status; sp<IAudioRecord> record = openRecord(input, - sampleRate, format, channelMask, frameCount, flags, tid, &sessionId, &status); + sampleRate, format, channelMask, frameCount, &flags, tid, &sessionId, &status); + LOG_ALWAYS_FATAL_IF((record != 0) != (status == NO_ERROR)); + reply->writeInt32(flags); reply->writeInt32(sessionId); reply->writeInt32(status); reply->writeStrongBinder(record->asBinder()); @@ -868,13 +925,19 @@ status_t BnAudioFlinger::onTransact( audio_channel_mask_t channelMask = (audio_channel_mask_t)data.readInt32(); uint32_t latency = data.readInt32(); audio_output_flags_t flags = (audio_output_flags_t) data.readInt32(); + bool hasOffloadInfo = data.readInt32() != 0; + audio_offload_info_t offloadInfo; + if (hasOffloadInfo) { + data.read(&offloadInfo, sizeof(audio_offload_info_t)); + } audio_io_handle_t output = openOutput(module, &devices, &samplingRate, &format, &channelMask, &latency, - flags); + flags, + hasOffloadInfo ? &offloadInfo : NULL); ALOGV("OPEN_OUTPUT output, %p", output); reply->writeInt32((int32_t) output); reply->writeInt32(devices); @@ -1056,6 +1119,12 @@ status_t BnAudioFlinger::onTransact( reply->writeInt32(getPrimaryOutputFrameCount()); return NO_ERROR; } break; + case SET_LOW_RAM_DEVICE: { + CHECK_INTERFACE(IAudioFlinger, data, reply); + bool isLowRamDevice = data.readInt32() != 0; + reply->writeInt32(setLowRamDevice(isLowRamDevice)); + return NO_ERROR; + } break; default: return BBinder::onTransact(code, data, reply, flags); } diff --git a/media/libmedia/IAudioFlingerClient.cpp b/media/libmedia/IAudioFlingerClient.cpp index 2d1e0f8..3c0d4cf 100644 --- a/media/libmedia/IAudioFlingerClient.cpp +++ b/media/libmedia/IAudioFlingerClient.cpp @@ -54,7 +54,7 @@ public: (const AudioSystem::OutputDescriptor *)param2; data.writeInt32(desc->samplingRate); data.writeInt32(desc->format); - data.writeInt32(desc->channels); + data.writeInt32(desc->channelMask); data.writeInt32(desc->frameCount); data.writeInt32(desc->latency); } @@ -83,8 +83,8 @@ status_t BnAudioFlingerClient::onTransact( ALOGV("STREAM_CONFIG_CHANGED stream %d", stream); } else if (event != AudioSystem::OUTPUT_CLOSED && event != AudioSystem::INPUT_CLOSED) { desc.samplingRate = data.readInt32(); - desc.format = data.readInt32(); - desc.channels = data.readInt32(); + desc.format = (audio_format_t) data.readInt32(); + desc.channelMask = (audio_channel_mask_t) data.readInt32(); desc.frameCount = data.readInt32(); desc.latency = data.readInt32(); param2 = &desc; diff --git a/media/libmedia/IAudioPolicyService.cpp b/media/libmedia/IAudioPolicyService.cpp index 386c351..4be3c09 100644 --- a/media/libmedia/IAudioPolicyService.cpp +++ b/media/libmedia/IAudioPolicyService.cpp @@ -56,7 +56,8 @@ enum { GET_DEVICES_FOR_STREAM, QUERY_DEFAULT_PRE_PROCESSING, SET_EFFECT_ENABLED, - IS_STREAM_ACTIVE_REMOTELY + IS_STREAM_ACTIVE_REMOTELY, + IS_OFFLOAD_SUPPORTED }; class BpAudioPolicyService : public BpInterface<IAudioPolicyService> @@ -126,7 +127,8 @@ public: uint32_t samplingRate, audio_format_t format, audio_channel_mask_t channelMask, - audio_output_flags_t flags) + audio_output_flags_t flags, + const audio_offload_info_t *offloadInfo) { Parcel data, reply; data.writeInterfaceToken(IAudioPolicyService::getInterfaceDescriptor()); @@ -135,6 +137,12 @@ public: data.writeInt32(static_cast <uint32_t>(format)); data.writeInt32(channelMask); data.writeInt32(static_cast <uint32_t>(flags)); + if (offloadInfo == NULL) { + data.writeInt32(0); + } else { + data.writeInt32(1); + data.write(offloadInfo, sizeof(audio_offload_info_t)); + } remote()->transact(GET_OUTPUT, data, &reply); return static_cast <audio_io_handle_t> (reply.readInt32()); } @@ -374,6 +382,14 @@ public: *count = retCount; return status; } + + virtual bool isOffloadSupported(const audio_offload_info_t& info) + { + Parcel data, reply; + data.writeInterfaceToken(IAudioPolicyService::getInterfaceDescriptor()); + data.write(&info, sizeof(audio_offload_info_t)); + remote()->transact(IS_OFFLOAD_SUPPORTED, data, &reply); + return reply.readInt32(); } }; IMPLEMENT_META_INTERFACE(AudioPolicyService, "android.media.IAudioPolicyService"); @@ -442,12 +458,17 @@ status_t BnAudioPolicyService::onTransact( audio_channel_mask_t channelMask = data.readInt32(); audio_output_flags_t flags = static_cast <audio_output_flags_t>(data.readInt32()); - + bool hasOffloadInfo = data.readInt32() != 0; + audio_offload_info_t offloadInfo; + if (hasOffloadInfo) { + data.read(&offloadInfo, sizeof(audio_offload_info_t)); + } audio_io_handle_t output = getOutput(stream, samplingRate, format, channelMask, - flags); + flags, + hasOffloadInfo ? &offloadInfo : NULL); reply->writeInt32(static_cast <int>(output)); return NO_ERROR; } break; @@ -654,6 +675,15 @@ status_t BnAudioPolicyService::onTransact( return status; } + case IS_OFFLOAD_SUPPORTED: { + CHECK_INTERFACE(IAudioPolicyService, data, reply); + audio_offload_info_t info; + data.read(&info, sizeof(audio_offload_info_t)); + bool isSupported = isOffloadSupported(info); + reply->writeInt32(isSupported); + return NO_ERROR; + } + default: return BBinder::onTransact(code, data, reply, flags); } diff --git a/media/libmedia/IAudioRecord.cpp b/media/libmedia/IAudioRecord.cpp index 0d06e98..4a7de65 100644 --- a/media/libmedia/IAudioRecord.cpp +++ b/media/libmedia/IAudioRecord.cpp @@ -42,6 +42,18 @@ public: { } + virtual sp<IMemory> getCblk() const + { + Parcel data, reply; + sp<IMemory> cblk; + data.writeInterfaceToken(IAudioRecord::getInterfaceDescriptor()); + status_t status = remote()->transact(GET_CBLK, data, &reply); + if (status == NO_ERROR) { + cblk = interface_cast<IMemory>(reply.readStrongBinder()); + } + return cblk; + } + virtual status_t start(int /*AudioSystem::sync_event_t*/ event, int triggerSession) { Parcel data, reply; @@ -64,17 +76,6 @@ public: remote()->transact(STOP, data, &reply); } - virtual sp<IMemory> getCblk() const - { - Parcel data, reply; - sp<IMemory> cblk; - data.writeInterfaceToken(IAudioRecord::getInterfaceDescriptor()); - status_t status = remote()->transact(GET_CBLK, data, &reply); - if (status == NO_ERROR) { - cblk = interface_cast<IMemory>(reply.readStrongBinder()); - } - return cblk; - } }; IMPLEMENT_META_INTERFACE(AudioRecord, "android.media.IAudioRecord"); diff --git a/media/libmedia/IAudioTrack.cpp b/media/libmedia/IAudioTrack.cpp index e92f8aa..3cd9cfd 100644 --- a/media/libmedia/IAudioTrack.cpp +++ b/media/libmedia/IAudioTrack.cpp @@ -39,6 +39,9 @@ enum { ALLOCATE_TIMED_BUFFER, QUEUE_TIMED_BUFFER, SET_MEDIA_TIME_TRANSFORM, + SET_PARAMETERS, + GET_TIMESTAMP, + SIGNAL, }; class BpAudioTrack : public BpInterface<IAudioTrack> @@ -154,6 +157,38 @@ public: } return status; } + + virtual status_t setParameters(const String8& keyValuePairs) { + Parcel data, reply; + data.writeInterfaceToken(IAudioTrack::getInterfaceDescriptor()); + data.writeString8(keyValuePairs); + status_t status = remote()->transact(SET_PARAMETERS, data, &reply); + if (status == NO_ERROR) { + status = reply.readInt32(); + } + return status; + } + + virtual status_t getTimestamp(AudioTimestamp& timestamp) { + Parcel data, reply; + data.writeInterfaceToken(IAudioTrack::getInterfaceDescriptor()); + status_t status = remote()->transact(GET_TIMESTAMP, data, &reply); + if (status == NO_ERROR) { + status = reply.readInt32(); + if (status == NO_ERROR) { + timestamp.mPosition = reply.readInt32(); + timestamp.mTime.tv_sec = reply.readInt32(); + timestamp.mTime.tv_nsec = reply.readInt32(); + } + } + return status; + } + + virtual void signal() { + Parcel data, reply; + data.writeInterfaceToken(IAudioTrack::getInterfaceDescriptor()); + remote()->transact(SIGNAL, data, &reply); + } }; IMPLEMENT_META_INTERFACE(AudioTrack, "android.media.IAudioTrack"); @@ -223,6 +258,29 @@ status_t BnAudioTrack::onTransact( reply->writeInt32(setMediaTimeTransform(xform, target)); return NO_ERROR; } break; + case SET_PARAMETERS: { + CHECK_INTERFACE(IAudioTrack, data, reply); + String8 keyValuePairs(data.readString8()); + reply->writeInt32(setParameters(keyValuePairs)); + return NO_ERROR; + } break; + case GET_TIMESTAMP: { + CHECK_INTERFACE(IAudioTrack, data, reply); + AudioTimestamp timestamp; + status_t status = getTimestamp(timestamp); + reply->writeInt32(status); + if (status == NO_ERROR) { + reply->writeInt32(timestamp.mPosition); + reply->writeInt32(timestamp.mTime.tv_sec); + reply->writeInt32(timestamp.mTime.tv_nsec); + } + return NO_ERROR; + } break; + case SIGNAL: { + CHECK_INTERFACE(IAudioTrack, data, reply); + signal(); + return NO_ERROR; + } break; default: return BBinder::onTransact(code, data, reply, flags); } diff --git a/media/libmedia/IDrm.cpp b/media/libmedia/IDrm.cpp index 902aeb2..f7a9a75 100644 --- a/media/libmedia/IDrm.cpp +++ b/media/libmedia/IDrm.cpp @@ -68,10 +68,11 @@ struct BpDrm : public BpInterface<IDrm> { return reply.readInt32(); } - virtual bool isCryptoSchemeSupported(const uint8_t uuid[16]) { + virtual bool isCryptoSchemeSupported(const uint8_t uuid[16], const String8 &mimeType) { Parcel data, reply; data.writeInterfaceToken(IDrm::getInterfaceDescriptor()); data.write(uuid, 16); + data.writeString8(mimeType); remote()->transact(IS_CRYPTO_SUPPORTED, data, &reply); return reply.readInt32() != 0; @@ -438,7 +439,9 @@ status_t BnDrm::onTransact( CHECK_INTERFACE(IDrm, data, reply); uint8_t uuid[16]; data.read(uuid, sizeof(uuid)); - reply->writeInt32(isCryptoSchemeSupported(uuid)); + String8 mimeType = data.readString8(); + reply->writeInt32(isCryptoSchemeSupported(uuid, mimeType)); + return OK; } diff --git a/media/libmedia/IHDCP.cpp b/media/libmedia/IHDCP.cpp index f13addc..1cf987a 100644 --- a/media/libmedia/IHDCP.cpp +++ b/media/libmedia/IHDCP.cpp @@ -30,7 +30,9 @@ enum { HDCP_SET_OBSERVER, HDCP_INIT_ASYNC, HDCP_SHUTDOWN_ASYNC, + HDCP_GET_CAPS, HDCP_ENCRYPT, + HDCP_ENCRYPT_NATIVE, HDCP_DECRYPT, }; @@ -84,6 +86,13 @@ struct BpHDCP : public BpInterface<IHDCP> { return reply.readInt32(); } + virtual uint32_t getCaps() { + Parcel data, reply; + data.writeInterfaceToken(IHDCP::getInterfaceDescriptor()); + remote()->transact(HDCP_GET_CAPS, data, &reply); + return reply.readInt32(); + } + virtual status_t encrypt( const void *inData, size_t size, uint32_t streamCTR, uint64_t *outInputCTR, void *outData) { @@ -108,6 +117,31 @@ struct BpHDCP : public BpInterface<IHDCP> { return err; } + virtual status_t encryptNative( + const sp<GraphicBuffer> &graphicBuffer, + size_t offset, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData) { + Parcel data, reply; + data.writeInterfaceToken(IHDCP::getInterfaceDescriptor()); + data.write(*graphicBuffer); + data.writeInt32(offset); + data.writeInt32(size); + data.writeInt32(streamCTR); + remote()->transact(HDCP_ENCRYPT_NATIVE, data, &reply); + + status_t err = reply.readInt32(); + + if (err != OK) { + *outInputCTR = 0; + return err; + } + + *outInputCTR = reply.readInt64(); + reply.read(outData, size); + + return err; + } + virtual status_t decrypt( const void *inData, size_t size, uint32_t streamCTR, uint64_t inputCTR, @@ -196,6 +230,14 @@ status_t BnHDCP::onTransact( return OK; } + case HDCP_GET_CAPS: + { + CHECK_INTERFACE(IHDCP, data, reply); + + reply->writeInt32(getCaps()); + return OK; + } + case HDCP_ENCRYPT: { size_t size = data.readInt32(); @@ -222,6 +264,34 @@ status_t BnHDCP::onTransact( return OK; } + case HDCP_ENCRYPT_NATIVE: + { + CHECK_INTERFACE(IHDCP, data, reply); + + sp<GraphicBuffer> graphicBuffer = new GraphicBuffer(); + data.read(*graphicBuffer); + size_t offset = data.readInt32(); + size_t size = data.readInt32(); + uint32_t streamCTR = data.readInt32(); + void *outData = malloc(size); + uint64_t inputCTR; + + status_t err = encryptNative(graphicBuffer, offset, size, + streamCTR, &inputCTR, outData); + + reply->writeInt32(err); + + if (err == OK) { + reply->writeInt64(inputCTR); + reply->write(outData, size); + } + + free(outData); + outData = NULL; + + return OK; + } + case HDCP_DECRYPT: { size_t size = data.readInt32(); diff --git a/media/libmedia/IMediaDeathNotifier.cpp b/media/libmedia/IMediaDeathNotifier.cpp index 9199db6..9db5b1b 100644 --- a/media/libmedia/IMediaDeathNotifier.cpp +++ b/media/libmedia/IMediaDeathNotifier.cpp @@ -49,10 +49,10 @@ IMediaDeathNotifier::getMediaPlayerService() } while (true); if (sDeathNotifier == NULL) { - sDeathNotifier = new DeathNotifier(); - } - binder->linkToDeath(sDeathNotifier); - sMediaPlayerService = interface_cast<IMediaPlayerService>(binder); + sDeathNotifier = new DeathNotifier(); + } + binder->linkToDeath(sDeathNotifier); + sMediaPlayerService = interface_cast<IMediaPlayerService>(binder); } ALOGE_IF(sMediaPlayerService == 0, "no media player service!?"); return sMediaPlayerService; diff --git a/media/libmedia/IMediaPlayerService.cpp b/media/libmedia/IMediaPlayerService.cpp index 74f574d..3c22b4c 100644 --- a/media/libmedia/IMediaPlayerService.cpp +++ b/media/libmedia/IMediaPlayerService.cpp @@ -86,30 +86,48 @@ public: return interface_cast<IMediaRecorder>(reply.readStrongBinder()); } - virtual sp<IMemory> decode(const char* url, uint32_t *pSampleRate, int* pNumChannels, audio_format_t* pFormat) + virtual status_t decode(const char* url, uint32_t *pSampleRate, int* pNumChannels, + audio_format_t* pFormat, + const sp<IMemoryHeap>& heap, size_t *pSize) { Parcel data, reply; data.writeInterfaceToken(IMediaPlayerService::getInterfaceDescriptor()); data.writeCString(url); - remote()->transact(DECODE_URL, data, &reply); - *pSampleRate = uint32_t(reply.readInt32()); - *pNumChannels = reply.readInt32(); - *pFormat = (audio_format_t) reply.readInt32(); - return interface_cast<IMemory>(reply.readStrongBinder()); + data.writeStrongBinder(heap->asBinder()); + status_t status = remote()->transact(DECODE_URL, data, &reply); + if (status == NO_ERROR) { + status = (status_t)reply.readInt32(); + if (status == NO_ERROR) { + *pSampleRate = uint32_t(reply.readInt32()); + *pNumChannels = reply.readInt32(); + *pFormat = (audio_format_t)reply.readInt32(); + *pSize = (size_t)reply.readInt32(); + } + } + return status; } - virtual sp<IMemory> decode(int fd, int64_t offset, int64_t length, uint32_t *pSampleRate, int* pNumChannels, audio_format_t* pFormat) + virtual status_t decode(int fd, int64_t offset, int64_t length, uint32_t *pSampleRate, + int* pNumChannels, audio_format_t* pFormat, + const sp<IMemoryHeap>& heap, size_t *pSize) { Parcel data, reply; data.writeInterfaceToken(IMediaPlayerService::getInterfaceDescriptor()); data.writeFileDescriptor(fd); data.writeInt64(offset); data.writeInt64(length); - remote()->transact(DECODE_FD, data, &reply); - *pSampleRate = uint32_t(reply.readInt32()); - *pNumChannels = reply.readInt32(); - *pFormat = (audio_format_t) reply.readInt32(); - return interface_cast<IMemory>(reply.readStrongBinder()); + data.writeStrongBinder(heap->asBinder()); + status_t status = remote()->transact(DECODE_FD, data, &reply); + if (status == NO_ERROR) { + status = (status_t)reply.readInt32(); + if (status == NO_ERROR) { + *pSampleRate = uint32_t(reply.readInt32()); + *pNumChannels = reply.readInt32(); + *pFormat = (audio_format_t)reply.readInt32(); + *pSize = (size_t)reply.readInt32(); + } + } + return status; } virtual sp<IOMX> getOMX() { @@ -205,14 +223,19 @@ status_t BnMediaPlayerService::onTransact( case DECODE_URL: { CHECK_INTERFACE(IMediaPlayerService, data, reply); const char* url = data.readCString(); + sp<IMemoryHeap> heap = interface_cast<IMemoryHeap>(data.readStrongBinder()); uint32_t sampleRate; int numChannels; audio_format_t format; - sp<IMemory> player = decode(url, &sampleRate, &numChannels, &format); - reply->writeInt32(sampleRate); - reply->writeInt32(numChannels); - reply->writeInt32((int32_t) format); - reply->writeStrongBinder(player->asBinder()); + size_t size; + status_t status = decode(url, &sampleRate, &numChannels, &format, heap, &size); + reply->writeInt32(status); + if (status == NO_ERROR) { + reply->writeInt32(sampleRate); + reply->writeInt32(numChannels); + reply->writeInt32((int32_t)format); + reply->writeInt32((int32_t)size); + } return NO_ERROR; } break; case DECODE_FD: { @@ -220,14 +243,20 @@ status_t BnMediaPlayerService::onTransact( int fd = dup(data.readFileDescriptor()); int64_t offset = data.readInt64(); int64_t length = data.readInt64(); + sp<IMemoryHeap> heap = interface_cast<IMemoryHeap>(data.readStrongBinder()); uint32_t sampleRate; int numChannels; audio_format_t format; - sp<IMemory> player = decode(fd, offset, length, &sampleRate, &numChannels, &format); - reply->writeInt32(sampleRate); - reply->writeInt32(numChannels); - reply->writeInt32((int32_t) format); - reply->writeStrongBinder(player->asBinder()); + size_t size; + status_t status = decode(fd, offset, length, &sampleRate, &numChannels, &format, + heap, &size); + reply->writeInt32(status); + if (status == NO_ERROR) { + reply->writeInt32(sampleRate); + reply->writeInt32(numChannels); + reply->writeInt32((int32_t)format); + reply->writeInt32((int32_t)size); + } return NO_ERROR; } break; case CREATE_MEDIA_RECORDER: { diff --git a/media/libmedia/IOMX.cpp b/media/libmedia/IOMX.cpp index d6cd43a..71ce320 100644 --- a/media/libmedia/IOMX.cpp +++ b/media/libmedia/IOMX.cpp @@ -43,6 +43,7 @@ enum { CREATE_INPUT_SURFACE, SIGNAL_END_OF_INPUT_STREAM, STORE_META_DATA_IN_BUFFERS, + PREPARE_FOR_ADAPTIVE_PLAYBACK, ALLOC_BUFFER, ALLOC_BUFFER_WITH_BACKUP, FREE_BUFFER, @@ -51,6 +52,8 @@ enum { GET_EXTENSION_INDEX, OBSERVER_ON_MSG, GET_GRAPHIC_BUFFER_USAGE, + SET_INTERNAL_OPTION, + UPDATE_GRAPHIC_BUFFER_IN_META, }; class BpOMX : public BpInterface<IOMX> { @@ -282,6 +285,21 @@ public: return err; } + virtual status_t updateGraphicBufferInMeta( + node_id node, OMX_U32 port_index, + const sp<GraphicBuffer> &graphicBuffer, buffer_id buffer) { + Parcel data, reply; + data.writeInterfaceToken(IOMX::getInterfaceDescriptor()); + data.writeIntPtr((intptr_t)node); + data.writeInt32(port_index); + data.write(*graphicBuffer); + data.writeIntPtr((intptr_t)buffer); + remote()->transact(UPDATE_GRAPHIC_BUFFER_IN_META, data, &reply); + + status_t err = reply.readInt32(); + return err; + } + virtual status_t createInputSurface( node_id node, OMX_U32 port_index, sp<IGraphicBufferProducer> *bufferProducer) { @@ -334,6 +352,22 @@ public: return err; } + virtual status_t prepareForAdaptivePlayback( + node_id node, OMX_U32 port_index, OMX_BOOL enable, + OMX_U32 max_width, OMX_U32 max_height) { + Parcel data, reply; + data.writeInterfaceToken(IOMX::getInterfaceDescriptor()); + data.writeIntPtr((intptr_t)node); + data.writeInt32(port_index); + data.writeInt32((int32_t)enable); + data.writeInt32(max_width); + data.writeInt32(max_height); + remote()->transact(PREPARE_FOR_ADAPTIVE_PLAYBACK, data, &reply); + + status_t err = reply.readInt32(); + return err; + } + virtual status_t allocateBuffer( node_id node, OMX_U32 port_index, size_t size, buffer_id *buffer, void **buffer_data) { @@ -439,6 +473,24 @@ public: return err; } + + virtual status_t setInternalOption( + node_id node, + OMX_U32 port_index, + InternalOptionType type, + const void *optionData, + size_t size) { + Parcel data, reply; + data.writeInterfaceToken(IOMX::getInterfaceDescriptor()); + data.writeIntPtr((intptr_t)node); + data.writeInt32(port_index); + data.writeInt32(size); + data.write(optionData, size); + data.writeInt32(type); + remote()->transact(SET_INTERNAL_OPTION, data, &reply); + + return reply.readInt32(); + } }; IMPLEMENT_META_INTERFACE(OMX, "android.hardware.IOMX"); @@ -537,6 +589,7 @@ status_t BnOMX::onTransact( case SET_PARAMETER: case GET_CONFIG: case SET_CONFIG: + case SET_INTERNAL_OPTION: { CHECK_OMX_INTERFACE(IOMX, data, reply); @@ -562,6 +615,15 @@ status_t BnOMX::onTransact( case SET_CONFIG: err = setConfig(node, index, params, size); break; + case SET_INTERNAL_OPTION: + { + InternalOptionType type = + (InternalOptionType)data.readInt32(); + + err = setInternalOption(node, index, type, params, size); + break; + } + default: TRESPASS(); } @@ -662,6 +724,23 @@ status_t BnOMX::onTransact( return NO_ERROR; } + case UPDATE_GRAPHIC_BUFFER_IN_META: + { + CHECK_OMX_INTERFACE(IOMX, data, reply); + + node_id node = (void*)data.readIntPtr(); + OMX_U32 port_index = data.readInt32(); + sp<GraphicBuffer> graphicBuffer = new GraphicBuffer(); + data.read(*graphicBuffer); + buffer_id buffer = (void*)data.readIntPtr(); + + status_t err = updateGraphicBufferInMeta( + node, port_index, graphicBuffer, buffer); + reply->writeInt32(err); + + return NO_ERROR; + } + case CREATE_INPUT_SURFACE: { CHECK_OMX_INTERFACE(IOMX, data, reply); @@ -708,6 +787,23 @@ status_t BnOMX::onTransact( return NO_ERROR; } + case PREPARE_FOR_ADAPTIVE_PLAYBACK: + { + CHECK_OMX_INTERFACE(IOMX, data, reply); + + node_id node = (void*)data.readIntPtr(); + OMX_U32 port_index = data.readInt32(); + OMX_BOOL enable = (OMX_BOOL)data.readInt32(); + OMX_U32 max_width = data.readInt32(); + OMX_U32 max_height = data.readInt32(); + + status_t err = prepareForAdaptivePlayback( + node, port_index, enable, max_width, max_height); + reply->writeInt32(err); + + return NO_ERROR; + } + case ALLOC_BUFFER: { CHECK_OMX_INTERFACE(IOMX, data, reply); diff --git a/media/libmedia/IRemoteDisplayClient.cpp b/media/libmedia/IRemoteDisplayClient.cpp index 5c494b3..7190879 100644 --- a/media/libmedia/IRemoteDisplayClient.cpp +++ b/media/libmedia/IRemoteDisplayClient.cpp @@ -38,7 +38,7 @@ public: } void onDisplayConnected(const sp<IGraphicBufferProducer>& bufferProducer, - uint32_t width, uint32_t height, uint32_t flags) + uint32_t width, uint32_t height, uint32_t flags, uint32_t session) { Parcel data, reply; data.writeInterfaceToken(IRemoteDisplayClient::getInterfaceDescriptor()); @@ -46,6 +46,7 @@ public: data.writeInt32(width); data.writeInt32(height); data.writeInt32(flags); + data.writeInt32(session); remote()->transact(ON_DISPLAY_CONNECTED, data, &reply, IBinder::FLAG_ONEWAY); } @@ -80,7 +81,8 @@ status_t BnRemoteDisplayClient::onTransact( uint32_t width = data.readInt32(); uint32_t height = data.readInt32(); uint32_t flags = data.readInt32(); - onDisplayConnected(surfaceTexture, width, height, flags); + uint32_t session = data.readInt32(); + onDisplayConnected(surfaceTexture, width, height, flags, session); return NO_ERROR; } case ON_DISPLAY_DISCONNECTED: { diff --git a/media/libmedia/JetPlayer.cpp b/media/libmedia/JetPlayer.cpp index 59e538f..e914b34 100644 --- a/media/libmedia/JetPlayer.cpp +++ b/media/libmedia/JetPlayer.cpp @@ -18,8 +18,6 @@ #define LOG_TAG "JetPlayer-C" #include <utils/Log.h> -#include <utils/threads.h> - #include <media/JetPlayer.h> @@ -39,7 +37,6 @@ JetPlayer::JetPlayer(void *javaJetPlayer, int maxTracks, int trackBufferSize) : mMaxTracks(maxTracks), mEasData(NULL), mEasJetFileLoc(NULL), - mAudioTrack(NULL), mTrackBufferSize(trackBufferSize) { ALOGV("JetPlayer constructor"); @@ -140,11 +137,10 @@ int JetPlayer::release() free(mEasJetFileLoc); mEasJetFileLoc = NULL; } - if (mAudioTrack) { + if (mAudioTrack != 0) { mAudioTrack->stop(); mAudioTrack->flush(); - delete mAudioTrack; - mAudioTrack = NULL; + mAudioTrack.clear(); } if (mAudioBuffer) { delete mAudioBuffer; diff --git a/media/libmedia/MediaScannerClient.cpp b/media/libmedia/MediaScannerClient.cpp index e1e3348..93a4a4c 100644 --- a/media/libmedia/MediaScannerClient.cpp +++ b/media/libmedia/MediaScannerClient.cpp @@ -16,7 +16,7 @@ #include <media/mediascanner.h> -#include <utils/StringArray.h> +#include "StringArray.h" #include "autodetect.h" #include "unicode/ucnv.h" diff --git a/media/libmedia/SingleStateQueueInstantiations.cpp b/media/libmedia/SingleStateQueueInstantiations.cpp index 2afebe9..0265c8c 100644 --- a/media/libmedia/SingleStateQueueInstantiations.cpp +++ b/media/libmedia/SingleStateQueueInstantiations.cpp @@ -16,11 +16,13 @@ #include <media/SingleStateQueue.h> #include <private/media/StaticAudioTrackState.h> +#include <media/AudioTimestamp.h> // FIXME hack for gcc namespace android { template class SingleStateQueue<StaticAudioTrackState>; // typedef StaticAudioTrackSingleStateQueue +template class SingleStateQueue<AudioTimestamp>; // typedef AudioTimestampSingleStateQueue } diff --git a/media/libmedia/SoundPool.cpp b/media/libmedia/SoundPool.cpp index ee70ef7..22e9fad 100644 --- a/media/libmedia/SoundPool.cpp +++ b/media/libmedia/SoundPool.cpp @@ -18,16 +18,10 @@ #define LOG_TAG "SoundPool" #include <utils/Log.h> -//#define USE_SHARED_MEM_BUFFER - -// XXX needed for timing latency -#include <utils/Timers.h> +#define USE_SHARED_MEM_BUFFER #include <media/AudioTrack.h> #include <media/mediaplayer.h> - -#include <system/audio.h> - #include <media/SoundPool.h> #include "SoundPoolThread.h" @@ -38,6 +32,8 @@ int kDefaultBufferCount = 4; uint32_t kMaxSampleRate = 48000; uint32_t kDefaultSampleRate = 44100; uint32_t kDefaultFrameCount = 1200; +size_t kDefaultHeapSize = 1024 * 1024; // 1MB + SoundPool::SoundPool(int maxChannels, audio_stream_type_t streamType, int srcQuality) { @@ -470,7 +466,6 @@ Sample::Sample(int sampleID, int fd, int64_t offset, int64_t length) void Sample::init() { - mData = 0; mSize = 0; mRefCount = 0; mSampleID = 0; @@ -488,7 +483,6 @@ Sample::~Sample() ALOGV("close(%d)", mFd); ::close(mFd); } - mData.clear(); free(mUrl); } @@ -497,44 +491,48 @@ status_t Sample::doLoad() uint32_t sampleRate; int numChannels; audio_format_t format; - sp<IMemory> p; + status_t status; + mHeap = new MemoryHeapBase(kDefaultHeapSize); + ALOGV("Start decode"); if (mUrl) { - p = MediaPlayer::decode(mUrl, &sampleRate, &numChannels, &format); + status = MediaPlayer::decode(mUrl, &sampleRate, &numChannels, &format, mHeap, &mSize); } else { - p = MediaPlayer::decode(mFd, mOffset, mLength, &sampleRate, &numChannels, &format); + status = MediaPlayer::decode(mFd, mOffset, mLength, &sampleRate, &numChannels, &format, + mHeap, &mSize); ALOGV("close(%d)", mFd); ::close(mFd); mFd = -1; } - if (p == 0) { + if (status != NO_ERROR) { ALOGE("Unable to load sample: %s", mUrl); - return -1; + goto error; } ALOGV("pointer = %p, size = %u, sampleRate = %u, numChannels = %d", - p->pointer(), p->size(), sampleRate, numChannels); + mHeap->getBase(), mSize, sampleRate, numChannels); if (sampleRate > kMaxSampleRate) { ALOGE("Sample rate (%u) out of range", sampleRate); - return - 1; + status = BAD_VALUE; + goto error; } if ((numChannels < 1) || (numChannels > 2)) { ALOGE("Sample channel count (%d) out of range", numChannels); - return - 1; + status = BAD_VALUE; + goto error; } - //_dumpBuffer(p->pointer(), p->size()); - uint8_t* q = static_cast<uint8_t*>(p->pointer()) + p->size() - 10; - //_dumpBuffer(q, 10, 10, false); - - mData = p; - mSize = p->size(); + mData = new MemoryBase(mHeap, 0, mSize); mSampleRate = sampleRate; mNumChannels = numChannels; mFormat = format; mState = READY; - return 0; + return NO_ERROR; + +error: + mHeap.clear(); + return status; } @@ -547,8 +545,8 @@ void SoundChannel::init(SoundPool* soundPool) void SoundChannel::play(const sp<Sample>& sample, int nextChannelID, float leftVolume, float rightVolume, int priority, int loop, float rate) { - AudioTrack* oldTrack; - AudioTrack* newTrack; + sp<AudioTrack> oldTrack; + sp<AudioTrack> newTrack; status_t status; { // scope for the lock @@ -608,7 +606,7 @@ void SoundChannel::play(const sp<Sample>& sample, int nextChannelID, float leftV // do not create a new audio track if current track is compatible with sample parameters #ifdef USE_SHARED_MEM_BUFFER newTrack = new AudioTrack(streamType, sampleRate, sample->format(), - channels, sample->getIMemory(), AUDIO_OUTPUT_FLAG_NONE, callback, userData); + channels, sample->getIMemory(), AUDIO_OUTPUT_FLAG_FAST, callback, userData); #else newTrack = new AudioTrack(streamType, sampleRate, sample->format(), channels, frameCount, AUDIO_OUTPUT_FLAG_FAST, callback, userData, @@ -620,7 +618,7 @@ void SoundChannel::play(const sp<Sample>& sample, int nextChannelID, float leftV ALOGE("Error creating AudioTrack"); goto exit; } - ALOGV("setVolume %p", newTrack); + ALOGV("setVolume %p", newTrack.get()); newTrack->setVolume(leftVolume, rightVolume); newTrack->setLoop(0, frameCount, loop); @@ -643,11 +641,9 @@ void SoundChannel::play(const sp<Sample>& sample, int nextChannelID, float leftV } exit: - ALOGV("delete oldTrack %p", oldTrack); - delete oldTrack; + ALOGV("delete oldTrack %p", oldTrack.get()); if (status != NO_ERROR) { - delete newTrack; - mAudioTrack = NULL; + mAudioTrack.clear(); } } @@ -748,11 +744,16 @@ void SoundChannel::process(int event, void *info, unsigned long toggle) b->size = count; //ALOGV("buffer=%p, [0]=%d", b->i16, b->i16[0]); } - } else if (event == AudioTrack::EVENT_UNDERRUN) { - ALOGV("process %p channel %d EVENT_UNDERRUN", this, mChannelID); + } else if (event == AudioTrack::EVENT_UNDERRUN || event == AudioTrack::EVENT_BUFFER_END || + event == AudioTrack::EVENT_NEW_IAUDIOTRACK) { + ALOGV("process %p channel %d event %s", + this, mChannelID, (event == AudioTrack::EVENT_UNDERRUN) ? "UNDERRUN" : + (event == AudioTrack::EVENT_BUFFER_END) ? "BUFFER_END" : "NEW_IAUDIOTRACK"); mSoundPool->addToStopList(this); } else if (event == AudioTrack::EVENT_LOOP_END) { - ALOGV("End loop %p channel %d count %d", this, mChannelID, *(int *)info); + ALOGV("End loop %p channel %d", this, mChannelID); + } else { + ALOGW("SoundChannel::process unexpected event %d", event); } } @@ -884,7 +885,7 @@ SoundChannel::~SoundChannel() } // do not call AudioTrack destructor with mLock held as it will wait for the AudioTrack // callback thread to exit which may need to execute process() and acquire the mLock. - delete mAudioTrack; + mAudioTrack.clear(); } void SoundChannel::dump() diff --git a/media/libmedia/StringArray.cpp b/media/libmedia/StringArray.cpp new file mode 100644 index 0000000..5f5b57a --- /dev/null +++ b/media/libmedia/StringArray.cpp @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// +// Sortable array of strings. STL-ish, but STL-free. +// + +#include <stdlib.h> +#include <string.h> + +#include "StringArray.h" + +namespace android { + +// +// An expanding array of strings. Add, get, sort, delete. +// +StringArray::StringArray() + : mMax(0), mCurrent(0), mArray(NULL) +{ +} + +StringArray:: ~StringArray() { + for (int i = 0; i < mCurrent; i++) + delete[] mArray[i]; + delete[] mArray; +} + +// +// Add a string. A copy of the string is made. +// +bool StringArray::push_back(const char* str) { + if (mCurrent >= mMax) { + char** tmp; + + if (mMax == 0) + mMax = 16; // initial storage + else + mMax *= 2; + + tmp = new char*[mMax]; + if (tmp == NULL) + return false; + + memcpy(tmp, mArray, mCurrent * sizeof(char*)); + delete[] mArray; + mArray = tmp; + } + + int len = strlen(str); + mArray[mCurrent] = new char[len+1]; + memcpy(mArray[mCurrent], str, len+1); + mCurrent++; + + return true; +} + +// +// Delete an entry. +// +void StringArray::erase(int idx) { + if (idx < 0 || idx >= mCurrent) + return; + delete[] mArray[idx]; + if (idx < mCurrent-1) { + memmove(&mArray[idx], &mArray[idx+1], + (mCurrent-1 - idx) * sizeof(char*)); + } + mCurrent--; +} + +// +// Sort the array. +// +void StringArray::sort(int (*compare)(const void*, const void*)) { + qsort(mArray, mCurrent, sizeof(char*), compare); +} + +// +// Pass this to the sort routine to do an ascending alphabetical sort. +// +int StringArray::cmpAscendingAlpha(const void* pstr1, const void* pstr2) { + return strcmp(*(const char**)pstr1, *(const char**)pstr2); +} + +// +// Set entry N to specified string. +// [should use operator[] here] +// +void StringArray::setEntry(int idx, const char* str) { + if (idx < 0 || idx >= mCurrent) + return; + delete[] mArray[idx]; + int len = strlen(str); + mArray[idx] = new char[len+1]; + memcpy(mArray[idx], str, len+1); +} + + +}; // namespace android diff --git a/media/libmedia/StringArray.h b/media/libmedia/StringArray.h new file mode 100644 index 0000000..ae47085 --- /dev/null +++ b/media/libmedia/StringArray.h @@ -0,0 +1,83 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// +// Sortable array of strings. STL-ish, but STL-free. +// +#ifndef _LIBS_MEDIA_STRING_ARRAY_H +#define _LIBS_MEDIA_STRING_ARRAY_H + +#include <stdlib.h> +#include <string.h> + +namespace android { + +// +// An expanding array of strings. Add, get, sort, delete. +// +class StringArray { +public: + StringArray(); + virtual ~StringArray(); + + // + // Add a string. A copy of the string is made. + // + bool push_back(const char* str); + + // + // Delete an entry. + // + void erase(int idx); + + // + // Sort the array. + // + void sort(int (*compare)(const void*, const void*)); + + // + // Pass this to the sort routine to do an ascending alphabetical sort. + // + static int cmpAscendingAlpha(const void* pstr1, const void* pstr2); + + // + // Get the #of items in the array. + // + inline int size(void) const { return mCurrent; } + + // + // Return entry N. + // [should use operator[] here] + // + const char* getEntry(int idx) const { + return (unsigned(idx) >= unsigned(mCurrent)) ? NULL : mArray[idx]; + } + + // + // Set entry N to specified string. + // [should use operator[] here] + // + void setEntry(int idx, const char* str); + +private: + int mMax; + int mCurrent; + char** mArray; +}; + +}; // namespace android + +#endif // _LIBS_MEDIA_STRING_ARRAY_H diff --git a/media/libmedia/ToneGenerator.cpp b/media/libmedia/ToneGenerator.cpp index f55b697..adef3be 100644 --- a/media/libmedia/ToneGenerator.cpp +++ b/media/libmedia/ToneGenerator.cpp @@ -16,13 +16,9 @@ //#define LOG_NDEBUG 0 #define LOG_TAG "ToneGenerator" -#include <utils/threads.h> -#include <stdio.h> #include <math.h> #include <utils/Log.h> -#include <utils/RefBase.h> -#include <utils/Timers.h> #include <cutils/properties.h> #include "media/ToneGenerator.h" @@ -803,7 +799,6 @@ ToneGenerator::ToneGenerator(audio_stream_type_t streamType, float volume, bool ALOGV("ToneGenerator constructor: streamType=%d, volume=%f", streamType, volume); mState = TONE_IDLE; - mpAudioTrack = NULL; if (AudioSystem::getOutputSamplingRate(&mSamplingRate, streamType) != NO_ERROR) { ALOGE("Unable to marshal AudioFlinger"); @@ -855,10 +850,10 @@ ToneGenerator::ToneGenerator(audio_stream_type_t streamType, float volume, bool ToneGenerator::~ToneGenerator() { ALOGV("ToneGenerator destructor"); - if (mpAudioTrack != NULL) { + if (mpAudioTrack != 0) { stopTone(); - ALOGV("Delete Track: %p", mpAudioTrack); - delete mpAudioTrack; + ALOGV("Delete Track: %p", mpAudioTrack.get()); + mpAudioTrack.clear(); } } @@ -1047,14 +1042,9 @@ void ToneGenerator::stopTone() { //////////////////////////////////////////////////////////////////////////////// bool ToneGenerator::initAudioTrack() { - if (mpAudioTrack) { - delete mpAudioTrack; - mpAudioTrack = NULL; - } - // Open audio track in mono, PCM 16bit, default sampling rate, default buffer size mpAudioTrack = new AudioTrack(); - ALOGV("Create Track: %p", mpAudioTrack); + ALOGV("Create Track: %p", mpAudioTrack.get()); mpAudioTrack->set(mStreamType, 0, // sampleRate @@ -1066,7 +1056,9 @@ bool ToneGenerator::initAudioTrack() { this, // user 0, // notificationFrames 0, // sharedBuffer - mThreadCanCallJava); + mThreadCanCallJava, + 0, // sessionId + AudioTrack::TRANSFER_CALLBACK); if (mpAudioTrack->initCheck() != NO_ERROR) { ALOGE("AudioTrack->initCheck failed"); @@ -1081,12 +1073,10 @@ bool ToneGenerator::initAudioTrack() { initAudioTrack_exit: + ALOGV("Init failed: %p", mpAudioTrack.get()); + // Cleanup - if (mpAudioTrack != NULL) { - ALOGV("Delete Track I: %p", mpAudioTrack); - delete mpAudioTrack; - mpAudioTrack = NULL; - } + mpAudioTrack.clear(); return false; } diff --git a/media/libmedia/Visualizer.cpp b/media/libmedia/Visualizer.cpp index 5b4071b..c146b8d 100644 --- a/media/libmedia/Visualizer.cpp +++ b/media/libmedia/Visualizer.cpp @@ -28,6 +28,7 @@ #include <media/Visualizer.h> #include <audio_utils/fixedfft.h> +#include <utils/Thread.h> namespace android { @@ -42,6 +43,7 @@ Visualizer::Visualizer (int32_t priority, mCaptureSize(CAPTURE_SIZE_DEF), mSampleRate(44100000), mScalingMode(VISUALIZER_SCALING_MODE_NORMALIZED), + mMeasurementMode(MEASUREMENT_MODE_NONE), mCaptureCallBack(NULL), mCaptureCbkUser(NULL) { @@ -185,6 +187,73 @@ status_t Visualizer::setScalingMode(uint32_t mode) { return status; } +status_t Visualizer::setMeasurementMode(uint32_t mode) { + if ((mode != MEASUREMENT_MODE_NONE) + //Note: needs to be handled as a mask when more measurement modes are added + && ((mode & MEASUREMENT_MODE_PEAK_RMS) != mode)) { + return BAD_VALUE; + } + + Mutex::Autolock _l(mCaptureLock); + + uint32_t buf32[sizeof(effect_param_t) / sizeof(uint32_t) + 2]; + effect_param_t *p = (effect_param_t *)buf32; + + p->psize = sizeof(uint32_t); + p->vsize = sizeof(uint32_t); + *(int32_t *)p->data = VISUALIZER_PARAM_MEASUREMENT_MODE; + *((int32_t *)p->data + 1)= mode; + status_t status = setParameter(p); + + ALOGV("setMeasurementMode mode %d status %d p->status %d", mode, status, p->status); + + if (status == NO_ERROR) { + status = p->status; + if (status == NO_ERROR) { + mMeasurementMode = mode; + } + } + return status; +} + +status_t Visualizer::getIntMeasurements(uint32_t type, uint32_t number, int32_t *measurements) { + if (mMeasurementMode == MEASUREMENT_MODE_NONE) { + ALOGE("Cannot retrieve int measurements, no measurement mode set"); + return INVALID_OPERATION; + } + if (!(mMeasurementMode & type)) { + // measurement type has not been set on this Visualizer + ALOGE("Cannot retrieve int measurements, requested measurement mode 0x%x not set(0x%x)", + type, mMeasurementMode); + return INVALID_OPERATION; + } + // only peak+RMS measurement supported + if ((type != MEASUREMENT_MODE_PEAK_RMS) + // for peak+RMS measurement, the results are 2 int32_t values + || (number != 2)) { + ALOGE("Cannot retrieve int measurements, MEASUREMENT_MODE_PEAK_RMS returns 2 ints, not %d", + number); + return BAD_VALUE; + } + + status_t status = NO_ERROR; + if (mEnabled) { + uint32_t replySize = number * sizeof(int32_t); + status = command(VISUALIZER_CMD_MEASURE, + sizeof(uint32_t) /*cmdSize*/, + &type /*cmdData*/, + &replySize, measurements); + ALOGV("getMeasurements() command returned %d", status); + if ((status == NO_ERROR) && (replySize == 0)) { + status = NOT_ENOUGH_DATA; + } + } else { + ALOGV("getMeasurements() disabled"); + return INVALID_OPERATION; + } + return status; +} + status_t Visualizer::getWaveForm(uint8_t *waveform) { if (waveform == NULL) { diff --git a/media/libmedia/mediaplayer.cpp b/media/libmedia/mediaplayer.cpp index 963b04f..0f6d897 100644 --- a/media/libmedia/mediaplayer.cpp +++ b/media/libmedia/mediaplayer.cpp @@ -756,6 +756,9 @@ void MediaPlayer::notify(int msg, int ext1, int ext2, const Parcel *obj) case MEDIA_TIMED_TEXT: ALOGV("Received timed text message"); break; + case MEDIA_SUBTITLE_DATA: + ALOGV("Received subtitle data message"); + break; default: ALOGV("unrecognized message: (%d, %d, %d)", msg, ext1, ext2); break; @@ -773,17 +776,20 @@ void MediaPlayer::notify(int msg, int ext1, int ext2, const Parcel *obj) } } -/*static*/ sp<IMemory> MediaPlayer::decode(const char* url, uint32_t *pSampleRate, int* pNumChannels, audio_format_t* pFormat) +/*static*/ status_t MediaPlayer::decode(const char* url, uint32_t *pSampleRate, + int* pNumChannels, audio_format_t* pFormat, + const sp<IMemoryHeap>& heap, size_t *pSize) { ALOGV("decode(%s)", url); - sp<IMemory> p; + status_t status; const sp<IMediaPlayerService>& service = getMediaPlayerService(); if (service != 0) { - p = service->decode(url, pSampleRate, pNumChannels, pFormat); + status = service->decode(url, pSampleRate, pNumChannels, pFormat, heap, pSize); } else { ALOGE("Unable to locate media service"); + status = DEAD_OBJECT; } - return p; + return status; } @@ -793,17 +799,22 @@ void MediaPlayer::died() notify(MEDIA_ERROR, MEDIA_ERROR_SERVER_DIED, 0); } -/*static*/ sp<IMemory> MediaPlayer::decode(int fd, int64_t offset, int64_t length, uint32_t *pSampleRate, int* pNumChannels, audio_format_t* pFormat) +/*static*/ status_t MediaPlayer::decode(int fd, int64_t offset, int64_t length, + uint32_t *pSampleRate, int* pNumChannels, + audio_format_t* pFormat, + const sp<IMemoryHeap>& heap, size_t *pSize) { ALOGV("decode(%d, %lld, %lld)", fd, offset, length); - sp<IMemory> p; + status_t status; const sp<IMediaPlayerService>& service = getMediaPlayerService(); if (service != 0) { - p = service->decode(fd, offset, length, pSampleRate, pNumChannels, pFormat); + status = service->decode(fd, offset, length, pSampleRate, + pNumChannels, pFormat, heap, pSize); } else { ALOGE("Unable to locate media service"); + status = DEAD_OBJECT; } - return p; + return status; } @@ -811,6 +822,13 @@ status_t MediaPlayer::setNextMediaPlayer(const sp<MediaPlayer>& next) { if (mPlayer == NULL) { return NO_INIT; } + + if (next != NULL && !(next->mCurrentState & + (MEDIA_PLAYER_PREPARED | MEDIA_PLAYER_PAUSED | MEDIA_PLAYER_PLAYBACK_COMPLETE))) { + ALOGE("next player is not prepared"); + return INVALID_OPERATION; + } + return mPlayer->setNextPlayer(next == NULL ? NULL : next->mPlayer); } diff --git a/media/libmediaplayerservice/Android.mk b/media/libmediaplayerservice/Android.mk index d87bc7f..8f21632 100644 --- a/media/libmediaplayerservice/Android.mk +++ b/media/libmediaplayerservice/Android.mk @@ -34,6 +34,7 @@ LOCAL_SHARED_LIBRARIES := \ libsonivox \ libstagefright \ libstagefright_foundation \ + libstagefright_httplive \ libstagefright_omx \ libstagefright_wfd \ libutils \ diff --git a/media/libmediaplayerservice/Crypto.cpp b/media/libmediaplayerservice/Crypto.cpp index ae4d845..62593b2 100644 --- a/media/libmediaplayerservice/Crypto.cpp +++ b/media/libmediaplayerservice/Crypto.cpp @@ -134,7 +134,6 @@ void Crypto::findFactoryForScheme(const uint8_t uuid[16]) { return; } - ALOGE("Failed to find crypto plugin"); mInitCheck = ERROR_UNSUPPORTED; } @@ -151,6 +150,7 @@ bool Crypto::loadLibraryForScheme(const String8 &path, const uint8_t uuid[16]) { if (!mLibrary.get()) { mLibrary = new SharedLibrary(path); if (!*mLibrary) { + ALOGE("loadLibraryForScheme failed:%s", mLibrary->lastError()); return false; } @@ -165,6 +165,7 @@ bool Crypto::loadLibraryForScheme(const String8 &path, const uint8_t uuid[16]) { if (createCryptoFactory == NULL || (mFactory = createCryptoFactory()) == NULL || !mFactory->isCryptoSchemeSupported(uuid)) { + ALOGE("createCryptoFactory failed:%s", mLibrary->lastError()); closeFactory(); return false; } diff --git a/media/libmediaplayerservice/Drm.cpp b/media/libmediaplayerservice/Drm.cpp index f00f488..eebcb79 100644 --- a/media/libmediaplayerservice/Drm.cpp +++ b/media/libmediaplayerservice/Drm.cpp @@ -211,15 +211,22 @@ bool Drm::loadLibraryForScheme(const String8 &path, const uint8_t uuid[16]) { return true; } -bool Drm::isCryptoSchemeSupported(const uint8_t uuid[16]) { +bool Drm::isCryptoSchemeSupported(const uint8_t uuid[16], const String8 &mimeType) { + Mutex::Autolock autoLock(mLock); - if (mFactory && mFactory->isCryptoSchemeSupported(uuid)) { - return true; + if (!mFactory || !mFactory->isCryptoSchemeSupported(uuid)) { + findFactoryForScheme(uuid); + if (mInitCheck != OK) { + return false; + } + } + + if (mimeType != "") { + return mFactory->isContentTypeSupported(mimeType); } - findFactoryForScheme(uuid); - return (mInitCheck == OK); + return true; } status_t Drm::createPlugin(const uint8_t uuid[16]) { diff --git a/media/libmediaplayerservice/Drm.h b/media/libmediaplayerservice/Drm.h index 3f460f1..119fd50 100644 --- a/media/libmediaplayerservice/Drm.h +++ b/media/libmediaplayerservice/Drm.h @@ -37,7 +37,7 @@ struct Drm : public BnDrm, virtual status_t initCheck() const; - virtual bool isCryptoSchemeSupported(const uint8_t uuid[16]); + virtual bool isCryptoSchemeSupported(const uint8_t uuid[16], const String8 &mimeType); virtual status_t createPlugin(const uint8_t uuid[16]); diff --git a/media/libmediaplayerservice/HDCP.cpp b/media/libmediaplayerservice/HDCP.cpp index 469a02e..c2ac1a3 100644 --- a/media/libmediaplayerservice/HDCP.cpp +++ b/media/libmediaplayerservice/HDCP.cpp @@ -100,6 +100,20 @@ status_t HDCP::shutdownAsync() { return mHDCPModule->shutdownAsync(); } +uint32_t HDCP::getCaps() { + Mutex::Autolock autoLock(mLock); + + if (mHDCPModule == NULL) { + return NO_INIT; + } + + // TO-DO: + // Only support HDCP_CAPS_ENCRYPT (byte-array to byte-array) for now. + // use mHDCPModule->getCaps() when the HDCP libraries get updated. + //return mHDCPModule->getCaps(); + return HDCPModule::HDCP_CAPS_ENCRYPT; +} + status_t HDCP::encrypt( const void *inData, size_t size, uint32_t streamCTR, uint64_t *outInputCTR, void *outData) { @@ -116,6 +130,24 @@ status_t HDCP::encrypt( return mHDCPModule->encrypt(inData, size, streamCTR, outInputCTR, outData); } +status_t HDCP::encryptNative( + const sp<GraphicBuffer> &graphicBuffer, + size_t offset, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData) { + Mutex::Autolock autoLock(mLock); + + CHECK(mIsEncryptionModule); + + if (mHDCPModule == NULL) { + *outInputCTR = 0; + + return NO_INIT; + } + + return mHDCPModule->encryptNative(graphicBuffer->handle, + offset, size, streamCTR, outInputCTR, outData); +} + status_t HDCP::decrypt( const void *inData, size_t size, uint32_t streamCTR, uint64_t outInputCTR, void *outData) { diff --git a/media/libmediaplayerservice/HDCP.h b/media/libmediaplayerservice/HDCP.h index 42e6467..26ddc86 100644 --- a/media/libmediaplayerservice/HDCP.h +++ b/media/libmediaplayerservice/HDCP.h @@ -30,11 +30,17 @@ struct HDCP : public BnHDCP { virtual status_t setObserver(const sp<IHDCPObserver> &observer); virtual status_t initAsync(const char *host, unsigned port); virtual status_t shutdownAsync(); + virtual uint32_t getCaps(); virtual status_t encrypt( const void *inData, size_t size, uint32_t streamCTR, uint64_t *outInputCTR, void *outData); + virtual status_t encryptNative( + const sp<GraphicBuffer> &graphicBuffer, + size_t offset, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData); + virtual status_t decrypt( const void *inData, size_t size, uint32_t streamCTR, uint64_t outInputCTR, void *outData); diff --git a/media/libmediaplayerservice/MediaPlayerService.cpp b/media/libmediaplayerservice/MediaPlayerService.cpp index 57ec7ea..9553458 100644 --- a/media/libmediaplayerservice/MediaPlayerService.cpp +++ b/media/libmediaplayerservice/MediaPlayerService.cpp @@ -53,6 +53,8 @@ #include <media/AudioTrack.h> #include <media/MemoryLeakTrackUtil.h> #include <media/stagefright/MediaErrors.h> +#include <media/stagefright/AudioPlayer.h> +#include <media/stagefright/foundation/ADebug.h> #include <system/audio.h> @@ -317,8 +319,8 @@ status_t MediaPlayerService::AudioCache::dump(int fd, const Vector<String16>& ar result.append(" AudioCache\n"); if (mHeap != 0) { - snprintf(buffer, 255, " heap base(%p), size(%d), flags(%d), device(%s)\n", - mHeap->getBase(), mHeap->getSize(), mHeap->getFlags(), mHeap->getDevice()); + snprintf(buffer, 255, " heap base(%p), size(%d), flags(%d)\n", + mHeap->getBase(), mHeap->getSize(), mHeap->getFlags()); result.append(buffer); } snprintf(buffer, 255, " msec per frame(%f), channel count(%d), format(%d), frame count(%zd)\n", @@ -742,7 +744,7 @@ status_t MediaPlayerService::Client::setVideoSurfaceTexture( sp<ANativeWindow> anw; if (bufferProducer != NULL) { - anw = new Surface(bufferProducer); + anw = new Surface(bufferProducer, true /* controlledByApp */); status_t err = native_window_api_connect(anw.get(), NATIVE_WINDOW_API_MEDIA); @@ -1174,13 +1176,13 @@ int Antagonizer::callbackThread(void* user) } #endif -static size_t kDefaultHeapSize = 1024 * 1024; // 1MB - -sp<IMemory> MediaPlayerService::decode(const char* url, uint32_t *pSampleRate, int* pNumChannels, audio_format_t* pFormat) +status_t MediaPlayerService::decode(const char* url, uint32_t *pSampleRate, int* pNumChannels, + audio_format_t* pFormat, + const sp<IMemoryHeap>& heap, size_t *pSize) { ALOGV("decode(%s)", url); - sp<MemoryBase> mem; sp<MediaPlayerBase> player; + status_t status = BAD_VALUE; // Protect our precious, precious DRMd ringtones by only allowing // decoding of http, but not filesystem paths or content Uris. @@ -1188,7 +1190,7 @@ sp<IMemory> MediaPlayerService::decode(const char* url, uint32_t *pSampleRate, i // filedescriptor for them and use that. if (url != NULL && strncmp(url, "http://", 7) != 0) { ALOGD("Can't decode %s by path, use filedescriptor instead", url); - return mem; + return BAD_VALUE; } player_type playerType = @@ -1196,7 +1198,7 @@ sp<IMemory> MediaPlayerService::decode(const char* url, uint32_t *pSampleRate, i ALOGV("player type = %d", playerType); // create the right type of player - sp<AudioCache> cache = new AudioCache(url); + sp<AudioCache> cache = new AudioCache(heap); player = MediaPlayerFactory::createPlayer(playerType, cache.get(), cache->notify); if (player == NULL) goto Exit; if (player->hardwareOutput()) goto Exit; @@ -1222,22 +1224,27 @@ sp<IMemory> MediaPlayerService::decode(const char* url, uint32_t *pSampleRate, i goto Exit; } - mem = new MemoryBase(cache->getHeap(), 0, cache->size()); + *pSize = cache->size(); *pSampleRate = cache->sampleRate(); *pNumChannels = cache->channelCount(); *pFormat = cache->format(); - ALOGV("return memory @ %p, sampleRate=%u, channelCount = %d, format = %d", mem->pointer(), *pSampleRate, *pNumChannels, *pFormat); + ALOGV("return size %d sampleRate=%u, channelCount = %d, format = %d", + *pSize, *pSampleRate, *pNumChannels, *pFormat); + status = NO_ERROR; Exit: if (player != 0) player->reset(); - return mem; + return status; } -sp<IMemory> MediaPlayerService::decode(int fd, int64_t offset, int64_t length, uint32_t *pSampleRate, int* pNumChannels, audio_format_t* pFormat) +status_t MediaPlayerService::decode(int fd, int64_t offset, int64_t length, + uint32_t *pSampleRate, int* pNumChannels, + audio_format_t* pFormat, + const sp<IMemoryHeap>& heap, size_t *pSize) { ALOGV("decode(%d, %lld, %lld)", fd, offset, length); - sp<MemoryBase> mem; sp<MediaPlayerBase> player; + status_t status = BAD_VALUE; player_type playerType = MediaPlayerFactory::getPlayerType(NULL /* client */, fd, @@ -1246,7 +1253,7 @@ sp<IMemory> MediaPlayerService::decode(int fd, int64_t offset, int64_t length, u ALOGV("player type = %d", playerType); // create the right type of player - sp<AudioCache> cache = new AudioCache("decode_fd"); + sp<AudioCache> cache = new AudioCache(heap); player = MediaPlayerFactory::createPlayer(playerType, cache.get(), cache->notify); if (player == NULL) goto Exit; if (player->hardwareOutput()) goto Exit; @@ -1272,16 +1279,18 @@ sp<IMemory> MediaPlayerService::decode(int fd, int64_t offset, int64_t length, u goto Exit; } - mem = new MemoryBase(cache->getHeap(), 0, cache->size()); + *pSize = cache->size(); *pSampleRate = cache->sampleRate(); *pNumChannels = cache->channelCount(); *pFormat = cache->format(); - ALOGV("return memory @ %p, sampleRate=%u, channelCount = %d, format = %d", mem->pointer(), *pSampleRate, *pNumChannels, *pFormat); + ALOGV("return size %d, sampleRate=%u, channelCount = %d, format = %d", + *pSize, *pSampleRate, *pNumChannels, *pFormat); + status = NO_ERROR; Exit: if (player != 0) player->reset(); ::close(fd); - return mem; + return status; } @@ -1295,8 +1304,6 @@ MediaPlayerService::AudioOutput::AudioOutput(int sessionId) mSessionId(sessionId), mFlags(AUDIO_OUTPUT_FLAG_NONE) { ALOGV("AudioOutput(%d)", sessionId); - mTrack = 0; - mRecycledTrack = 0; mStreamType = AUDIO_STREAM_MUSIC; mLeftVolume = 1.0; mRightVolume = 1.0; @@ -1311,7 +1318,6 @@ MediaPlayerService::AudioOutput::AudioOutput(int sessionId) MediaPlayerService::AudioOutput::~AudioOutput() { close(); - delete mRecycledTrack; delete mCallbackData; } @@ -1384,11 +1390,51 @@ status_t MediaPlayerService::AudioOutput::getFramesWritten(uint32_t *frameswritt return OK; } +status_t MediaPlayerService::AudioOutput::setParameters(const String8& keyValuePairs) +{ + if (mTrack == 0) return NO_INIT; + return mTrack->setParameters(keyValuePairs); +} + +String8 MediaPlayerService::AudioOutput::getParameters(const String8& keys) +{ + if (mTrack == 0) return String8::empty(); + return mTrack->getParameters(keys); +} + +void MediaPlayerService::AudioOutput::deleteRecycledTrack() +{ + ALOGV("deleteRecycledTrack"); + + if (mRecycledTrack != 0) { + + if (mCallbackData != NULL) { + mCallbackData->setOutput(NULL); + mCallbackData->endTrackSwitch(); + } + + if ((mRecycledTrack->getFlags() & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) == 0) { + mRecycledTrack->flush(); + } + // An offloaded track isn't flushed because the STREAM_END is reported + // slightly prematurely to allow time for the gapless track switch + // but this means that if we decide not to recycle the track there + // could be a small amount of residual data still playing. We leave + // AudioFlinger to drain the track. + + mRecycledTrack.clear(); + delete mCallbackData; + mCallbackData = NULL; + close(); + } +} + status_t MediaPlayerService::AudioOutput::open( uint32_t sampleRate, int channelCount, audio_channel_mask_t channelMask, audio_format_t format, int bufferCount, AudioCallback cb, void *cookie, - audio_output_flags_t flags) + audio_output_flags_t flags, + const audio_offload_info_t *offloadInfo) { mCallback = cb; mCallbackCookie = cookie; @@ -1399,20 +1445,34 @@ status_t MediaPlayerService::AudioOutput::open( bufferCount = mMinBufferCount; } - ALOGV("open(%u, %d, 0x%x, %d, %d, %d)", sampleRate, channelCount, channelMask, - format, bufferCount, mSessionId); + ALOGV("open(%u, %d, 0x%x, 0x%x, %d, %d 0x%x)", sampleRate, channelCount, channelMask, + format, bufferCount, mSessionId, flags); uint32_t afSampleRate; size_t afFrameCount; uint32_t frameCount; - if (AudioSystem::getOutputFrameCount(&afFrameCount, mStreamType) != NO_ERROR) { - return NO_INIT; - } - if (AudioSystem::getOutputSamplingRate(&afSampleRate, mStreamType) != NO_ERROR) { - return NO_INIT; + // offloading is only supported in callback mode for now. + // offloadInfo must be present if offload flag is set + if (((flags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) != 0) && + ((cb == NULL) || (offloadInfo == NULL))) { + return BAD_VALUE; } - frameCount = (sampleRate*afFrameCount*bufferCount)/afSampleRate; + if ((flags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) != 0) { + frameCount = 0; // AudioTrack will get frame count from AudioFlinger + } else { + uint32_t afSampleRate; + size_t afFrameCount; + + if (AudioSystem::getOutputFrameCount(&afFrameCount, mStreamType) != NO_ERROR) { + return NO_INIT; + } + if (AudioSystem::getOutputSamplingRate(&afSampleRate, mStreamType) != NO_ERROR) { + return NO_INIT; + } + + frameCount = (sampleRate*afFrameCount*bufferCount)/afSampleRate; + } if (channelMask == CHANNEL_MASK_USE_CHANNEL_ORDER) { channelMask = audio_channel_out_mask_from_count(channelCount); @@ -1422,90 +1482,127 @@ status_t MediaPlayerService::AudioOutput::open( } } - AudioTrack *t; - CallbackData *newcbd = NULL; - if (mCallback != NULL) { - newcbd = new CallbackData(this); - t = new AudioTrack( - mStreamType, - sampleRate, - format, - channelMask, - frameCount, - flags, - CallbackWrapper, - newcbd, - 0, // notification frames - mSessionId); - } else { - t = new AudioTrack( - mStreamType, - sampleRate, - format, - channelMask, - frameCount, - flags, - NULL, - NULL, - 0, - mSessionId); - } - - if ((t == 0) || (t->initCheck() != NO_ERROR)) { - ALOGE("Unable to create audio track"); - delete t; - delete newcbd; - return NO_INIT; - } + // Check whether we can recycle the track + bool reuse = false; + bool bothOffloaded = false; + if (mRecycledTrack != 0) { + // check whether we are switching between two offloaded tracks + bothOffloaded = (flags & mRecycledTrack->getFlags() + & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) != 0; - if (mRecycledTrack) { // check if the existing track can be reused as-is, or if a new track needs to be created. + reuse = true; - bool reuse = true; if ((mCallbackData == NULL && mCallback != NULL) || (mCallbackData != NULL && mCallback == NULL)) { // recycled track uses callbacks but the caller wants to use writes, or vice versa ALOGV("can't chain callback and write"); reuse = false; } else if ((mRecycledTrack->getSampleRate() != sampleRate) || - (mRecycledTrack->channelCount() != channelCount) || - (mRecycledTrack->frameCount() != t->frameCount())) { - ALOGV("samplerate, channelcount or framecount differ: %d/%d Hz, %d/%d ch, %d/%d frames", + (mRecycledTrack->channelCount() != (uint32_t)channelCount) ) { + ALOGV("samplerate, channelcount differ: %u/%u Hz, %u/%d ch", mRecycledTrack->getSampleRate(), sampleRate, - mRecycledTrack->channelCount(), channelCount, - mRecycledTrack->frameCount(), t->frameCount()); + mRecycledTrack->channelCount(), channelCount); reuse = false; } else if (flags != mFlags) { ALOGV("output flags differ %08x/%08x", flags, mFlags); reuse = false; + } else if (mRecycledTrack->format() != format) { + reuse = false; } + } else { + ALOGV("no track available to recycle"); + } + + ALOGV_IF(bothOffloaded, "both tracks offloaded"); + + // If we can't recycle and both tracks are offloaded + // we must close the previous output before opening a new one + if (bothOffloaded && !reuse) { + ALOGV("both offloaded and not recycling"); + deleteRecycledTrack(); + } + + sp<AudioTrack> t; + CallbackData *newcbd = NULL; + + // We don't attempt to create a new track if we are recycling an + // offloaded track. But, if we are recycling a non-offloaded or we + // are switching where one is offloaded and one isn't then we create + // the new track in advance so that we can read additional stream info + + if (!(reuse && bothOffloaded)) { + ALOGV("creating new AudioTrack"); + + if (mCallback != NULL) { + newcbd = new CallbackData(this); + t = new AudioTrack( + mStreamType, + sampleRate, + format, + channelMask, + frameCount, + flags, + CallbackWrapper, + newcbd, + 0, // notification frames + mSessionId, + AudioTrack::TRANSFER_CALLBACK, + offloadInfo); + } else { + t = new AudioTrack( + mStreamType, + sampleRate, + format, + channelMask, + frameCount, + flags, + NULL, + NULL, + 0, + mSessionId); + } + + if ((t == 0) || (t->initCheck() != NO_ERROR)) { + ALOGE("Unable to create audio track"); + delete newcbd; + return NO_INIT; + } + } + + if (reuse) { + CHECK(mRecycledTrack != NULL); + + if (!bothOffloaded) { + if (mRecycledTrack->frameCount() != t->frameCount()) { + ALOGV("framecount differs: %u/%u frames", + mRecycledTrack->frameCount(), t->frameCount()); + reuse = false; + } + } + if (reuse) { - ALOGV("chaining to next output"); + ALOGV("chaining to next output and recycling track"); close(); mTrack = mRecycledTrack; - mRecycledTrack = NULL; + mRecycledTrack.clear(); if (mCallbackData != NULL) { mCallbackData->setOutput(this); } - delete t; delete newcbd; return OK; } + } - // if we're not going to reuse the track, unblock and flush it - if (mCallbackData != NULL) { - mCallbackData->setOutput(NULL); - mCallbackData->endTrackSwitch(); - } - mRecycledTrack->flush(); - delete mRecycledTrack; - mRecycledTrack = NULL; - delete mCallbackData; - mCallbackData = NULL; - close(); + // we're not going to reuse the track, unblock and flush it + // this was done earlier if both tracks are offloaded + if (!bothOffloaded) { + deleteRecycledTrack(); } + CHECK((t != NULL) && ((mCallback == NULL) || (newcbd != NULL))); + mCallbackData = newcbd; ALOGV("setVolume"); t->setVolume(mLeftVolume, mRightVolume); @@ -1519,25 +1616,30 @@ status_t MediaPlayerService::AudioOutput::open( } mTrack = t; - status_t res = t->setSampleRate(mPlaybackRatePermille * mSampleRateHz / 1000); - if (res != NO_ERROR) { - return res; + status_t res = NO_ERROR; + if ((flags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) == 0) { + res = t->setSampleRate(mPlaybackRatePermille * mSampleRateHz / 1000); + if (res == NO_ERROR) { + t->setAuxEffectSendLevel(mSendLevel); + res = t->attachAuxEffect(mAuxEffectId); + } } - t->setAuxEffectSendLevel(mSendLevel); - return t->attachAuxEffect(mAuxEffectId);; + ALOGV("open() DONE status %d", res); + return res; } -void MediaPlayerService::AudioOutput::start() +status_t MediaPlayerService::AudioOutput::start() { ALOGV("start"); if (mCallbackData != NULL) { mCallbackData->endTrackSwitch(); } - if (mTrack) { + if (mTrack != 0) { mTrack->setVolume(mLeftVolume, mRightVolume); mTrack->setAuxEffectSendLevel(mSendLevel); - mTrack->start(); + return mTrack->start(); } + return NO_INIT; } void MediaPlayerService::AudioOutput::setNextOutput(const sp<AudioOutput>& nextOutput) { @@ -1555,7 +1657,7 @@ void MediaPlayerService::AudioOutput::switchToNextOutput() { mNextOutput->mCallbackData = mCallbackData; mCallbackData = NULL; mNextOutput->mRecycledTrack = mTrack; - mTrack = NULL; + mTrack.clear(); mNextOutput->mSampleRateHz = mSampleRateHz; mNextOutput->mMsecsPerFrame = mMsecsPerFrame; mNextOutput->mBytesWritten = mBytesWritten; @@ -1568,7 +1670,7 @@ ssize_t MediaPlayerService::AudioOutput::write(const void* buffer, size_t size) LOG_FATAL_IF(mCallback != NULL, "Don't call write if supplying a callback."); //ALOGV("write(%p, %u)", buffer, size); - if (mTrack) { + if (mTrack != 0) { ssize_t ret = mTrack->write(buffer, size); mBytesWritten += ret; return ret; @@ -1579,26 +1681,25 @@ ssize_t MediaPlayerService::AudioOutput::write(const void* buffer, size_t size) void MediaPlayerService::AudioOutput::stop() { ALOGV("stop"); - if (mTrack) mTrack->stop(); + if (mTrack != 0) mTrack->stop(); } void MediaPlayerService::AudioOutput::flush() { ALOGV("flush"); - if (mTrack) mTrack->flush(); + if (mTrack != 0) mTrack->flush(); } void MediaPlayerService::AudioOutput::pause() { ALOGV("pause"); - if (mTrack) mTrack->pause(); + if (mTrack != 0) mTrack->pause(); } void MediaPlayerService::AudioOutput::close() { ALOGV("close"); - delete mTrack; - mTrack = 0; + mTrack.clear(); } void MediaPlayerService::AudioOutput::setVolume(float left, float right) @@ -1606,7 +1707,7 @@ void MediaPlayerService::AudioOutput::setVolume(float left, float right) ALOGV("setVolume(%f, %f)", left, right); mLeftVolume = left; mRightVolume = right; - if (mTrack) { + if (mTrack != 0) { mTrack->setVolume(left, right); } } @@ -1615,7 +1716,7 @@ status_t MediaPlayerService::AudioOutput::setPlaybackRatePermille(int32_t ratePe { ALOGV("setPlaybackRatePermille(%d)", ratePermille); status_t res = NO_ERROR; - if (mTrack) { + if (mTrack != 0) { res = mTrack->setSampleRate(ratePermille * mSampleRateHz / 1000); } else { res = NO_INIT; @@ -1631,7 +1732,7 @@ status_t MediaPlayerService::AudioOutput::setAuxEffectSendLevel(float level) { ALOGV("setAuxEffectSendLevel(%f)", level); mSendLevel = level; - if (mTrack) { + if (mTrack != 0) { return mTrack->setAuxEffectSendLevel(level); } return NO_ERROR; @@ -1641,7 +1742,7 @@ status_t MediaPlayerService::AudioOutput::attachAuxEffect(int effectId) { ALOGV("attachAuxEffect(%d)", effectId); mAuxEffectId = effectId; - if (mTrack) { + if (mTrack != 0) { return mTrack->attachAuxEffect(effectId); } return NO_ERROR; @@ -1651,10 +1752,6 @@ status_t MediaPlayerService::AudioOutput::attachAuxEffect(int effectId) void MediaPlayerService::AudioOutput::CallbackWrapper( int event, void *cookie, void *info) { //ALOGV("callbackwrapper"); - if (event != AudioTrack::EVENT_MORE_DATA) { - return; - } - CallbackData *data = (CallbackData*)cookie; data->lock(); AudioOutput *me = data->getOutput(); @@ -1663,22 +1760,46 @@ void MediaPlayerService::AudioOutput::CallbackWrapper( // no output set, likely because the track was scheduled to be reused // by another player, but the format turned out to be incompatible. data->unlock(); - buffer->size = 0; + if (buffer != NULL) { + buffer->size = 0; + } return; } - size_t actualSize = (*me->mCallback)( - me, buffer->raw, buffer->size, me->mCallbackCookie); + switch(event) { + case AudioTrack::EVENT_MORE_DATA: { + size_t actualSize = (*me->mCallback)( + me, buffer->raw, buffer->size, me->mCallbackCookie, + CB_EVENT_FILL_BUFFER); + + if (actualSize == 0 && buffer->size > 0 && me->mNextOutput == NULL) { + // We've reached EOS but the audio track is not stopped yet, + // keep playing silence. + + memset(buffer->raw, 0, buffer->size); + actualSize = buffer->size; + } + + buffer->size = actualSize; + } break; - if (actualSize == 0 && buffer->size > 0 && me->mNextOutput == NULL) { - // We've reached EOS but the audio track is not stopped yet, - // keep playing silence. - memset(buffer->raw, 0, buffer->size); - actualSize = buffer->size; + case AudioTrack::EVENT_STREAM_END: + ALOGV("callbackwrapper: deliver EVENT_STREAM_END"); + (*me->mCallback)(me, NULL /* buffer */, 0 /* size */, + me->mCallbackCookie, CB_EVENT_STREAM_END); + break; + + case AudioTrack::EVENT_NEW_IAUDIOTRACK : + ALOGV("callbackwrapper: deliver EVENT_TEAR_DOWN"); + (*me->mCallback)(me, NULL /* buffer */, 0 /* size */, + me->mCallbackCookie, CB_EVENT_TEAR_DOWN); + break; + + default: + ALOGE("received unknown event type: %d inside CallbackWrapper !", event); } - buffer->size = actualSize; data->unlock(); } @@ -1689,12 +1810,10 @@ int MediaPlayerService::AudioOutput::getSessionId() const #undef LOG_TAG #define LOG_TAG "AudioCache" -MediaPlayerService::AudioCache::AudioCache(const char* name) : - mChannelCount(0), mFrameCount(1024), mSampleRate(0), mSize(0), - mError(NO_ERROR), mCommandComplete(false) +MediaPlayerService::AudioCache::AudioCache(const sp<IMemoryHeap>& heap) : + mHeap(heap), mChannelCount(0), mFrameCount(1024), mSampleRate(0), mSize(0), + mError(NO_ERROR), mCommandComplete(false) { - // create ashmem heap - mHeap = new MemoryHeapBase(kDefaultHeapSize, 0, name); } uint32_t MediaPlayerService::AudioCache::latency () const @@ -1774,7 +1893,8 @@ bool CallbackThread::threadLoop() { } size_t actualSize = - (*mCallback)(sink.get(), mBuffer, mBufferSize, mCookie); + (*mCallback)(sink.get(), mBuffer, mBufferSize, mCookie, + MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER); if (actualSize > 0) { sink->write(mBuffer, actualSize); @@ -1788,7 +1908,8 @@ bool CallbackThread::threadLoop() { status_t MediaPlayerService::AudioCache::open( uint32_t sampleRate, int channelCount, audio_channel_mask_t channelMask, audio_format_t format, int bufferCount, - AudioCallback cb, void *cookie, audio_output_flags_t flags) + AudioCallback cb, void *cookie, audio_output_flags_t flags, + const audio_offload_info_t *offloadInfo) { ALOGV("open(%u, %d, 0x%x, %d, %d)", sampleRate, channelCount, channelMask, format, bufferCount); if (mHeap->getHeapID() < 0) { @@ -1806,10 +1927,11 @@ status_t MediaPlayerService::AudioCache::open( return NO_ERROR; } -void MediaPlayerService::AudioCache::start() { +status_t MediaPlayerService::AudioCache::start() { if (mCallbackThread != NULL) { mCallbackThread->run("AudioCache callback"); } + return NO_ERROR; } void MediaPlayerService::AudioCache::stop() { diff --git a/media/libmediaplayerservice/MediaPlayerService.h b/media/libmediaplayerservice/MediaPlayerService.h index b33805d..21f4117 100644 --- a/media/libmediaplayerservice/MediaPlayerService.h +++ b/media/libmediaplayerservice/MediaPlayerService.h @@ -20,15 +20,12 @@ #include <arpa/inet.h> -#include <utils/Log.h> #include <utils/threads.h> -#include <utils/List.h> #include <utils/Errors.h> #include <utils/KeyedVector.h> #include <utils/String8.h> #include <utils/Vector.h> -#include <media/IMediaPlayerService.h> #include <media/MediaPlayerInterface.h> #include <media/Metadata.h> #include <media/stagefright/foundation/ABase.h> @@ -78,7 +75,7 @@ class MediaPlayerService : public BnMediaPlayerService AudioOutput(int sessionId); virtual ~AudioOutput(); - virtual bool ready() const { return mTrack != NULL; } + virtual bool ready() const { return mTrack != 0; } virtual bool realtime() const { return true; } virtual ssize_t bufferSize() const; virtual ssize_t frameCount() const; @@ -94,9 +91,10 @@ class MediaPlayerService : public BnMediaPlayerService uint32_t sampleRate, int channelCount, audio_channel_mask_t channelMask, audio_format_t format, int bufferCount, AudioCallback cb, void *cookie, - audio_output_flags_t flags = AUDIO_OUTPUT_FLAG_NONE); + audio_output_flags_t flags = AUDIO_OUTPUT_FLAG_NONE, + const audio_offload_info_t *offloadInfo = NULL); - virtual void start(); + virtual status_t start(); virtual ssize_t write(const void* buffer, size_t size); virtual void stop(); virtual void flush(); @@ -114,14 +112,17 @@ class MediaPlayerService : public BnMediaPlayerService void setNextOutput(const sp<AudioOutput>& nextOutput); void switchToNextOutput(); virtual bool needsTrailingPadding() { return mNextOutput == NULL; } + virtual status_t setParameters(const String8& keyValuePairs); + virtual String8 getParameters(const String8& keys); private: static void setMinBufferCount(); static void CallbackWrapper( int event, void *me, void *info); + void deleteRecycledTrack(); - AudioTrack* mTrack; - AudioTrack* mRecycledTrack; + sp<AudioTrack> mTrack; + sp<AudioTrack> mRecycledTrack; sp<AudioOutput> mNextOutput; AudioCallback mCallback; void * mCallbackCookie; @@ -176,7 +177,7 @@ class MediaPlayerService : public BnMediaPlayerService class AudioCache : public MediaPlayerBase::AudioSink { public: - AudioCache(const char* name); + AudioCache(const sp<IMemoryHeap>& heap); virtual ~AudioCache() {} virtual bool ready() const { return (mChannelCount > 0) && (mHeap->getHeapID() > 0); } @@ -195,9 +196,10 @@ class MediaPlayerService : public BnMediaPlayerService uint32_t sampleRate, int channelCount, audio_channel_mask_t channelMask, audio_format_t format, int bufferCount = 1, AudioCallback cb = NULL, void *cookie = NULL, - audio_output_flags_t flags = AUDIO_OUTPUT_FLAG_NONE); + audio_output_flags_t flags = AUDIO_OUTPUT_FLAG_NONE, + const audio_offload_info_t *offloadInfo = NULL); - virtual void start(); + virtual status_t start(); virtual ssize_t write(const void* buffer, size_t size); virtual void stop(); virtual void flush() {} @@ -222,7 +224,7 @@ class MediaPlayerService : public BnMediaPlayerService Mutex mLock; Condition mSignal; - sp<MemoryHeapBase> mHeap; + sp<IMemoryHeap> mHeap; float mMsecsPerFrame; uint16_t mChannelCount; audio_format_t mFormat; @@ -245,8 +247,13 @@ public: virtual sp<IMediaPlayer> create(const sp<IMediaPlayerClient>& client, int audioSessionId); - virtual sp<IMemory> decode(const char* url, uint32_t *pSampleRate, int* pNumChannels, audio_format_t* pFormat); - virtual sp<IMemory> decode(int fd, int64_t offset, int64_t length, uint32_t *pSampleRate, int* pNumChannels, audio_format_t* pFormat); + virtual status_t decode(const char* url, uint32_t *pSampleRate, int* pNumChannels, + audio_format_t* pFormat, + const sp<IMemoryHeap>& heap, size_t *pSize); + virtual status_t decode(int fd, int64_t offset, int64_t length, + uint32_t *pSampleRate, int* pNumChannels, + audio_format_t* pFormat, + const sp<IMemoryHeap>& heap, size_t *pSize); virtual sp<IOMX> getOMX(); virtual sp<ICrypto> makeCrypto(); virtual sp<IDrm> makeDrm(); diff --git a/media/libmediaplayerservice/MidiFile.cpp b/media/libmediaplayerservice/MidiFile.cpp index 8db5b9b..0a6aa90 100644 --- a/media/libmediaplayerservice/MidiFile.cpp +++ b/media/libmediaplayerservice/MidiFile.cpp @@ -220,6 +220,9 @@ status_t MidiFile::start() } mRender = true; + if (mState == EAS_STATE_PLAY) { + sendEvent(MEDIA_STARTED); + } // wake up render thread ALOGV(" wakeup render thread"); @@ -242,6 +245,7 @@ status_t MidiFile::stop() } } mPaused = false; + sendEvent(MEDIA_STOPPED); return NO_ERROR; } @@ -279,6 +283,7 @@ status_t MidiFile::pause() return ERROR_EAS_FAILURE; } mPaused = true; + sendEvent(MEDIA_PAUSED); return NO_ERROR; } @@ -382,6 +387,7 @@ status_t MidiFile::reset() status_t MidiFile::reset_nosync() { ALOGV("MidiFile::reset_nosync"); + sendEvent(MEDIA_STOPPED); // close file if (mEasHandle) { EAS_CloseFile(mEasData, mEasHandle); @@ -422,7 +428,7 @@ status_t MidiFile::setLooping(int loop) status_t MidiFile::createOutputTrack() { if (mAudioSink->open(pLibConfig->sampleRate, pLibConfig->numChannels, - CHANNEL_MASK_USE_CHANNEL_ORDER, AUDIO_FORMAT_PCM_16_BIT, 2) != NO_ERROR) { + CHANNEL_MASK_USE_CHANNEL_ORDER, AUDIO_FORMAT_PCM_16_BIT, 2 /*bufferCount*/) != NO_ERROR) { ALOGE("mAudioSink open failed"); return ERROR_OPEN_FAILED; } diff --git a/media/libmediaplayerservice/RemoteDisplay.cpp b/media/libmediaplayerservice/RemoteDisplay.cpp index 20e6513..eb959b4 100644 --- a/media/libmediaplayerservice/RemoteDisplay.cpp +++ b/media/libmediaplayerservice/RemoteDisplay.cpp @@ -16,19 +16,23 @@ #include "RemoteDisplay.h" -#include "ANetworkSession.h" #include "source/WifiDisplaySource.h" #include <media/IRemoteDisplayClient.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/foundation/ANetworkSession.h> namespace android { RemoteDisplay::RemoteDisplay( - const sp<IRemoteDisplayClient> &client, const char *iface) + const sp<IRemoteDisplayClient> &client, + const char *iface) : mLooper(new ALooper), - mNetSession(new ANetworkSession), - mSource(new WifiDisplaySource(mNetSession, client)) { + mNetSession(new ANetworkSession) { mLooper->setName("wfd_looper"); + + mSource = new WifiDisplaySource(mNetSession, client); mLooper->registerHandler(mSource); mNetSession->start(); @@ -50,6 +54,7 @@ status_t RemoteDisplay::resume() { status_t RemoteDisplay::dispose() { mSource->stop(); + mSource.clear(); mLooper->stop(); mNetSession->stop(); diff --git a/media/libmediaplayerservice/RemoteDisplay.h b/media/libmediaplayerservice/RemoteDisplay.h index bd8b684..82a0116 100644 --- a/media/libmediaplayerservice/RemoteDisplay.h +++ b/media/libmediaplayerservice/RemoteDisplay.h @@ -18,6 +18,7 @@ #define REMOTE_DISPLAY_H_ +#include <media/IMediaPlayerService.h> #include <media/IRemoteDisplay.h> #include <media/stagefright/foundation/ABase.h> #include <utils/Errors.h> @@ -31,7 +32,9 @@ struct IRemoteDisplayClient; struct WifiDisplaySource; struct RemoteDisplay : public BnRemoteDisplay { - RemoteDisplay(const sp<IRemoteDisplayClient> &client, const char *iface); + RemoteDisplay( + const sp<IRemoteDisplayClient> &client, + const char *iface); virtual status_t pause(); virtual status_t resume(); diff --git a/media/libmediaplayerservice/SharedLibrary.cpp b/media/libmediaplayerservice/SharedLibrary.cpp index 178e15d..34db761 100644 --- a/media/libmediaplayerservice/SharedLibrary.cpp +++ b/media/libmediaplayerservice/SharedLibrary.cpp @@ -46,4 +46,10 @@ namespace android { } return dlsym(mLibHandle, symbol); } + + const char *SharedLibrary::lastError() const { + const char *error = dlerror(); + return error ? error : "No errors or unknown error"; + } + }; diff --git a/media/libmediaplayerservice/SharedLibrary.h b/media/libmediaplayerservice/SharedLibrary.h index 5353642..88451a0 100644 --- a/media/libmediaplayerservice/SharedLibrary.h +++ b/media/libmediaplayerservice/SharedLibrary.h @@ -29,6 +29,7 @@ namespace android { bool operator!() const; void *lookup(const char *symbol) const; + const char *lastError() const; private: void *mLibHandle; diff --git a/media/libmediaplayerservice/nuplayer/HTTPLiveSource.cpp b/media/libmediaplayerservice/nuplayer/HTTPLiveSource.cpp index 655ee55..d8b35d7 100644 --- a/media/libmediaplayerservice/nuplayer/HTTPLiveSource.cpp +++ b/media/libmediaplayerservice/nuplayer/HTTPLiveSource.cpp @@ -20,7 +20,6 @@ #include "HTTPLiveSource.h" -#include "ATSParser.h" #include "AnotherPacketSource.h" #include "LiveDataSource.h" #include "LiveSession.h" @@ -44,7 +43,8 @@ NuPlayer::HTTPLiveSource::HTTPLiveSource( mUID(uid), mFlags(0), mFinalResult(OK), - mOffset(0) { + mOffset(0), + mFetchSubtitleDataGeneration(0) { if (headers) { mExtraHeaders = *headers; @@ -62,7 +62,10 @@ NuPlayer::HTTPLiveSource::HTTPLiveSource( NuPlayer::HTTPLiveSource::~HTTPLiveSource() { if (mLiveSession != NULL) { mLiveSession->disconnect(); + mLiveSession.clear(); + mLiveLooper->stop(); + mLiveLooper.clear(); } } @@ -76,128 +79,72 @@ void NuPlayer::HTTPLiveSource::prepareAsync() { mLiveSession = new LiveSession( notify, (mFlags & kFlagIncognito) ? LiveSession::kFlagIncognito : 0, - mUIDValid, mUID); + mUIDValid, + mUID); mLiveLooper->registerHandler(mLiveSession); - mLiveSession->connect( + mLiveSession->connectAsync( mURL.c_str(), mExtraHeaders.isEmpty() ? NULL : &mExtraHeaders); - - mTSParser = new ATSParser; } void NuPlayer::HTTPLiveSource::start() { } -sp<MetaData> NuPlayer::HTTPLiveSource::getFormatMeta(bool audio) { - ATSParser::SourceType type = - audio ? ATSParser::AUDIO : ATSParser::VIDEO; +sp<AMessage> NuPlayer::HTTPLiveSource::getFormat(bool audio) { + sp<AMessage> format; + status_t err = mLiveSession->getStreamFormat( + audio ? LiveSession::STREAMTYPE_AUDIO + : LiveSession::STREAMTYPE_VIDEO, + &format); - sp<AnotherPacketSource> source = - static_cast<AnotherPacketSource *>(mTSParser->getSource(type).get()); - - if (source == NULL) { + if (err != OK) { return NULL; } - return source->getFormat(); + return format; } status_t NuPlayer::HTTPLiveSource::feedMoreTSData() { - if (mFinalResult != OK) { - return mFinalResult; - } - - sp<LiveDataSource> source = - static_cast<LiveDataSource *>(mLiveSession->getDataSource().get()); - - for (int32_t i = 0; i < 50; ++i) { - char buffer[188]; - ssize_t n = source->readAtNonBlocking(mOffset, buffer, sizeof(buffer)); - - if (n == -EWOULDBLOCK) { - break; - } else if (n < 0) { - if (n != ERROR_END_OF_STREAM) { - ALOGI("input data EOS reached, error %ld", n); - } else { - ALOGI("input data EOS reached."); - } - mTSParser->signalEOS(n); - mFinalResult = n; - break; - } else { - if (buffer[0] == 0x00) { - // XXX legacy - - uint8_t type = buffer[1]; - - sp<AMessage> extra = new AMessage; - - if (type & 2) { - int64_t mediaTimeUs; - memcpy(&mediaTimeUs, &buffer[2], sizeof(mediaTimeUs)); - - extra->setInt64(IStreamListener::kKeyMediaTimeUs, mediaTimeUs); - } - - mTSParser->signalDiscontinuity( - ((type & 1) == 0) - ? ATSParser::DISCONTINUITY_SEEK - : ATSParser::DISCONTINUITY_FORMATCHANGE, - extra); - } else { - status_t err = mTSParser->feedTSPacket(buffer, sizeof(buffer)); - - if (err != OK) { - ALOGE("TS Parser returned error %d", err); - mTSParser->signalEOS(err); - mFinalResult = err; - break; - } - } - - mOffset += n; - } - } - return OK; } status_t NuPlayer::HTTPLiveSource::dequeueAccessUnit( bool audio, sp<ABuffer> *accessUnit) { - ATSParser::SourceType type = - audio ? ATSParser::AUDIO : ATSParser::VIDEO; - - sp<AnotherPacketSource> source = - static_cast<AnotherPacketSource *>(mTSParser->getSource(type).get()); - - if (source == NULL) { - return -EWOULDBLOCK; - } - - status_t finalResult; - if (!source->hasBufferAvailable(&finalResult)) { - return finalResult == OK ? -EWOULDBLOCK : finalResult; - } - - return source->dequeueAccessUnit(accessUnit); + return mLiveSession->dequeueAccessUnit( + audio ? LiveSession::STREAMTYPE_AUDIO + : LiveSession::STREAMTYPE_VIDEO, + accessUnit); } status_t NuPlayer::HTTPLiveSource::getDuration(int64_t *durationUs) { return mLiveSession->getDuration(durationUs); } -status_t NuPlayer::HTTPLiveSource::seekTo(int64_t seekTimeUs) { - // We need to make sure we're not seeking until we have seen the very first - // PTS timestamp in the whole stream (from the beginning of the stream). - while (!mTSParser->PTSTimeDeltaEstablished() && feedMoreTSData() == OK) { - usleep(100000); +status_t NuPlayer::HTTPLiveSource::getTrackInfo(Parcel *reply) const { + return mLiveSession->getTrackInfo(reply); +} + +status_t NuPlayer::HTTPLiveSource::selectTrack(size_t trackIndex, bool select) { + status_t err = mLiveSession->selectTrack(trackIndex, select); + + if (err == OK) { + mFetchSubtitleDataGeneration++; + if (select) { + sp<AMessage> msg = new AMessage(kWhatFetchSubtitleData, id()); + msg->setInt32("generation", mFetchSubtitleDataGeneration); + msg->post(); + } } - mLiveSession->seekTo(seekTimeUs); + // LiveSession::selectTrack returns BAD_VALUE when selecting the currently + // selected track, or unselecting a non-selected track. In this case it's an + // no-op so we return OK. + return (err == OK || err == BAD_VALUE) ? OK : err; +} - return OK; +status_t NuPlayer::HTTPLiveSource::seekTo(int64_t seekTimeUs) { + return mLiveSession->seekTo(seekTimeUs); } void NuPlayer::HTTPLiveSource::onMessageReceived(const sp<AMessage> &msg) { @@ -208,6 +155,39 @@ void NuPlayer::HTTPLiveSource::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatFetchSubtitleData: + { + int32_t generation; + CHECK(msg->findInt32("generation", &generation)); + + if (generation != mFetchSubtitleDataGeneration) { + // stale + break; + } + + sp<ABuffer> buffer; + if (mLiveSession->dequeueAccessUnit( + LiveSession::STREAMTYPE_SUBTITLES, &buffer) == OK) { + sp<AMessage> notify = dupNotify(); + notify->setInt32("what", kWhatSubtitleData); + notify->setBuffer("buffer", buffer); + notify->post(); + + int64_t timeUs, baseUs, durationUs, delayUs; + CHECK(buffer->meta()->findInt64("baseUs", &baseUs)); + CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); + CHECK(buffer->meta()->findInt64("durationUs", &durationUs)); + delayUs = baseUs + timeUs - ALooper::GetNowUs(); + + msg->post(delayUs > 0ll ? delayUs : 0ll); + } else { + // try again in 1 second + msg->post(1000000ll); + } + + break; + } + default: Source::onMessageReceived(msg); break; @@ -249,6 +229,32 @@ void NuPlayer::HTTPLiveSource::onSessionNotify(const sp<AMessage> &msg) { break; } + case LiveSession::kWhatStreamsChanged: + { + uint32_t changedMask; + CHECK(msg->findInt32( + "changedMask", (int32_t *)&changedMask)); + + bool audio = changedMask & LiveSession::STREAMTYPE_AUDIO; + bool video = changedMask & LiveSession::STREAMTYPE_VIDEO; + + sp<AMessage> reply; + CHECK(msg->findMessage("reply", &reply)); + + sp<AMessage> notify = dupNotify(); + notify->setInt32("what", kWhatQueueDecoderShutdown); + notify->setInt32("audio", audio); + notify->setInt32("video", video); + notify->setMessage("reply", reply); + notify->post(); + break; + } + + case LiveSession::kWhatError: + { + break; + } + default: TRESPASS(); } diff --git a/media/libmediaplayerservice/nuplayer/HTTPLiveSource.h b/media/libmediaplayerservice/nuplayer/HTTPLiveSource.h index 067d1da..bcc3f8b 100644 --- a/media/libmediaplayerservice/nuplayer/HTTPLiveSource.h +++ b/media/libmediaplayerservice/nuplayer/HTTPLiveSource.h @@ -23,7 +23,6 @@ namespace android { -struct ATSParser; struct LiveSession; struct NuPlayer::HTTPLiveSource : public NuPlayer::Source { @@ -37,18 +36,18 @@ struct NuPlayer::HTTPLiveSource : public NuPlayer::Source { virtual void prepareAsync(); virtual void start(); - virtual status_t feedMoreTSData(); - virtual status_t dequeueAccessUnit(bool audio, sp<ABuffer> *accessUnit); + virtual sp<AMessage> getFormat(bool audio); + virtual status_t feedMoreTSData(); virtual status_t getDuration(int64_t *durationUs); + virtual status_t getTrackInfo(Parcel *reply) const; + virtual status_t selectTrack(size_t trackIndex, bool select); virtual status_t seekTo(int64_t seekTimeUs); protected: virtual ~HTTPLiveSource(); - virtual sp<MetaData> getFormatMeta(bool audio); - virtual void onMessageReceived(const sp<AMessage> &msg); private: @@ -59,6 +58,7 @@ private: enum { kWhatSessionNotify, + kWhatFetchSubtitleData, }; AString mURL; @@ -70,7 +70,7 @@ private: off64_t mOffset; sp<ALooper> mLiveLooper; sp<LiveSession> mLiveSession; - sp<ATSParser> mTSParser; + int32_t mFetchSubtitleDataGeneration; void onSessionNotify(const sp<AMessage> &msg); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp index b89b1c8..750287f 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp @@ -89,6 +89,38 @@ private: DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction); }; +struct NuPlayer::ShutdownDecoderAction : public Action { + ShutdownDecoderAction(bool audio, bool video) + : mAudio(audio), + mVideo(video) { + } + + virtual void execute(NuPlayer *player) { + player->performDecoderShutdown(mAudio, mVideo); + } + +private: + bool mAudio; + bool mVideo; + + DISALLOW_EVIL_CONSTRUCTORS(ShutdownDecoderAction); +}; + +struct NuPlayer::PostMessageAction : public Action { + PostMessageAction(const sp<AMessage> &msg) + : mMessage(msg) { + } + + virtual void execute(NuPlayer *) { + mMessage->post(); + } + +private: + sp<AMessage> mMessage; + + DISALLOW_EVIL_CONSTRUCTORS(PostMessageAction); +}; + // Use this if there's no state necessary to save in order to execute // the action. struct NuPlayer::SimpleAction : public Action { @@ -308,6 +340,46 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatGetTrackInfo: + { + uint32_t replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + + status_t err = INVALID_OPERATION; + if (mSource != NULL) { + Parcel* reply; + CHECK(msg->findPointer("reply", (void**)&reply)); + err = mSource->getTrackInfo(reply); + } + + sp<AMessage> response = new AMessage; + response->setInt32("err", err); + + response->postReply(replyID); + break; + } + + case kWhatSelectTrack: + { + uint32_t replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + + status_t err = INVALID_OPERATION; + if (mSource != NULL) { + size_t trackIndex; + int32_t select; + CHECK(msg->findSize("trackIndex", &trackIndex)); + CHECK(msg->findInt32("select", &select)); + err = mSource->selectTrack(trackIndex, select); + } + + sp<AMessage> response = new AMessage; + response->setInt32("err", err); + + response->postReply(replyID); + break; + } + case kWhatPollDuration: { int32_t generation; @@ -335,7 +407,8 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { ALOGV("kWhatSetVideoNativeWindow"); mDeferredActions.push_back( - new SimpleAction(&NuPlayer::performDecoderShutdown)); + new ShutdownDecoderAction( + false /* audio */, true /* video */)); sp<RefBase> obj; CHECK(msg->findObject("native-window", &obj)); @@ -698,6 +771,9 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { ALOGV("renderer %s flush completed.", audio ? "audio" : "video"); } else if (what == Renderer::kWhatVideoRenderingStart) { notifyListener(MEDIA_INFO, MEDIA_INFO_RENDERING_START, 0); + } else if (what == Renderer::kWhatMediaRenderingStart) { + ALOGV("media rendering started"); + notifyListener(MEDIA_STARTED, 0, 0); } break; } @@ -712,7 +788,8 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { ALOGV("kWhatReset"); mDeferredActions.push_back( - new SimpleAction(&NuPlayer::performDecoderShutdown)); + new ShutdownDecoderAction( + true /* audio */, true /* video */)); mDeferredActions.push_back( new SimpleAction(&NuPlayer::performReset)); @@ -1008,7 +1085,7 @@ void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) { mRenderer->queueBuffer(audio, buffer, reply); } -void NuPlayer::notifyListener(int msg, int ext1, int ext2) { +void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) { if (mDriver == NULL) { return; } @@ -1019,10 +1096,13 @@ void NuPlayer::notifyListener(int msg, int ext1, int ext2) { return; } - driver->notifyListener(msg, ext1, ext2); + driver->notifyListener(msg, ext1, ext2, in); } void NuPlayer::flushDecoder(bool audio, bool needShutdown) { + ALOGV("[%s] flushDecoder needShutdown=%d", + audio ? "audio" : "video", needShutdown); + if ((audio && mAudioDecoder == NULL) || (!audio && mVideoDecoder == NULL)) { ALOGI("flushDecoder %s without decoder present", audio ? "audio" : "video"); @@ -1092,6 +1172,26 @@ status_t NuPlayer::setVideoScalingMode(int32_t mode) { return OK; } +status_t NuPlayer::getTrackInfo(Parcel* reply) const { + sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, id()); + msg->setPointer("reply", reply); + + sp<AMessage> response; + status_t err = msg->postAndAwaitResponse(&response); + return err; +} + +status_t NuPlayer::selectTrack(size_t trackIndex, bool select) { + sp<AMessage> msg = new AMessage(kWhatSelectTrack, id()); + msg->setSize("trackIndex", trackIndex); + msg->setInt32("select", select); + + sp<AMessage> response; + status_t err = msg->postAndAwaitResponse(&response); + + return err; +} + void NuPlayer::schedulePollDuration() { sp<AMessage> msg = new AMessage(kWhatPollDuration, id()); msg->setInt32("generation", mPollDurationGeneration); @@ -1173,20 +1273,29 @@ void NuPlayer::performDecoderFlush() { } } -void NuPlayer::performDecoderShutdown() { - ALOGV("performDecoderShutdown"); +void NuPlayer::performDecoderShutdown(bool audio, bool video) { + ALOGV("performDecoderShutdown audio=%d, video=%d", audio, video); - if (mAudioDecoder == NULL && mVideoDecoder == NULL) { + if ((!audio || mAudioDecoder == NULL) + && (!video || mVideoDecoder == NULL)) { return; } mTimeDiscontinuityPending = true; - if (mAudioDecoder != NULL) { + if (mFlushingAudio == NONE && (!audio || mAudioDecoder == NULL)) { + mFlushingAudio = FLUSHED; + } + + if (mFlushingVideo == NONE && (!video || mVideoDecoder == NULL)) { + mFlushingVideo = FLUSHED; + } + + if (audio && mAudioDecoder != NULL) { flushDecoder(true /* audio */, true /* needShutdown */); } - if (mVideoDecoder != NULL) { + if (video && mVideoDecoder != NULL) { flushDecoder(false /* audio */, true /* needShutdown */); } } @@ -1287,6 +1396,11 @@ void NuPlayer::onSourceNotify(const sp<AMessage> &msg) { uint32_t flags; CHECK(msg->findInt32("flags", (int32_t *)&flags)); + sp<NuPlayerDriver> driver = mDriver.promote(); + if (driver != NULL) { + driver->notifyFlagsChanged(flags); + } + if ((mSourceFlags & Source::FLAG_DYNAMIC_DURATION) && (!(flags & Source::FLAG_DYNAMIC_DURATION))) { cancelPollDuration(); @@ -1322,6 +1436,42 @@ void NuPlayer::onSourceNotify(const sp<AMessage> &msg) { break; } + case Source::kWhatSubtitleData: + { + sp<ABuffer> buffer; + CHECK(msg->findBuffer("buffer", &buffer)); + + int32_t trackIndex; + int64_t timeUs, durationUs; + CHECK(buffer->meta()->findInt32("trackIndex", &trackIndex)); + CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); + CHECK(buffer->meta()->findInt64("durationUs", &durationUs)); + + Parcel in; + in.writeInt32(trackIndex); + in.writeInt64(timeUs); + in.writeInt64(durationUs); + in.writeInt32(buffer->size()); + in.writeInt32(buffer->size()); + in.write(buffer->data(), buffer->size()); + + notifyListener(MEDIA_SUBTITLE_DATA, 0, 0, &in); + break; + } + + case Source::kWhatQueueDecoderShutdown: + { + int32_t audio, video; + CHECK(msg->findInt32("audio", &audio)); + CHECK(msg->findInt32("video", &video)); + + sp<AMessage> reply; + CHECK(msg->findMessage("reply", &reply)); + + queueDecoderShutdown(audio, video, reply); + break; + } + default: TRESPASS(); } @@ -1355,4 +1505,19 @@ void NuPlayer::Source::onMessageReceived(const sp<AMessage> &msg) { TRESPASS(); } +void NuPlayer::queueDecoderShutdown( + bool audio, bool video, const sp<AMessage> &reply) { + ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video); + + mDeferredActions.push_back( + new ShutdownDecoderAction(audio, video)); + + mDeferredActions.push_back( + new SimpleAction(&NuPlayer::performScanSources)); + + mDeferredActions.push_back(new PostMessageAction(reply)); + + processDeferredActions(); +} + } // namespace android diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h index 50d0462..13350f3 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h @@ -60,6 +60,8 @@ struct NuPlayer : public AHandler { void seekToAsync(int64_t seekTimeUs); status_t setVideoScalingMode(int32_t mode); + status_t getTrackInfo(Parcel* reply) const; + status_t selectTrack(size_t trackIndex, bool select); protected: virtual ~NuPlayer(); @@ -80,6 +82,8 @@ private: struct Action; struct SeekAction; struct SetSurfaceAction; + struct ShutdownDecoderAction; + struct PostMessageAction; struct SimpleAction; enum { @@ -99,6 +103,8 @@ private: kWhatResume = 'rsme', kWhatPollDuration = 'polD', kWhatSourceNotify = 'srcN', + kWhatGetTrackInfo = 'gTrI', + kWhatSelectTrack = 'selT', }; wp<NuPlayerDriver> mDriver; @@ -155,7 +161,7 @@ private: status_t feedDecoderInputData(bool audio, const sp<AMessage> &msg); void renderBuffer(bool audio, const sp<AMessage> &msg); - void notifyListener(int msg, int ext1, int ext2); + void notifyListener(int msg, int ext1, int ext2, const Parcel *in = NULL); void finishFlushIfPossible(); @@ -172,13 +178,16 @@ private: void performSeek(int64_t seekTimeUs); void performDecoderFlush(); - void performDecoderShutdown(); + void performDecoderShutdown(bool audio, bool video); void performReset(); void performScanSources(); void performSetSurface(const sp<NativeWindowWrapper> &wrapper); void onSourceNotify(const sp<AMessage> &msg); + void queueDecoderShutdown( + bool audio, bool video, const sp<AMessage> &reply); + DISALLOW_EVIL_CONSTRUCTORS(NuPlayer); }; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp index 68b9623..47834fd 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp @@ -255,6 +255,7 @@ status_t NuPlayerDriver::pause() { return OK; case STATE_RUNNING: + notifyListener(MEDIA_PAUSED); mPlayer->pause(); break; @@ -287,6 +288,8 @@ status_t NuPlayerDriver::seekTo(int msec) { case STATE_PAUSED: { mAtEOS = false; + // seeks can take a while, so we essentially paused + notifyListener(MEDIA_PAUSED); mPlayer->seekToAsync(seekTimeUs); break; } @@ -345,6 +348,8 @@ status_t NuPlayerDriver::reset() { break; } + notifyListener(MEDIA_STOPPED); + mState = STATE_RESET_IN_PROGRESS; mPlayer->resetAsync(); @@ -387,6 +392,23 @@ status_t NuPlayerDriver::invoke(const Parcel &request, Parcel *reply) { return mPlayer->setVideoScalingMode(mode); } + case INVOKE_ID_GET_TRACK_INFO: + { + return mPlayer->getTrackInfo(reply); + } + + case INVOKE_ID_SELECT_TRACK: + { + int trackIndex = request.readInt32(); + return mPlayer->selectTrack(trackIndex, true /* select */); + } + + case INVOKE_ID_UNSELECT_TRACK: + { + int trackIndex = request.readInt32(); + return mPlayer->selectTrack(trackIndex, false /* select */); + } + default: { return INVALID_OPERATION; @@ -490,12 +512,13 @@ status_t NuPlayerDriver::dump(int fd, const Vector<String16> &args) const { return OK; } -void NuPlayerDriver::notifyListener(int msg, int ext1, int ext2) { +void NuPlayerDriver::notifyListener( + int msg, int ext1, int ext2, const Parcel *in) { if (msg == MEDIA_PLAYBACK_COMPLETE || msg == MEDIA_ERROR) { mAtEOS = true; } - sendEvent(msg, ext1, ext2); + sendEvent(msg, ext1, ext2, in); } void NuPlayerDriver::notifySetDataSourceCompleted(status_t err) { diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h index 5df0cfb..99f72a6 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h @@ -69,7 +69,7 @@ struct NuPlayerDriver : public MediaPlayerInterface { void notifyPosition(int64_t positionUs); void notifySeekComplete(); void notifyFrameStats(int64_t numFramesTotal, int64_t numFramesDropped); - void notifyListener(int msg, int ext1 = 0, int ext2 = 0); + void notifyListener(int msg, int ext1 = 0, int ext2 = 0, const Parcel *in = NULL); void notifyFlagsChanged(uint32_t flags); protected: diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp index 404b56f..3b2784b 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp @@ -50,6 +50,8 @@ NuPlayer::Renderer::Renderer( mSyncQueues(false), mPaused(false), mVideoRenderingStarted(false), + mVideoRenderingStartGeneration(0), + mAudioRenderingStartGeneration(0), mLastPositionUpdateUs(-1ll), mVideoLateByUs(0ll) { } @@ -95,11 +97,11 @@ void NuPlayer::Renderer::flush(bool audio) { } void NuPlayer::Renderer::signalTimeDiscontinuity() { - CHECK(mAudioQueue.empty()); - CHECK(mVideoQueue.empty()); + // CHECK(mAudioQueue.empty()); + // CHECK(mVideoQueue.empty()); mAnchorTimeMediaUs = -1; mAnchorTimeRealUs = -1; - mSyncQueues = mHasAudio && mHasVideo; + mSyncQueues = false; } void NuPlayer::Renderer::pause() { @@ -220,6 +222,23 @@ void NuPlayer::Renderer::signalAudioSinkChanged() { (new AMessage(kWhatAudioSinkChanged, id()))->post(); } +void NuPlayer::Renderer::prepareForMediaRenderingStart() { + mAudioRenderingStartGeneration = mAudioQueueGeneration; + mVideoRenderingStartGeneration = mVideoQueueGeneration; +} + +void NuPlayer::Renderer::notifyIfMediaRenderingStarted() { + if (mVideoRenderingStartGeneration == mVideoQueueGeneration && + mAudioRenderingStartGeneration == mAudioQueueGeneration) { + mVideoRenderingStartGeneration = -1; + mAudioRenderingStartGeneration = -1; + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatMediaRenderingStart); + notify->post(); + } +} + bool NuPlayer::Renderer::onDrainAudioQueue() { uint32_t numFramesPlayed; if (mAudioSink->getPosition(&numFramesPlayed) != OK) { @@ -299,6 +318,8 @@ bool NuPlayer::Renderer::onDrainAudioQueue() { numBytesAvailableToWrite -= copy; size_t copiedFrames = copy / mAudioSink->frameSize(); mNumFramesWritten += copiedFrames; + + notifyIfMediaRenderingStarted(); } notifyPosition(); @@ -405,6 +426,8 @@ void NuPlayer::Renderer::onDrainVideoQueue() { notifyVideoRenderingStart(); } + notifyIfMediaRenderingStarted(); + notifyPosition(); } @@ -552,6 +575,7 @@ void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { // is flushed. syncQueuesDone(); + ALOGV("flushing %s", audio ? "audio" : "video"); if (audio) { flushQueue(&mAudioQueue); @@ -560,6 +584,8 @@ void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { mDrainAudioQueuePending = false; ++mAudioQueueGeneration; + + prepareForMediaRenderingStart(); } else { flushQueue(&mVideoQueue); @@ -568,6 +594,8 @@ void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { mDrainVideoQueuePending = false; ++mVideoQueueGeneration; + + prepareForMediaRenderingStart(); } notifyFlushComplete(audio); @@ -658,6 +686,8 @@ void NuPlayer::Renderer::onPause() { mDrainVideoQueuePending = false; ++mVideoQueueGeneration; + prepareForMediaRenderingStart(); + if (mHasAudio) { mAudioSink->pause(); } diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h index c9796e2..94a05ea 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h @@ -53,6 +53,7 @@ struct NuPlayer::Renderer : public AHandler { kWhatFlushComplete = 'fluC', kWhatPosition = 'posi', kWhatVideoRenderingStart = 'vdrd', + kWhatMediaRenderingStart = 'mdrd', }; protected: @@ -106,6 +107,8 @@ private: bool mPaused; bool mVideoRenderingStarted; + int32_t mVideoRenderingStartGeneration; + int32_t mAudioRenderingStartGeneration; int64_t mLastPositionUpdateUs; int64_t mVideoLateByUs; @@ -116,6 +119,9 @@ private: void onDrainVideoQueue(); void postDrainVideoQueue(); + void prepareForMediaRenderingStart(); + void notifyIfMediaRenderingStarted(); + void onQueueBuffer(const sp<AMessage> &msg); void onQueueEOS(const sp<AMessage> &msg); void onFlush(const sp<AMessage> &msg); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerSource.h b/media/libmediaplayerservice/nuplayer/NuPlayerSource.h index 1cbf575..e50533a 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerSource.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerSource.h @@ -42,6 +42,8 @@ struct NuPlayer::Source : public AHandler { kWhatVideoSizeChanged, kWhatBufferingStart, kWhatBufferingEnd, + kWhatSubtitleData, + kWhatQueueDecoderShutdown, }; // The provides message is used to notify the player about various @@ -70,6 +72,14 @@ struct NuPlayer::Source : public AHandler { return INVALID_OPERATION; } + virtual status_t getTrackInfo(Parcel* reply) const { + return INVALID_OPERATION; + } + + virtual status_t selectTrack(size_t trackIndex, bool select) { + return INVALID_OPERATION; + } + virtual status_t seekTo(int64_t seekTimeUs) { return INVALID_OPERATION; } diff --git a/media/libmediaplayerservice/nuplayer/RTSPSource.cpp b/media/libmediaplayerservice/nuplayer/RTSPSource.cpp index 3385a19..18cf6d1 100644 --- a/media/libmediaplayerservice/nuplayer/RTSPSource.cpp +++ b/media/libmediaplayerservice/nuplayer/RTSPSource.cpp @@ -358,11 +358,10 @@ void NuPlayer::RTSPSource::onMessageReceived(const sp<AMessage> &msg) { uint32_t flags = 0; if (mHandler->isSeekable()) { - flags = FLAG_CAN_PAUSE | FLAG_CAN_SEEK; - - // Seeking 10secs forward or backward is a very expensive - // operation for rtsp, so let's not enable that. - // The user can always use the seek bar. + flags = FLAG_CAN_PAUSE + | FLAG_CAN_SEEK + | FLAG_CAN_SEEK_BACKWARD + | FLAG_CAN_SEEK_FORWARD; } notifyFlagsChanged(flags); diff --git a/media/libnbaio/Android.mk b/media/libnbaio/Android.mk index 5d00d15..69c75b8 100644 --- a/media/libnbaio/Android.mk +++ b/media/libnbaio/Android.mk @@ -31,6 +31,9 @@ LOCAL_SHARED_LIBRARIES := \ libcommon_time_client \ libcutils \ libutils \ - liblog + liblog \ + libmedia +# This dependency on libmedia is for SingleStateQueueInstantiations. +# Consider a separate a library for SingleStateQueueInstantiations. include $(BUILD_SHARED_LIBRARY) diff --git a/media/libnbaio/AudioStreamOutSink.cpp b/media/libnbaio/AudioStreamOutSink.cpp index 6f525e5..e4341d7 100644 --- a/media/libnbaio/AudioStreamOutSink.cpp +++ b/media/libnbaio/AudioStreamOutSink.cpp @@ -79,4 +79,19 @@ status_t AudioStreamOutSink::getNextWriteTimestamp(int64_t *timestamp) { return mStream->get_next_write_timestamp(mStream, timestamp); } +status_t AudioStreamOutSink::getTimestamp(AudioTimestamp& timestamp) +{ + if (mStream->get_presentation_position == NULL) { + return INVALID_OPERATION; + } + // FIXME position64 won't be needed after AudioTimestamp.mPosition is changed to uint64_t + uint64_t position64; + int ok = mStream->get_presentation_position(mStream, &position64, ×tamp.mTime); + if (ok != 0) { + return INVALID_OPERATION; + } + timestamp.mPosition = position64; + return OK; +} + } // namespace android diff --git a/media/libnbaio/MonoPipe.cpp b/media/libnbaio/MonoPipe.cpp index e8d3d9b..de0ad28 100644 --- a/media/libnbaio/MonoPipe.cpp +++ b/media/libnbaio/MonoPipe.cpp @@ -42,7 +42,10 @@ MonoPipe::MonoPipe(size_t reqFrames, NBAIO_Format format, bool writeCanBlock) : // mWriteTs mSetpoint((reqFrames * 11) / 16), mWriteCanBlock(writeCanBlock), - mIsShutdown(false) + mIsShutdown(false), + // mTimestampShared + mTimestampMutator(&mTimestampShared), + mTimestampObserver(&mTimestampShared) { CCHelper tmpHelper; status_t res; @@ -310,4 +313,12 @@ bool MonoPipe::isShutdown() return mIsShutdown; } +status_t MonoPipe::getTimestamp(AudioTimestamp& timestamp) +{ + if (mTimestampObserver.poll(timestamp)) { + return OK; + } + return INVALID_OPERATION; +} + } // namespace android diff --git a/media/libnbaio/MonoPipeReader.cpp b/media/libnbaio/MonoPipeReader.cpp index 394f6ac..851341a 100644 --- a/media/libnbaio/MonoPipeReader.cpp +++ b/media/libnbaio/MonoPipeReader.cpp @@ -86,4 +86,9 @@ ssize_t MonoPipeReader::read(void *buffer, size_t count, int64_t readPTS) return red; } +void MonoPipeReader::onTimestamp(const AudioTimestamp& timestamp) +{ + mPipe->mTimestampMutator.push(timestamp); +} + } // namespace android diff --git a/media/libnbaio/SourceAudioBufferProvider.cpp b/media/libnbaio/SourceAudioBufferProvider.cpp index d11a86c..062fa0f 100644 --- a/media/libnbaio/SourceAudioBufferProvider.cpp +++ b/media/libnbaio/SourceAudioBufferProvider.cpp @@ -25,7 +25,7 @@ namespace android { SourceAudioBufferProvider::SourceAudioBufferProvider(const sp<NBAIO_Source>& source) : mSource(source), // mFrameBitShiftFormat below - mAllocated(NULL), mSize(0), mOffset(0), mRemaining(0), mGetCount(0) + mAllocated(NULL), mSize(0), mOffset(0), mRemaining(0), mGetCount(0), mFramesReleased(0) { ALOG_ASSERT(source != 0); @@ -90,6 +90,7 @@ void SourceAudioBufferProvider::releaseBuffer(Buffer *buffer) (mOffset + mRemaining <= mSize)); mOffset += buffer->frameCount; mRemaining -= buffer->frameCount; + mFramesReleased += buffer->frameCount; buffer->raw = NULL; buffer->frameCount = 0; mGetCount = 0; @@ -101,4 +102,14 @@ size_t SourceAudioBufferProvider::framesReady() const return avail < 0 ? 0 : (size_t) avail; } +size_t SourceAudioBufferProvider::framesReleased() const +{ + return mFramesReleased; +} + +void SourceAudioBufferProvider::onTimestamp(const AudioTimestamp& timestamp) +{ + mSource->onTimestamp(timestamp); +} + } // namespace android diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp index cf41cf2..1adab38 100644 --- a/media/libstagefright/ACodec.cpp +++ b/media/libstagefright/ACodec.cpp @@ -255,6 +255,8 @@ private: struct ACodec::ExecutingState : public ACodec::BaseState { ExecutingState(ACodec *codec); + void submitRegularOutputBuffers(); + void submitOutputMetaBuffers(); void submitOutputBuffers(); // Submit output buffers to the decoder, submit input buffers to client @@ -359,11 +361,16 @@ ACodec::ACodec() mNode(NULL), mSentFormat(false), mIsEncoder(false), + mUseMetadataOnEncoderOutput(false), mShutdownInProgress(false), mEncoderDelay(0), mEncoderPadding(0), mChannelMaskPresent(false), - mChannelMask(0) { + mChannelMask(0), + mDequeueCounter(0), + mStoreMetaDataInOutputBuffers(false), + mMetaDataBuffersToSubmit(0), + mRepeatFrameDelayUs(-1ll) { mUninitializedState = new UninitializedState(this); mLoadedState = new LoadedState(this); mLoadedToIdleState = new LoadedToIdleState(this); @@ -453,7 +460,11 @@ status_t ACodec::allocateBuffersOnPort(OMX_U32 portIndex) { status_t err; if (mNativeWindow != NULL && portIndex == kPortIndexOutput) { - err = allocateOutputBuffersFromNativeWindow(); + if (mStoreMetaDataInOutputBuffers) { + err = allocateOutputMetaDataBuffers(); + } else { + err = allocateOutputBuffersFromNativeWindow(); + } } else { OMX_PARAM_PORTDEFINITIONTYPE def; InitOMXParams(&def); @@ -483,7 +494,8 @@ status_t ACodec::allocateBuffersOnPort(OMX_U32 portIndex) { ? OMXCodec::kRequiresAllocateBufferOnInputPorts : OMXCodec::kRequiresAllocateBufferOnOutputPorts; - if (portIndex == kPortIndexInput && (mFlags & kFlagIsSecure)) { + if ((portIndex == kPortIndexInput && (mFlags & kFlagIsSecure)) + || mUseMetadataOnEncoderOutput) { mem.clear(); void *ptr; @@ -491,7 +503,10 @@ status_t ACodec::allocateBuffersOnPort(OMX_U32 portIndex) { mNode, portIndex, def.nBufferSize, &info.mBufferID, &ptr); - info.mData = new ABuffer(ptr, def.nBufferSize); + int32_t bufSize = mUseMetadataOnEncoderOutput ? + (4 + sizeof(buffer_handle_t)) : def.nBufferSize; + + info.mData = new ABuffer(ptr, bufSize); } else if (mQuirks & requiresAllocateBufferBit) { err = mOMX->allocateBufferWithBackup( mNode, portIndex, mem, &info.mBufferID); @@ -531,7 +546,9 @@ status_t ACodec::allocateBuffersOnPort(OMX_U32 portIndex) { return OK; } -status_t ACodec::allocateOutputBuffersFromNativeWindow() { +status_t ACodec::configureOutputBuffersFromNativeWindow( + OMX_U32 *bufferCount, OMX_U32 *bufferSize, + OMX_U32 *minUndequeuedBuffers) { OMX_PARAM_PORTDEFINITIONTYPE def; InitOMXParams(&def); def.nPortIndex = kPortIndexOutput; @@ -596,10 +613,10 @@ status_t ACodec::allocateOutputBuffersFromNativeWindow() { return err; } - int minUndequeuedBufs = 0; + *minUndequeuedBuffers = 0; err = mNativeWindow->query( mNativeWindow.get(), NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, - &minUndequeuedBufs); + (int *)minUndequeuedBuffers); if (err != 0) { ALOGE("NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS query failed: %s (%d)", @@ -610,8 +627,8 @@ status_t ACodec::allocateOutputBuffersFromNativeWindow() { // XXX: Is this the right logic to use? It's not clear to me what the OMX // buffer counts refer to - how do they account for the renderer holding on // to buffers? - if (def.nBufferCountActual < def.nBufferCountMin + minUndequeuedBufs) { - OMX_U32 newBufferCount = def.nBufferCountMin + minUndequeuedBufs; + if (def.nBufferCountActual < def.nBufferCountMin + *minUndequeuedBuffers) { + OMX_U32 newBufferCount = def.nBufferCountMin + *minUndequeuedBuffers; def.nBufferCountActual = newBufferCount; err = mOMX->setParameter( mNode, OMX_IndexParamPortDefinition, &def, sizeof(def)); @@ -632,12 +649,24 @@ status_t ACodec::allocateOutputBuffersFromNativeWindow() { return err; } + *bufferCount = def.nBufferCountActual; + *bufferSize = def.nBufferSize; + return err; +} + +status_t ACodec::allocateOutputBuffersFromNativeWindow() { + OMX_U32 bufferCount, bufferSize, minUndequeuedBuffers; + status_t err = configureOutputBuffersFromNativeWindow( + &bufferCount, &bufferSize, &minUndequeuedBuffers); + if (err != 0) + return err; + ALOGV("[%s] Allocating %lu buffers from a native window of size %lu on " "output port", - mComponentName.c_str(), def.nBufferCountActual, def.nBufferSize); + mComponentName.c_str(), bufferCount, bufferSize); // Dequeue buffers and send them to OMX - for (OMX_U32 i = 0; i < def.nBufferCountActual; i++) { + for (OMX_U32 i = 0; i < bufferCount; i++) { ANativeWindowBuffer *buf; err = native_window_dequeue_buffer_and_wait(mNativeWindow.get(), &buf); if (err != 0) { @@ -648,7 +677,7 @@ status_t ACodec::allocateOutputBuffersFromNativeWindow() { sp<GraphicBuffer> graphicBuffer(new GraphicBuffer(buf, false)); BufferInfo info; info.mStatus = BufferInfo::OWNED_BY_US; - info.mData = new ABuffer(NULL /* data */, def.nBufferSize /* capacity */); + info.mData = new ABuffer(NULL /* data */, bufferSize /* capacity */); info.mGraphicBuffer = graphicBuffer; mBuffers[kPortIndexOutput].push(info); @@ -677,9 +706,9 @@ status_t ACodec::allocateOutputBuffersFromNativeWindow() { cancelStart = 0; cancelEnd = mBuffers[kPortIndexOutput].size(); } else { - // Return the last two buffers to the native window. - cancelStart = def.nBufferCountActual - minUndequeuedBufs; - cancelEnd = def.nBufferCountActual; + // Return the required minimum undequeued buffers to the native window. + cancelStart = bufferCount - minUndequeuedBuffers; + cancelEnd = bufferCount; } for (OMX_U32 i = cancelStart; i < cancelEnd; i++) { @@ -690,6 +719,65 @@ status_t ACodec::allocateOutputBuffersFromNativeWindow() { return err; } +status_t ACodec::allocateOutputMetaDataBuffers() { + OMX_U32 bufferCount, bufferSize, minUndequeuedBuffers; + status_t err = configureOutputBuffersFromNativeWindow( + &bufferCount, &bufferSize, &minUndequeuedBuffers); + if (err != 0) + return err; + + ALOGV("[%s] Allocating %lu meta buffers on output port", + mComponentName.c_str(), bufferCount); + + size_t totalSize = bufferCount * 8; + mDealer[kPortIndexOutput] = new MemoryDealer(totalSize, "ACodec"); + + // Dequeue buffers and send them to OMX + for (OMX_U32 i = 0; i < bufferCount; i++) { + BufferInfo info; + info.mStatus = BufferInfo::OWNED_BY_NATIVE_WINDOW; + info.mGraphicBuffer = NULL; + info.mDequeuedAt = mDequeueCounter; + + sp<IMemory> mem = mDealer[kPortIndexOutput]->allocate( + sizeof(struct VideoDecoderOutputMetaData)); + CHECK(mem.get() != NULL); + info.mData = new ABuffer(mem->pointer(), mem->size()); + + // we use useBuffer for metadata regardless of quirks + err = mOMX->useBuffer( + mNode, kPortIndexOutput, mem, &info.mBufferID); + + mBuffers[kPortIndexOutput].push(info); + + ALOGV("[%s] allocated meta buffer with ID %p (pointer = %p)", + mComponentName.c_str(), info.mBufferID, mem->pointer()); + } + + mMetaDataBuffersToSubmit = bufferCount - minUndequeuedBuffers; + return err; +} + +status_t ACodec::submitOutputMetaDataBuffer() { + CHECK(mStoreMetaDataInOutputBuffers); + if (mMetaDataBuffersToSubmit == 0) + return OK; + + BufferInfo *info = dequeueBufferFromNativeWindow(); + if (info == NULL) + return ERROR_IO; + + ALOGV("[%s] submitting output meta buffer ID %p for graphic buffer %p", + mComponentName.c_str(), info->mBufferID, info->mGraphicBuffer.get()); + + --mMetaDataBuffersToSubmit; + CHECK_EQ(mOMX->fillBuffer(mNode, info->mBufferID), + (status_t)OK); + + info->mStatus = BufferInfo::OWNED_BY_COMPONENT; + return OK; +} + status_t ACodec::cancelBufferToNativeWindow(BufferInfo *info) { CHECK_EQ((int)info->mStatus, (int)BufferInfo::OWNED_BY_US); @@ -709,16 +797,19 @@ status_t ACodec::cancelBufferToNativeWindow(BufferInfo *info) { ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { ANativeWindowBuffer *buf; int fenceFd = -1; + CHECK(mNativeWindow.get() != NULL); if (native_window_dequeue_buffer_and_wait(mNativeWindow.get(), &buf) != 0) { ALOGE("dequeueBuffer failed."); return NULL; } + BufferInfo *oldest = NULL; for (size_t i = mBuffers[kPortIndexOutput].size(); i-- > 0;) { BufferInfo *info = &mBuffers[kPortIndexOutput].editItemAt(i); - if (info->mGraphicBuffer->handle == buf->handle) { + if (info->mGraphicBuffer != NULL && + info->mGraphicBuffer->handle == buf->handle) { CHECK_EQ((int)info->mStatus, (int)BufferInfo::OWNED_BY_NATIVE_WINDOW); @@ -726,6 +817,39 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { return info; } + + if (info->mStatus == BufferInfo::OWNED_BY_NATIVE_WINDOW && + (oldest == NULL || + // avoid potential issues from counter rolling over + mDequeueCounter - info->mDequeuedAt > + mDequeueCounter - oldest->mDequeuedAt)) { + oldest = info; + } + } + + if (oldest) { + CHECK(mStoreMetaDataInOutputBuffers); + + // discard buffer in LRU info and replace with new buffer + oldest->mGraphicBuffer = new GraphicBuffer(buf, false); + oldest->mStatus = BufferInfo::OWNED_BY_US; + + mOMX->updateGraphicBufferInMeta( + mNode, kPortIndexOutput, oldest->mGraphicBuffer, + oldest->mBufferID); + + VideoDecoderOutputMetaData *metaData = + reinterpret_cast<VideoDecoderOutputMetaData *>( + oldest->mData->base()); + CHECK_EQ(metaData->eType, kMetadataBufferTypeGrallocSource); + + ALOGV("replaced oldest buffer #%u with age %u (%p/%p stored in %p)", + oldest - &mBuffers[kPortIndexOutput][0], + mDequeueCounter - oldest->mDequeuedAt, + metaData->pHandle, + oldest->mGraphicBuffer->handle, oldest->mData->base()); + + return oldest; } TRESPASS(); @@ -831,8 +955,10 @@ status_t ACodec::setComponentRole( "video_decoder.mpeg4", "video_encoder.mpeg4" }, { MEDIA_MIMETYPE_VIDEO_H263, "video_decoder.h263", "video_encoder.h263" }, - { MEDIA_MIMETYPE_VIDEO_VPX, - "video_decoder.vpx", "video_encoder.vpx" }, + { MEDIA_MIMETYPE_VIDEO_VP8, + "video_decoder.vp8", "video_encoder.vp8" }, + { MEDIA_MIMETYPE_VIDEO_VP9, + "video_decoder.vp9", "video_encoder.vp9" }, { MEDIA_MIMETYPE_AUDIO_RAW, "audio_decoder.raw", "audio_encoder.raw" }, { MEDIA_MIMETYPE_AUDIO_FLAC, @@ -912,14 +1038,14 @@ status_t ACodec::configureCodec( err = mOMX->storeMetaDataInBuffers(mNode, kPortIndexInput, OMX_TRUE); if (err != OK) { - ALOGE("[%s] storeMetaDataInBuffers failed w/ err %d", - mComponentName.c_str(), err); + ALOGE("[%s] storeMetaDataInBuffers (input) failed w/ err %d", + mComponentName.c_str(), err); - return err; - } - } + return err; + } + } - int32_t prependSPSPPS; + int32_t prependSPSPPS = 0; if (encoder && msg->findInt32("prepend-sps-pps-to-idr-frames", &prependSPSPPS) && prependSPSPPS != 0) { @@ -946,7 +1072,97 @@ status_t ACodec::configureCodec( } } - if (!strncasecmp(mime, "video/", 6)) { + // Only enable metadata mode on encoder output if encoder can prepend + // sps/pps to idr frames, since in metadata mode the bitstream is in an + // opaque handle, to which we don't have access. + int32_t video = !strncasecmp(mime, "video/", 6); + if (encoder && video) { + OMX_BOOL enable = (OMX_BOOL) (prependSPSPPS + && msg->findInt32("store-metadata-in-buffers-output", &storeMeta) + && storeMeta != 0); + + err = mOMX->storeMetaDataInBuffers(mNode, kPortIndexOutput, enable); + + if (err != OK) { + ALOGE("[%s] storeMetaDataInBuffers (output) failed w/ err %d", + mComponentName.c_str(), err); + mUseMetadataOnEncoderOutput = 0; + } else { + mUseMetadataOnEncoderOutput = enable; + } + + if (!msg->findInt64( + "repeat-previous-frame-after", + &mRepeatFrameDelayUs)) { + mRepeatFrameDelayUs = -1ll; + } + } + + // Always try to enable dynamic output buffers on native surface + sp<RefBase> obj; + int32_t haveNativeWindow = msg->findObject("native-window", &obj) && + obj != NULL; + mStoreMetaDataInOutputBuffers = false; + if (!encoder && video && haveNativeWindow) { + err = mOMX->storeMetaDataInBuffers(mNode, kPortIndexOutput, OMX_TRUE); + if (err != OK) { + ALOGE("[%s] storeMetaDataInBuffers failed w/ err %d", + mComponentName.c_str(), err); + + // if adaptive playback has been requested, try JB fallback + // NOTE: THIS FALLBACK MECHANISM WILL BE REMOVED DUE TO ITS + // LARGE MEMORY REQUIREMENT + + // we will not do adaptive playback on software accessed + // surfaces as they never had to respond to changes in the + // crop window, and we don't trust that they will be able to. + int usageBits = 0; + bool canDoAdaptivePlayback; + + sp<NativeWindowWrapper> windowWrapper( + static_cast<NativeWindowWrapper *>(obj.get())); + sp<ANativeWindow> nativeWindow = windowWrapper->getNativeWindow(); + + if (nativeWindow->query( + nativeWindow.get(), + NATIVE_WINDOW_CONSUMER_USAGE_BITS, + &usageBits) != OK) { + canDoAdaptivePlayback = false; + } else { + canDoAdaptivePlayback = + (usageBits & + (GRALLOC_USAGE_SW_READ_MASK | + GRALLOC_USAGE_SW_WRITE_MASK)) == 0; + } + + int32_t maxWidth = 0, maxHeight = 0; + if (canDoAdaptivePlayback && + msg->findInt32("max-width", &maxWidth) && + msg->findInt32("max-height", &maxHeight)) { + ALOGV("[%s] prepareForAdaptivePlayback(%ldx%ld)", + mComponentName.c_str(), maxWidth, maxHeight); + + err = mOMX->prepareForAdaptivePlayback( + mNode, kPortIndexOutput, OMX_TRUE, maxWidth, maxHeight); + ALOGW_IF(err != OK, + "[%s] prepareForAdaptivePlayback failed w/ err %d", + mComponentName.c_str(), err); + } + // allow failure + err = OK; + } else { + ALOGV("[%s] storeMetaDataInBuffers succeeded", mComponentName.c_str()); + mStoreMetaDataInOutputBuffers = true; + } + + int32_t push; + if (msg->findInt32("push-blank-buffers-on-shutdown", &push) + && push != 0) { + mFlags |= kFlagPushBlankBuffersToNativeWindowOnShutdown; + } + } + + if (video) { if (encoder) { err = setupVideoEncoder(mime, msg); } else { @@ -1476,7 +1692,8 @@ static const struct VideoCodingMapEntry { { MEDIA_MIMETYPE_VIDEO_MPEG4, OMX_VIDEO_CodingMPEG4 }, { MEDIA_MIMETYPE_VIDEO_H263, OMX_VIDEO_CodingH263 }, { MEDIA_MIMETYPE_VIDEO_MPEG2, OMX_VIDEO_CodingMPEG2 }, - { MEDIA_MIMETYPE_VIDEO_VPX, OMX_VIDEO_CodingVPX }, + { MEDIA_MIMETYPE_VIDEO_VP8, OMX_VIDEO_CodingVP8 }, + { MEDIA_MIMETYPE_VIDEO_VP9, OMX_VIDEO_CodingVP9 }, }; static status_t GetVideoCodingTypeFromMime( @@ -2189,6 +2406,10 @@ void ACodec::waitUntilAllPossibleNativeWindowBuffersAreReturnedToUs() { while (countBuffersOwnedByNativeWindow() > (size_t)minUndequeuedBufs && dequeueBufferFromNativeWindow() != NULL) { + // these buffers will be submitted as regular buffers; account for this + if (mStoreMetaDataInOutputBuffers && mMetaDataBuffersToSubmit > 0) { + --mMetaDataBuffersToSubmit; + } } } @@ -2321,10 +2542,15 @@ void ACodec::sendFormatChange(const sp<AMessage> &reply) { ¶ms, sizeof(params)), (status_t)OK); + CHECK_GT(params.nChannels, 0); CHECK(params.nChannels == 1 || params.bInterleaved); CHECK_EQ(params.nBitPerSample, 16u); - CHECK_EQ((int)params.eNumData, (int)OMX_NumericalDataSigned); - CHECK_EQ((int)params.ePCMMode, (int)OMX_AUDIO_PCMModeLinear); + + CHECK_EQ((int)params.eNumData, + (int)OMX_NumericalDataSigned); + + CHECK_EQ((int)params.ePCMMode, + (int)OMX_AUDIO_PCMModeLinear); notify->setString("mime", MEDIA_MIMETYPE_AUDIO_RAW); notify->setInt32("channel-count", params.nChannels); @@ -2334,11 +2560,14 @@ void ACodec::sendFormatChange(const sp<AMessage> &reply) { if (mSkipCutBuffer != NULL) { size_t prevbufsize = mSkipCutBuffer->size(); if (prevbufsize != 0) { - ALOGW("Replacing SkipCutBuffer holding %d bytes", prevbufsize); + ALOGW("Replacing SkipCutBuffer holding %d " + "bytes", + prevbufsize); } } - mSkipCutBuffer = new SkipCutBuffer(mEncoderDelay * frameSize, - mEncoderPadding * frameSize); + mSkipCutBuffer = new SkipCutBuffer( + mEncoderDelay * frameSize, + mEncoderPadding * frameSize); } if (mChannelMaskPresent) { @@ -2463,6 +2692,14 @@ status_t ACodec::pushBlankBuffersToNativeWindow() { goto error; } + err = native_window_set_scaling_mode(mNativeWindow.get(), + NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW); + if (err != NO_ERROR) { + ALOGE("error pushing blank_frames: set_scaling_mode failed: %s (%d)", + strerror(-err), -err); + goto error; + } + err = native_window_set_usage(mNativeWindow.get(), GRALLOC_USAGE_SW_WRITE_OFTEN); if (err != NO_ERROR) { @@ -2829,16 +3066,17 @@ void ACodec::BaseState::onInputBufferFilled(const sp<AMessage> &msg) { sp<ABuffer> buffer; int32_t err = OK; bool eos = false; + PortMode mode = getPortMode(kPortIndexInput); if (!msg->findBuffer("buffer", &buffer)) { + /* these are unfilled buffers returned by client */ CHECK(msg->findInt32("err", &err)); ALOGV("[%s] saw error %d instead of an input buffer", mCodec->mComponentName.c_str(), err); buffer.clear(); - - eos = true; + mode = KEEP_BUFFERS; } int32_t tmp; @@ -2852,8 +3090,6 @@ void ACodec::BaseState::onInputBufferFilled(const sp<AMessage> &msg) { info->mStatus = BufferInfo::OWNED_BY_US; - PortMode mode = getPortMode(kPortIndexInput); - switch (mode) { case KEEP_BUFFERS: { @@ -2916,6 +3152,20 @@ void ACodec::BaseState::onInputBufferFilled(const sp<AMessage> &msg) { mCodec->mBufferStats.add(timeUs, stats); #endif + if (mCodec->mStoreMetaDataInOutputBuffers) { + // try to submit an output buffer for each input buffer + PortMode outputMode = getPortMode(kPortIndexOutput); + + ALOGV("MetaDataBuffersToSubmit=%u portMode=%s", + mCodec->mMetaDataBuffersToSubmit, + (outputMode == FREE_BUFFERS ? "FREE" : + outputMode == KEEP_BUFFERS ? "KEEP" : "RESUBMIT")); + if (outputMode == RESUBMIT_BUFFERS) { + CHECK_EQ(mCodec->submitOutputMetaDataBuffer(), + (status_t)OK); + } + } + CHECK_EQ(mCodec->mOMX->emptyBuffer( mCodec->mNode, bufferID, @@ -3033,6 +3283,7 @@ bool ACodec::BaseState::onOMXFillBufferDone( CHECK_EQ((int)info->mStatus, (int)BufferInfo::OWNED_BY_COMPONENT); + info->mDequeuedAt = ++mCodec->mDequeueCounter; info->mStatus = BufferInfo::OWNED_BY_US; PortMode mode = getPortMode(kPortIndexOutput); @@ -3062,7 +3313,15 @@ bool ACodec::BaseState::onOMXFillBufferDone( mCodec->sendFormatChange(reply); } - info->mData->setRange(rangeOffset, rangeLength); + if (mCodec->mUseMetadataOnEncoderOutput) { + native_handle_t* handle = + *(native_handle_t**)(info->mData->data() + 4); + info->mData->meta()->setPointer("handle", handle); + info->mData->meta()->setInt32("rangeOffset", rangeOffset); + info->mData->meta()->setInt32("rangeLength", rangeLength); + } else { + info->mData->setRange(rangeOffset, rangeLength); + } #if 0 if (mCodec->mNativeWindow == NULL) { if (IsIDR(info->mData)) { @@ -3220,6 +3479,7 @@ void ACodec::UninitializedState::stateEntered() { mCodec->mOMX.clear(); mCodec->mQuirks = 0; mCodec->mFlags = 0; + mCodec->mUseMetadataOnEncoderOutput = 0; mCodec->mComponentName.clear(); } @@ -3373,6 +3633,7 @@ bool ACodec::UninitializedState::onAllocateComponent(const sp<AMessage> &msg) { if (componentName.endsWith(".secure")) { mCodec->mFlags |= kFlagIsSecure; + mCodec->mFlags |= kFlagPushBlankBuffersToNativeWindowOnShutdown; } mCodec->mQuirks = quirks; @@ -3405,6 +3666,10 @@ void ACodec::LoadedState::stateEntered() { mCodec->mInputEOSResult = OK; + mCodec->mDequeueCounter = 0; + mCodec->mMetaDataBuffersToSubmit = 0; + mCodec->mRepeatFrameDelayUs = -1ll; + if (mCodec->mShutdownInProgress) { bool keepComponentAllocated = mCodec->mKeepComponentAllocated; @@ -3535,6 +3800,23 @@ void ACodec::LoadedState::onCreateInputSurface( err = mCodec->mOMX->createInputSurface(mCodec->mNode, kPortIndexInput, &bufferProducer); + + if (err == OK && mCodec->mRepeatFrameDelayUs > 0ll) { + err = mCodec->mOMX->setInternalOption( + mCodec->mNode, + kPortIndexInput, + IOMX::INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY, + &mCodec->mRepeatFrameDelayUs, + sizeof(mCodec->mRepeatFrameDelayUs)); + + if (err != OK) { + ALOGE("[%s] Unable to configure option to repeat previous " + "frames (err %d)", + mCodec->mComponentName.c_str(), + err); + } + } + if (err == OK) { notify->setObject("input-surface", new BufferProducerWrapper(bufferProducer)); @@ -3722,7 +4004,20 @@ ACodec::BaseState::PortMode ACodec::ExecutingState::getPortMode( return RESUBMIT_BUFFERS; } -void ACodec::ExecutingState::submitOutputBuffers() { +void ACodec::ExecutingState::submitOutputMetaBuffers() { + // submit as many buffers as there are input buffers with the codec + // in case we are in port reconfiguring + for (size_t i = 0; i < mCodec->mBuffers[kPortIndexInput].size(); ++i) { + BufferInfo *info = &mCodec->mBuffers[kPortIndexInput].editItemAt(i); + + if (info->mStatus == BufferInfo::OWNED_BY_COMPONENT) { + if (mCodec->submitOutputMetaDataBuffer() != OK) + break; + } + } +} + +void ACodec::ExecutingState::submitRegularOutputBuffers() { for (size_t i = 0; i < mCodec->mBuffers[kPortIndexOutput].size(); ++i) { BufferInfo *info = &mCodec->mBuffers[kPortIndexOutput].editItemAt(i); @@ -3747,6 +4042,13 @@ void ACodec::ExecutingState::submitOutputBuffers() { } } +void ACodec::ExecutingState::submitOutputBuffers() { + submitRegularOutputBuffers(); + if (mCodec->mStoreMetaDataInOutputBuffers) { + submitOutputMetaBuffers(); + } +} + void ACodec::ExecutingState::resume() { if (mActive) { ALOGV("[%s] We're already active, no need to resume.", @@ -3871,7 +4173,7 @@ bool ACodec::ExecutingState::onMessageReceived(const sp<AMessage> &msg) { status_t ACodec::setParameters(const sp<AMessage> ¶ms) { int32_t videoBitrate; - if (params->findInt32("videoBitrate", &videoBitrate)) { + if (params->findInt32("video-bitrate", &videoBitrate)) { OMX_VIDEO_CONFIG_BITRATETYPE configParams; InitOMXParams(&configParams); configParams.nPortIndex = kPortIndexOutput; @@ -3891,6 +4193,34 @@ status_t ACodec::setParameters(const sp<AMessage> ¶ms) { } } + int32_t dropInputFrames; + if (params->findInt32("drop-input-frames", &dropInputFrames)) { + bool suspend = dropInputFrames != 0; + + status_t err = + mOMX->setInternalOption( + mNode, + kPortIndexInput, + IOMX::INTERNAL_OPTION_SUSPEND, + &suspend, + sizeof(suspend)); + + if (err != OK) { + ALOGE("Failed to set parameter 'drop-input-frames' (err %d)", err); + return err; + } + } + + int32_t dummy; + if (params->findInt32("request-sync", &dummy)) { + status_t err = requestIDRFrame(); + + if (err != OK) { + ALOGE("Requesting a sync frame failed w/ err %d", err); + return err; + } + } + return OK; } @@ -3913,6 +4243,7 @@ bool ACodec::ExecutingState::onOMXEvent( CHECK_EQ(data1, (OMX_U32)kPortIndexOutput); if (data2 == 0 || data2 == OMX_IndexParamPortDefinition) { + mCodec->mMetaDataBuffersToSubmit = 0; CHECK_EQ(mCodec->mOMX->sendCommand( mCodec->mNode, OMX_CommandPortDisable, kPortIndexOutput), @@ -4131,7 +4462,8 @@ void ACodec::ExecutingToIdleState::changeStateIfWeOwnAllBuffers() { CHECK_EQ(mCodec->freeBuffersOnPort(kPortIndexInput), (status_t)OK); CHECK_EQ(mCodec->freeBuffersOnPort(kPortIndexOutput), (status_t)OK); - if (mCodec->mFlags & kFlagIsSecure && mCodec->mNativeWindow != NULL) { + if ((mCodec->mFlags & kFlagPushBlankBuffersToNativeWindowOnShutdown) + && mCodec->mNativeWindow != NULL) { // We push enough 1x1 blank buffers to ensure that one of // them has made it to the display. This allows the OMX // component teardown to zero out any protected buffers diff --git a/media/libstagefright/Android.mk b/media/libstagefright/Android.mk index acc3abf..6a2a696 100644 --- a/media/libstagefright/Android.mk +++ b/media/libstagefright/Android.mk @@ -62,6 +62,7 @@ LOCAL_C_INCLUDES:= \ $(TOP)/frameworks/av/include/media/stagefright/timedtext \ $(TOP)/frameworks/native/include/media/hardware \ $(TOP)/frameworks/native/include/media/openmax \ + $(TOP)/frameworks/native/services/connectivitymanager \ $(TOP)/external/flac/include \ $(TOP)/external/tremolo \ $(TOP)/external/openssl/include \ @@ -69,7 +70,7 @@ LOCAL_C_INCLUDES:= \ LOCAL_SHARED_LIBRARIES := \ libbinder \ libcamera_client \ - libcrypto \ + libconnectivitymanager \ libcutils \ libdl \ libdrmframework \ @@ -88,6 +89,7 @@ LOCAL_SHARED_LIBRARIES := \ libutils \ libvorbisidec \ libz \ + libpowermanager LOCAL_STATIC_LIBRARIES := \ libstagefright_color_conversion \ @@ -97,9 +99,9 @@ LOCAL_STATIC_LIBRARIES := \ libvpx \ libwebm \ libstagefright_mpeg2ts \ - libstagefright_httplive \ libstagefright_id3 \ libFLAC \ + libmedia_helper LOCAL_SRC_FILES += \ chromium_http_stub.cpp diff --git a/media/libstagefright/AudioPlayer.cpp b/media/libstagefright/AudioPlayer.cpp index 4208019..a8a8786 100644 --- a/media/libstagefright/AudioPlayer.cpp +++ b/media/libstagefright/AudioPlayer.cpp @@ -17,6 +17,7 @@ //#define LOG_NDEBUG 0 #define LOG_TAG "AudioPlayer" #include <utils/Log.h> +#include <cutils/compiler.h> #include <binder/IPCThreadState.h> #include <media/AudioTrack.h> @@ -27,6 +28,7 @@ #include <media/stagefright/MediaErrors.h> #include <media/stagefright/MediaSource.h> #include <media/stagefright/MetaData.h> +#include <media/stagefright/Utils.h> #include "include/AwesomePlayer.h" @@ -34,10 +36,9 @@ namespace android { AudioPlayer::AudioPlayer( const sp<MediaPlayerBase::AudioSink> &audioSink, - bool allowDeepBuffering, + uint32_t flags, AwesomePlayer *observer) - : mAudioTrack(NULL), - mInputBuffer(NULL), + : mInputBuffer(NULL), mSampleRate(0), mLatencyUs(0), mFrameSize(0), @@ -48,14 +49,17 @@ AudioPlayer::AudioPlayer( mSeeking(false), mReachedEOS(false), mFinalStatus(OK), + mSeekTimeUs(0), mStarted(false), mIsFirstBuffer(false), mFirstBufferResult(OK), mFirstBuffer(NULL), mAudioSink(audioSink), - mAllowDeepBuffering(allowDeepBuffering), mObserver(observer), - mPinnedTimeUs(-1ll) { + mPinnedTimeUs(-1ll), + mPlaying(false), + mStartPosUs(0), + mCreateFlags(flags) { } AudioPlayer::~AudioPlayer() { @@ -110,7 +114,7 @@ status_t AudioPlayer::start(bool sourceAlreadyStarted) { const char *mime; bool success = format->findCString(kKeyMIMEType, &mime); CHECK(success); - CHECK(!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_RAW)); + CHECK(useOffload() || !strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_RAW)); success = format->findInt32(kKeySampleRate, &mSampleRate); CHECK(success); @@ -126,16 +130,74 @@ status_t AudioPlayer::start(bool sourceAlreadyStarted) { channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER; } + audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT; + + if (useOffload()) { + if (mapMimeToAudioFormat(audioFormat, mime) != OK) { + ALOGE("Couldn't map mime type \"%s\" to a valid AudioSystem::audio_format", mime); + audioFormat = AUDIO_FORMAT_INVALID; + } else { + ALOGV("Mime type \"%s\" mapped to audio_format 0x%x", mime, audioFormat); + } + } + + int avgBitRate = -1; + format->findInt32(kKeyBitRate, &avgBitRate); + if (mAudioSink.get() != NULL) { + uint32_t flags = AUDIO_OUTPUT_FLAG_NONE; + audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER; + + if (allowDeepBuffering()) { + flags |= AUDIO_OUTPUT_FLAG_DEEP_BUFFER; + } + if (useOffload()) { + flags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD; + + int64_t durationUs; + if (format->findInt64(kKeyDuration, &durationUs)) { + offloadInfo.duration_us = durationUs; + } else { + offloadInfo.duration_us = -1; + } + + offloadInfo.sample_rate = mSampleRate; + offloadInfo.channel_mask = channelMask; + offloadInfo.format = audioFormat; + offloadInfo.stream_type = AUDIO_STREAM_MUSIC; + offloadInfo.bit_rate = avgBitRate; + offloadInfo.has_video = ((mCreateFlags & HAS_VIDEO) != 0); + offloadInfo.is_streaming = ((mCreateFlags & IS_STREAMING) != 0); + } + status_t err = mAudioSink->open( - mSampleRate, numChannels, channelMask, AUDIO_FORMAT_PCM_16_BIT, + mSampleRate, numChannels, channelMask, audioFormat, DEFAULT_AUDIOSINK_BUFFERCOUNT, &AudioPlayer::AudioSinkCallback, this, - (mAllowDeepBuffering ? - AUDIO_OUTPUT_FLAG_DEEP_BUFFER : - AUDIO_OUTPUT_FLAG_NONE)); + (audio_output_flags_t)flags, + useOffload() ? &offloadInfo : NULL); + + if (err == OK) { + mLatencyUs = (int64_t)mAudioSink->latency() * 1000; + mFrameSize = mAudioSink->frameSize(); + + if (useOffload()) { + // If the playback is offloaded to h/w we pass the + // HAL some metadata information + // We don't want to do this for PCM because it will be going + // through the AudioFlinger mixer before reaching the hardware + sendMetaDataToHal(mAudioSink, format); + } + + err = mAudioSink->start(); + // do not alter behavior for non offloaded tracks: ignore start status. + if (!useOffload()) { + err = OK; + } + } + if (err != OK) { if (mFirstBuffer != NULL) { mFirstBuffer->release(); @@ -149,10 +211,6 @@ status_t AudioPlayer::start(bool sourceAlreadyStarted) { return err; } - mLatencyUs = (int64_t)mAudioSink->latency() * 1000; - mFrameSize = mAudioSink->frameSize(); - - mAudioSink->start(); } else { // playing to an AudioTrack, set up mask if necessary audio_channel_mask_t audioMask = channelMask == CHANNEL_MASK_USE_CHANNEL_ORDER ? @@ -166,8 +224,7 @@ status_t AudioPlayer::start(bool sourceAlreadyStarted) { 0, AUDIO_OUTPUT_FLAG_NONE, &AudioCallback, this, 0); if ((err = mAudioTrack->initCheck()) != OK) { - delete mAudioTrack; - mAudioTrack = NULL; + mAudioTrack.clear(); if (mFirstBuffer != NULL) { mFirstBuffer->release(); @@ -188,6 +245,7 @@ status_t AudioPlayer::start(bool sourceAlreadyStarted) { } mStarted = true; + mPlaying = true; mPinnedTimeUs = -1ll; return OK; @@ -214,29 +272,57 @@ void AudioPlayer::pause(bool playPendingSamples) { mPinnedTimeUs = ALooper::GetNowUs(); } + + mPlaying = false; } -void AudioPlayer::resume() { +status_t AudioPlayer::resume() { CHECK(mStarted); + status_t err; if (mAudioSink.get() != NULL) { - mAudioSink->start(); + err = mAudioSink->start(); } else { - mAudioTrack->start(); + err = mAudioTrack->start(); } + + if (err == OK) { + mPlaying = true; + } + + return err; } void AudioPlayer::reset() { CHECK(mStarted); + ALOGV("reset: mPlaying=%d mReachedEOS=%d useOffload=%d", + mPlaying, mReachedEOS, useOffload() ); + if (mAudioSink.get() != NULL) { mAudioSink->stop(); + // If we're closing and have reached EOS, we don't want to flush + // the track because if it is offloaded there could be a small + // amount of residual data in the hardware buffer which we must + // play to give gapless playback. + // But if we're resetting when paused or before we've reached EOS + // we can't be doing a gapless playback and there could be a large + // amount of data queued in the hardware if the track is offloaded, + // so we must flush to prevent a track switch being delayed playing + // the buffered data that we don't want now + if (!mPlaying || !mReachedEOS) { + mAudioSink->flush(); + } + mAudioSink->close(); } else { mAudioTrack->stop(); - delete mAudioTrack; - mAudioTrack = NULL; + if (!mPlaying || !mReachedEOS) { + mAudioTrack->flush(); + } + + mAudioTrack.clear(); } // Make sure to release any buffer we hold onto so that the @@ -259,10 +345,16 @@ void AudioPlayer::reset() { // The following hack is necessary to ensure that the OMX // component is completely released by the time we may try // to instantiate it again. - wp<MediaSource> tmp = mSource; - mSource.clear(); - while (tmp.promote() != NULL) { - usleep(1000); + // When offloading, the OMX component is not used so this hack + // is not needed + if (!useOffload()) { + wp<MediaSource> tmp = mSource; + mSource.clear(); + while (tmp.promote() != NULL) { + usleep(1000); + } + } else { + mSource.clear(); } IPCThreadState::self()->flushCommands(); @@ -271,9 +363,12 @@ void AudioPlayer::reset() { mPositionTimeMediaUs = -1; mPositionTimeRealUs = -1; mSeeking = false; + mSeekTimeUs = 0; mReachedEOS = false; mFinalStatus = OK; mStarted = false; + mPlaying = false; + mStartPosUs = 0; } // static @@ -294,10 +389,19 @@ bool AudioPlayer::reachedEOS(status_t *finalStatus) { return mReachedEOS; } +void AudioPlayer::notifyAudioEOS() { + ALOGV("AudioPlayer@0x%p notifyAudioEOS", this); + + if (mObserver != NULL) { + mObserver->postAudioEOS(0); + ALOGV("Notified observer of EOS!"); + } +} + status_t AudioPlayer::setPlaybackRatePermille(int32_t ratePermille) { if (mAudioSink.get() != NULL) { return mAudioSink->setPlaybackRatePermille(ratePermille); - } else if (mAudioTrack != NULL){ + } else if (mAudioTrack != 0){ return mAudioTrack->setSampleRate(ratePermille * mSampleRate / 1000); } else { return NO_INIT; @@ -307,21 +411,44 @@ status_t AudioPlayer::setPlaybackRatePermille(int32_t ratePermille) { // static size_t AudioPlayer::AudioSinkCallback( MediaPlayerBase::AudioSink *audioSink, - void *buffer, size_t size, void *cookie) { + void *buffer, size_t size, void *cookie, + MediaPlayerBase::AudioSink::cb_event_t event) { AudioPlayer *me = (AudioPlayer *)cookie; - return me->fillBuffer(buffer, size); -} + switch(event) { + case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER: + return me->fillBuffer(buffer, size); -void AudioPlayer::AudioCallback(int event, void *info) { - if (event != AudioTrack::EVENT_MORE_DATA) { - return; + case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END: + ALOGV("AudioSinkCallback: stream end"); + me->mReachedEOS = true; + me->notifyAudioEOS(); + break; + + case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN: + ALOGV("AudioSinkCallback: Tear down event"); + me->mObserver->postAudioTearDown(); + break; } - AudioTrack::Buffer *buffer = (AudioTrack::Buffer *)info; - size_t numBytesWritten = fillBuffer(buffer->raw, buffer->size); + return 0; +} + +void AudioPlayer::AudioCallback(int event, void *info) { + switch (event) { + case AudioTrack::EVENT_MORE_DATA: + { + AudioTrack::Buffer *buffer = (AudioTrack::Buffer *)info; + size_t numBytesWritten = fillBuffer(buffer->raw, buffer->size); + buffer->size = numBytesWritten; + } + break; - buffer->size = numBytesWritten; + case AudioTrack::EVENT_STREAM_END: + mReachedEOS = true; + notifyAudioEOS(); + break; + } } uint32_t AudioPlayer::getNumFramesPendingPlayout() const { @@ -361,6 +488,7 @@ size_t AudioPlayer::fillBuffer(void *data, size_t size) { size_t size_remaining = size; while (size_remaining > 0) { MediaSource::ReadOptions options; + bool refreshSeekTime = false; { Mutex::Autolock autoLock(mLock); @@ -375,6 +503,7 @@ size_t AudioPlayer::fillBuffer(void *data, size_t size) { } options.setSeekTo(mSeekTimeUs); + refreshSeekTime = true; if (mInputBuffer != NULL) { mInputBuffer->release(); @@ -407,43 +536,56 @@ size_t AudioPlayer::fillBuffer(void *data, size_t size) { Mutex::Autolock autoLock(mLock); if (err != OK) { - if (mObserver && !mReachedEOS) { - // We don't want to post EOS right away but only - // after all frames have actually been played out. - - // These are the number of frames submitted to the - // AudioTrack that you haven't heard yet. - uint32_t numFramesPendingPlayout = - getNumFramesPendingPlayout(); - - // These are the number of frames we're going to - // submit to the AudioTrack by returning from this - // callback. - uint32_t numAdditionalFrames = size_done / mFrameSize; - - numFramesPendingPlayout += numAdditionalFrames; - - int64_t timeToCompletionUs = - (1000000ll * numFramesPendingPlayout) / mSampleRate; - - ALOGV("total number of frames played: %lld (%lld us)", - (mNumFramesPlayed + numAdditionalFrames), - 1000000ll * (mNumFramesPlayed + numAdditionalFrames) - / mSampleRate); - - ALOGV("%d frames left to play, %lld us (%.2f secs)", - numFramesPendingPlayout, - timeToCompletionUs, timeToCompletionUs / 1E6); - - postEOS = true; - if (mAudioSink->needsTrailingPadding()) { - postEOSDelayUs = timeToCompletionUs + mLatencyUs; + if (!mReachedEOS) { + if (useOffload()) { + // no more buffers to push - stop() and wait for STREAM_END + // don't set mReachedEOS until stream end received + if (mAudioSink != NULL) { + mAudioSink->stop(); + } else { + mAudioTrack->stop(); + } } else { - postEOSDelayUs = 0; + if (mObserver) { + // We don't want to post EOS right away but only + // after all frames have actually been played out. + + // These are the number of frames submitted to the + // AudioTrack that you haven't heard yet. + uint32_t numFramesPendingPlayout = + getNumFramesPendingPlayout(); + + // These are the number of frames we're going to + // submit to the AudioTrack by returning from this + // callback. + uint32_t numAdditionalFrames = size_done / mFrameSize; + + numFramesPendingPlayout += numAdditionalFrames; + + int64_t timeToCompletionUs = + (1000000ll * numFramesPendingPlayout) / mSampleRate; + + ALOGV("total number of frames played: %lld (%lld us)", + (mNumFramesPlayed + numAdditionalFrames), + 1000000ll * (mNumFramesPlayed + numAdditionalFrames) + / mSampleRate); + + ALOGV("%d frames left to play, %lld us (%.2f secs)", + numFramesPendingPlayout, + timeToCompletionUs, timeToCompletionUs / 1E6); + + postEOS = true; + if (mAudioSink->needsTrailingPadding()) { + postEOSDelayUs = timeToCompletionUs + mLatencyUs; + } else { + postEOSDelayUs = 0; + } + } + + mReachedEOS = true; } } - mReachedEOS = true; mFinalStatus = err; break; } @@ -454,17 +596,43 @@ size_t AudioPlayer::fillBuffer(void *data, size_t size) { mLatencyUs = (int64_t)mAudioTrack->latency() * 1000; } - CHECK(mInputBuffer->meta_data()->findInt64( + if(mInputBuffer->range_length() != 0) { + CHECK(mInputBuffer->meta_data()->findInt64( kKeyTime, &mPositionTimeMediaUs)); + } + + // need to adjust the mStartPosUs for offload decoding since parser + // might not be able to get the exact seek time requested. + if (refreshSeekTime) { + if (useOffload()) { + if (postSeekComplete) { + ALOGV("fillBuffer is going to post SEEK_COMPLETE"); + mObserver->postAudioSeekComplete(); + postSeekComplete = false; + } - mPositionTimeRealUs = - ((mNumFramesPlayed + size_done / mFrameSize) * 1000000) - / mSampleRate; + mStartPosUs = mPositionTimeMediaUs; + ALOGV("adjust seek time to: %.2f", mStartPosUs/ 1E6); + } + // clear seek time with mLock locked and once we have valid mPositionTimeMediaUs + // and mPositionTimeRealUs + // before clearing mSeekTimeUs check if a new seek request has been received while + // we were reading from the source with mLock released. + if (!mSeeking) { + mSeekTimeUs = 0; + } + } + + if (!useOffload()) { + mPositionTimeRealUs = + ((mNumFramesPlayed + size_done / mFrameSize) * 1000000) + / mSampleRate; + ALOGV("buffer->size() = %d, " + "mPositionTimeMediaUs=%.2f mPositionTimeRealUs=%.2f", + mInputBuffer->range_length(), + mPositionTimeMediaUs / 1E6, mPositionTimeRealUs / 1E6); + } - ALOGV("buffer->size() = %d, " - "mPositionTimeMediaUs=%.2f mPositionTimeRealUs=%.2f", - mInputBuffer->range_length(), - mPositionTimeMediaUs / 1E6, mPositionTimeRealUs / 1E6); } if (mInputBuffer->range_length() == 0) { @@ -490,6 +658,13 @@ size_t AudioPlayer::fillBuffer(void *data, size_t size) { size_remaining -= copy; } + if (useOffload()) { + // We must ask the hardware what it has played + mPositionTimeRealUs = getOutputPlayPositionUs_l(); + ALOGV("mPositionTimeMediaUs=%.2f mPositionTimeRealUs=%.2f", + mPositionTimeMediaUs / 1E6, mPositionTimeRealUs / 1E6); + } + { Mutex::Autolock autoLock(mLock); mNumFramesPlayed += size_done / mFrameSize; @@ -515,6 +690,14 @@ size_t AudioPlayer::fillBuffer(void *data, size_t size) { int64_t AudioPlayer::getRealTimeUs() { Mutex::Autolock autoLock(mLock); + if (useOffload()) { + if (mSeeking) { + return mSeekTimeUs; + } + mPositionTimeRealUs = getOutputPlayPositionUs_l(); + return mPositionTimeRealUs; + } + return getRealTimeUsLocked(); } @@ -538,15 +721,40 @@ int64_t AudioPlayer::getRealTimeUsLocked() const { return result + diffUs; } +int64_t AudioPlayer::getOutputPlayPositionUs_l() const +{ + uint32_t playedSamples = 0; + if (mAudioSink != NULL) { + mAudioSink->getPosition(&playedSamples); + } else { + mAudioTrack->getPosition(&playedSamples); + } + + const int64_t playedUs = (static_cast<int64_t>(playedSamples) * 1000000 ) / mSampleRate; + + // HAL position is relative to the first buffer we sent at mStartPosUs + const int64_t renderedDuration = mStartPosUs + playedUs; + ALOGV("getOutputPlayPositionUs_l %lld", renderedDuration); + return renderedDuration; +} + int64_t AudioPlayer::getMediaTimeUs() { Mutex::Autolock autoLock(mLock); - if (mPositionTimeMediaUs < 0 || mPositionTimeRealUs < 0) { + if (useOffload()) { if (mSeeking) { return mSeekTimeUs; } + mPositionTimeRealUs = getOutputPlayPositionUs_l(); + ALOGV("getMediaTimeUs getOutputPlayPositionUs_l() mPositionTimeRealUs %lld", + mPositionTimeRealUs); + return mPositionTimeRealUs; + } - return 0; + + if (mPositionTimeMediaUs < 0 || mPositionTimeRealUs < 0) { + // mSeekTimeUs is either seek time while seeking or 0 if playback did not start. + return mSeekTimeUs; } int64_t realTimeOffset = getRealTimeUsLocked() - mPositionTimeRealUs; @@ -561,8 +769,14 @@ bool AudioPlayer::getMediaTimeMapping( int64_t *realtime_us, int64_t *mediatime_us) { Mutex::Autolock autoLock(mLock); - *realtime_us = mPositionTimeRealUs; - *mediatime_us = mPositionTimeMediaUs; + if (useOffload()) { + mPositionTimeRealUs = getOutputPlayPositionUs_l(); + *realtime_us = mPositionTimeRealUs; + *mediatime_us = mPositionTimeRealUs; + } else { + *realtime_us = mPositionTimeRealUs; + *mediatime_us = mPositionTimeMediaUs; + } return mPositionTimeRealUs != -1 && mPositionTimeMediaUs != -1; } @@ -570,19 +784,34 @@ bool AudioPlayer::getMediaTimeMapping( status_t AudioPlayer::seekTo(int64_t time_us) { Mutex::Autolock autoLock(mLock); + ALOGV("seekTo( %lld )", time_us); + mSeeking = true; mPositionTimeRealUs = mPositionTimeMediaUs = -1; mReachedEOS = false; mSeekTimeUs = time_us; + mStartPosUs = time_us; // Flush resets the number of played frames mNumFramesPlayed = 0; mNumFramesPlayedSysTimeUs = ALooper::GetNowUs(); if (mAudioSink != NULL) { + if (mPlaying) { + mAudioSink->pause(); + } mAudioSink->flush(); + if (mPlaying) { + mAudioSink->start(); + } } else { + if (mPlaying) { + mAudioTrack->pause(); + } mAudioTrack->flush(); + if (mPlaying) { + mAudioTrack->start(); + } } return OK; diff --git a/media/libstagefright/AudioSource.cpp b/media/libstagefright/AudioSource.cpp index 3cf4d5c..bdd842f 100644 --- a/media/libstagefright/AudioSource.cpp +++ b/media/libstagefright/AudioSource.cpp @@ -49,8 +49,7 @@ static void AudioRecordCallbackFunction(int event, void *user, void *info) { AudioSource::AudioSource( audio_source_t inputSource, uint32_t sampleRate, uint32_t channelCount) - : mRecord(NULL), - mStarted(false), + : mStarted(false), mSampleRate(sampleRate), mPrevSampleTimeUs(0), mNumFramesReceived(0), @@ -91,9 +90,6 @@ AudioSource::~AudioSource() { if (mStarted) { reset(); } - - delete mRecord; - mRecord = NULL; } status_t AudioSource::initCheck() const { @@ -122,8 +118,7 @@ status_t AudioSource::start(MetaData *params) { if (err == OK) { mStarted = true; } else { - delete mRecord; - mRecord = NULL; + mRecord.clear(); } diff --git a/media/libstagefright/AwesomePlayer.cpp b/media/libstagefright/AwesomePlayer.cpp index d53f442..c912f75 100644 --- a/media/libstagefright/AwesomePlayer.cpp +++ b/media/libstagefright/AwesomePlayer.cpp @@ -47,6 +47,7 @@ #include <media/stagefright/MediaSource.h> #include <media/stagefright/MetaData.h> #include <media/stagefright/OMXCodec.h> +#include <media/stagefright/Utils.h> #include <gui/IGraphicBufferProducer.h> #include <gui/Surface.h> @@ -65,6 +66,11 @@ static int64_t kHighWaterMarkUs = 5000000ll; // 5secs static const size_t kLowWaterMarkBytes = 40000; static const size_t kHighWaterMarkBytes = 200000; +// maximum time in paused state when offloading audio decompression. When elapsed, the AudioPlayer +// is destroyed to allow the audio DSP to power down. +static int64_t kOffloadPauseMaxUs = 60000000ll; + + struct AwesomeEvent : public TimedEventQueue::Event { AwesomeEvent( AwesomePlayer *player, @@ -185,6 +191,8 @@ AwesomePlayer::AwesomePlayer() mTimeSource(NULL), mVideoRenderingStarted(false), mVideoRendererIsPreview(false), + mMediaRenderingStartGeneration(0), + mStartGeneration(0), mAudioPlayer(NULL), mDisplayWidth(0), mDisplayHeight(0), @@ -194,7 +202,9 @@ AwesomePlayer::AwesomePlayer() mVideoBuffer(NULL), mDecryptHandle(NULL), mLastVideoTimeUs(-1), - mTextDriver(NULL) { + mTextDriver(NULL), + mOffloadAudio(false), + mAudioTearDown(false) { CHECK_EQ(mClient.connect(), (status_t)OK); DataSource::RegisterDefaultSniffers(); @@ -206,13 +216,17 @@ AwesomePlayer::AwesomePlayer() mBufferingEvent = new AwesomeEvent(this, &AwesomePlayer::onBufferingUpdate); mBufferingEventPending = false; mVideoLagEvent = new AwesomeEvent(this, &AwesomePlayer::onVideoLagUpdate); - mVideoEventPending = false; + mVideoLagEventPending = false; mCheckAudioStatusEvent = new AwesomeEvent( this, &AwesomePlayer::onCheckAudioStatus); mAudioStatusEventPending = false; + mAudioTearDownEvent = new AwesomeEvent(this, + &AwesomePlayer::onAudioTearDownEvent); + mAudioTearDownEventPending = false; + reset(); } @@ -232,6 +246,11 @@ void AwesomePlayer::cancelPlayerEvents(bool keepNotifications) { mQueue.cancelEvent(mVideoLagEvent->eventID()); mVideoLagEventPending = false; + if (mOffloadAudio) { + mQueue.cancelEvent(mAudioTearDownEvent->eventID()); + mAudioTearDownEventPending = false; + } + if (!keepNotifications) { mQueue.cancelEvent(mStreamDoneEvent->eventID()); mStreamDoneEventPending = false; @@ -240,6 +259,7 @@ void AwesomePlayer::cancelPlayerEvents(bool keepNotifications) { mQueue.cancelEvent(mBufferingEvent->eventID()); mBufferingEventPending = false; + mAudioTearDown = false; } } @@ -474,6 +494,8 @@ void AwesomePlayer::reset_l() { mDisplayWidth = 0; mDisplayHeight = 0; + notifyListener_l(MEDIA_STOPPED); + if (mDecryptHandle != NULL) { mDrmManagerClient->setPlaybackStatus(mDecryptHandle, Playback::STOP, 0); @@ -518,7 +540,7 @@ void AwesomePlayer::reset_l() { mVideoTrack.clear(); mExtractor.clear(); - // Shutdown audio first, so that the respone to the reset request + // Shutdown audio first, so that the response to the reset request // appears to happen instantaneously as far as the user is concerned // If we did this later, audio would continue playing while we // shutdown the video-related resources and the player appear to @@ -531,6 +553,7 @@ void AwesomePlayer::reset_l() { mAudioSource->stop(); } mAudioSource.clear(); + mOmxSource.clear(); mTimeSource = NULL; @@ -586,7 +609,7 @@ void AwesomePlayer::reset_l() { } void AwesomePlayer::notifyListener_l(int msg, int ext1, int ext2) { - if (mListener != NULL) { + if ((mListener != NULL) && !mAudioTearDown) { sp<MediaPlayerBase> listener = mListener.promote(); if (listener != NULL) { @@ -597,7 +620,7 @@ void AwesomePlayer::notifyListener_l(int msg, int ext1, int ext2) { bool AwesomePlayer::getBitrate(int64_t *bitrate) { off64_t size; - if (mDurationUs >= 0 && mCachedSource != NULL + if (mDurationUs > 0 && mCachedSource != NULL && mCachedSource->getSize(&size) == OK) { *bitrate = size * 8000000ll / mDurationUs; // in bits/sec return true; @@ -775,7 +798,9 @@ void AwesomePlayer::onBufferingUpdate() { } } - postBufferingEvent_l(); + if (mFlags & (PLAYING | PREPARING)) { + postBufferingEvent_l(); + } } void AwesomePlayer::sendCacheStats() { @@ -842,6 +867,13 @@ void AwesomePlayer::onStreamDone() { pause_l(true /* at eos */); + // If audio hasn't completed MEDIA_SEEK_COMPLETE yet, + // notify MEDIA_SEEK_COMPLETE to observer immediately for state persistence. + if (mWatchForAudioSeekComplete) { + notifyListener_l(MEDIA_SEEK_COMPLETE); + mWatchForAudioSeekComplete = false; + } + modifyFlags(AT_EOS, SET); } } @@ -883,41 +915,49 @@ status_t AwesomePlayer::play_l() { if (mAudioSource != NULL) { if (mAudioPlayer == NULL) { - if (mAudioSink != NULL) { - bool allowDeepBuffering; - int64_t cachedDurationUs; - bool eos; - if (mVideoSource == NULL - && (mDurationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US || - (getCachedDuration_l(&cachedDurationUs, &eos) && - cachedDurationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US))) { - allowDeepBuffering = true; - } else { - allowDeepBuffering = false; - } - - mAudioPlayer = new AudioPlayer(mAudioSink, allowDeepBuffering, this); - mAudioPlayer->setSource(mAudioSource); - - mTimeSource = mAudioPlayer; - - // If there was a seek request before we ever started, - // honor the request now. - // Make sure to do this before starting the audio player - // to avoid a race condition. - seekAudioIfNecessary_l(); - } + createAudioPlayer_l(); } CHECK(!(mFlags & AUDIO_RUNNING)); if (mVideoSource == NULL) { + // We don't want to post an error notification at this point, // the error returned from MediaPlayer::start() will suffice. status_t err = startAudioPlayer_l( false /* sendErrorNotification */); + if ((err != OK) && mOffloadAudio) { + ALOGI("play_l() cannot create offload output, fallback to sw decode"); + int64_t curTimeUs; + getPosition(&curTimeUs); + + delete mAudioPlayer; + mAudioPlayer = NULL; + // if the player was started it will take care of stopping the source when destroyed + if (!(mFlags & AUDIOPLAYER_STARTED)) { + mAudioSource->stop(); + } + modifyFlags((AUDIO_RUNNING | AUDIOPLAYER_STARTED), CLEAR); + mOffloadAudio = false; + mAudioSource = mOmxSource; + if (mAudioSource != NULL) { + err = mAudioSource->start(); + + if (err != OK) { + mAudioSource.clear(); + } else { + mSeekNotificationSent = true; + if (mExtractorFlags & MediaExtractor::CAN_SEEK) { + seekTo_l(curTimeUs); + } + createAudioPlayer_l(); + err = startAudioPlayer_l(false); + } + } + } + if (err != OK) { delete mAudioPlayer; mAudioPlayer = NULL; @@ -963,22 +1003,72 @@ status_t AwesomePlayer::play_l() { } addBatteryData(params); + if (isStreamingHTTP()) { + postBufferingEvent_l(); + } + return OK; } +void AwesomePlayer::createAudioPlayer_l() +{ + uint32_t flags = 0; + int64_t cachedDurationUs; + bool eos; + + if (mOffloadAudio) { + flags |= AudioPlayer::USE_OFFLOAD; + } else if (mVideoSource == NULL + && (mDurationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US || + (getCachedDuration_l(&cachedDurationUs, &eos) && + cachedDurationUs > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US))) { + flags |= AudioPlayer::ALLOW_DEEP_BUFFERING; + } + if (isStreamingHTTP()) { + flags |= AudioPlayer::IS_STREAMING; + } + if (mVideoSource != NULL) { + flags |= AudioPlayer::HAS_VIDEO; + } + + mAudioPlayer = new AudioPlayer(mAudioSink, flags, this); + mAudioPlayer->setSource(mAudioSource); + + mTimeSource = mAudioPlayer; + + // If there was a seek request before we ever started, + // honor the request now. + // Make sure to do this before starting the audio player + // to avoid a race condition. + seekAudioIfNecessary_l(); +} + +void AwesomePlayer::notifyIfMediaStarted_l() { + if (mMediaRenderingStartGeneration == mStartGeneration) { + mMediaRenderingStartGeneration = -1; + notifyListener_l(MEDIA_STARTED); + } +} + status_t AwesomePlayer::startAudioPlayer_l(bool sendErrorNotification) { CHECK(!(mFlags & AUDIO_RUNNING)); + status_t err = OK; if (mAudioSource == NULL || mAudioPlayer == NULL) { return OK; } + if (mOffloadAudio) { + mQueue.cancelEvent(mAudioTearDownEvent->eventID()); + mAudioTearDownEventPending = false; + } + if (!(mFlags & AUDIOPLAYER_STARTED)) { bool wasSeeking = mAudioPlayer->isSeeking(); // We've already started the MediaSource in order to enable // the prefetcher to read its data. - status_t err = mAudioPlayer->start( + err = mAudioPlayer->start( true /* sourceAlreadyStarted */); if (err != OK) { @@ -996,16 +1086,20 @@ status_t AwesomePlayer::startAudioPlayer_l(bool sendErrorNotification) { // We will have finished the seek while starting the audio player. postAudioSeekComplete(); + } else { + notifyIfMediaStarted_l(); } } else { - mAudioPlayer->resume(); + err = mAudioPlayer->resume(); } - modifyFlags(AUDIO_RUNNING, SET); + if (err == OK) { + modifyFlags(AUDIO_RUNNING, SET); - mWatchForAudioEOS = true; + mWatchForAudioEOS = true; + } - return OK; + return err; } void AwesomePlayer::notifyVideoSize_l() { @@ -1131,21 +1225,29 @@ status_t AwesomePlayer::pause() { status_t AwesomePlayer::pause_l(bool at_eos) { if (!(mFlags & PLAYING)) { + if (mAudioTearDown && mAudioTearDownWasPlaying) { + ALOGV("pause_l() during teardown and finishSetDataSource_l() mFlags %x" , mFlags); + mAudioTearDownWasPlaying = false; + notifyListener_l(MEDIA_PAUSED); + mMediaRenderingStartGeneration = ++mStartGeneration; + } return OK; } + notifyListener_l(MEDIA_PAUSED); + mMediaRenderingStartGeneration = ++mStartGeneration; + cancelPlayerEvents(true /* keepNotifications */); if (mAudioPlayer != NULL && (mFlags & AUDIO_RUNNING)) { - if (at_eos) { - // If we played the audio stream to completion we - // want to make sure that all samples remaining in the audio - // track's queue are played out. - mAudioPlayer->pause(true /* playPendingSamples */); - } else { - mAudioPlayer->pause(); + // If we played the audio stream to completion we + // want to make sure that all samples remaining in the audio + // track's queue are played out. + mAudioPlayer->pause(at_eos /* playPendingSamples */); + // send us a reminder to tear down the AudioPlayer if paused for too long. + if (mOffloadAudio) { + postAudioTearDownEvent(kOffloadPauseMaxUs); } - modifyFlags(AUDIO_RUNNING, CLEAR); } @@ -1290,7 +1392,6 @@ status_t AwesomePlayer::getPosition(int64_t *positionUs) { } else { *positionUs = 0; } - return OK; } @@ -1324,6 +1425,11 @@ status_t AwesomePlayer::seekTo_l(int64_t timeUs) { mSeekTimeUs = timeUs; modifyFlags((AT_EOS | AUDIO_AT_EOS | VIDEO_AT_EOS), CLEAR); + if (mFlags & PLAYING) { + notifyListener_l(MEDIA_PAUSED); + mMediaRenderingStartGeneration = ++mStartGeneration; + } + seekAudioIfNecessary_l(); if (mFlags & TEXTPLAYER_INITIALIZED) { @@ -1385,14 +1491,29 @@ status_t AwesomePlayer::initAudioDecoder() { const char *mime; CHECK(meta->findCString(kKeyMIMEType, &mime)); + // Check whether there is a hardware codec for this stream + // This doesn't guarantee that the hardware has a free stream + // but it avoids us attempting to open (and re-open) an offload + // stream to hardware that doesn't have the necessary codec + mOffloadAudio = canOffloadStream(meta, (mVideoSource != NULL), isStreamingHTTP()); if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_RAW)) { + ALOGV("createAudioPlayer: bypass OMX (raw)"); mAudioSource = mAudioTrack; } else { - mAudioSource = OMXCodec::Create( + // If offloading we still create a OMX decoder as a fall-back + // but we don't start it + mOmxSource = OMXCodec::Create( mClient.interface(), mAudioTrack->getFormat(), false, // createEncoder mAudioTrack); + + if (mOffloadAudio) { + ALOGV("createAudioPlayer: bypass OMX (offload)"); + mAudioSource = mAudioTrack; + } else { + mAudioSource = mOmxSource; + } } if (mAudioSource != NULL) { @@ -1408,6 +1529,7 @@ status_t AwesomePlayer::initAudioDecoder() { if (err != OK) { mAudioSource.clear(); + mOmxSource.clear(); return err; } } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_QCELP)) { @@ -1551,6 +1673,16 @@ void AwesomePlayer::finishSeekIfNecessary(int64_t videoTimeUs) { return; } + // If we paused, then seeked, then resumed, it is possible that we have + // signaled SEEK_COMPLETE at a copmletely different media time than where + // we are now resuming. Signal new position to media time provider. + // Cannot signal another SEEK_COMPLETE, as existing clients may not expect + // multiple SEEK_COMPLETE responses to a single seek() request. + if (mSeekNotificationSent && abs(mSeekTimeUs - videoTimeUs) > 10000) { + // notify if we are resuming more than 10ms away from desired seek time + notifyListener_l(MEDIA_SKIPPED); + } + if (mAudioPlayer != NULL) { ALOGV("seeking audio to %lld us (%.2f secs).", videoTimeUs, videoTimeUs / 1E6); @@ -1822,6 +1954,9 @@ void AwesomePlayer::onVideoEvent() { notifyListener_l(MEDIA_INFO, MEDIA_INFO_RENDERING_START); } + if (mFlags & PLAYING) { + notifyIfMediaStarted_l(); + } } mVideoBuffer->release(); @@ -1885,6 +2020,15 @@ void AwesomePlayer::postCheckAudioStatusEvent(int64_t delayUs) { mQueue.postEventWithDelay(mCheckAudioStatusEvent, delayUs); } +void AwesomePlayer::postAudioTearDownEvent(int64_t delayUs) { + Mutex::Autolock autoLock(mAudioLock); + if (mAudioTearDownEventPending) { + return; + } + mAudioTearDownEventPending = true; + mQueue.postEventWithDelay(mAudioTearDownEvent, delayUs); +} + void AwesomePlayer::onCheckAudioStatus() { { Mutex::Autolock autoLock(mAudioLock); @@ -1908,6 +2052,8 @@ void AwesomePlayer::onCheckAudioStatus() { } mSeeking = NO_SEEK; + + notifyIfMediaStarted_l(); } status_t finalStatus; @@ -2189,6 +2335,7 @@ void AwesomePlayer::abortPrepare(status_t err) { modifyFlags((PREPARING|PREPARE_CANCELLED|PREPARING_CONNECTED), CLEAR); mAsyncPrepareEvent = NULL; mPreparedCondition.broadcast(); + mAudioTearDown = false; } // static @@ -2200,7 +2347,10 @@ bool AwesomePlayer::ContinuePreparation(void *cookie) { void AwesomePlayer::onPrepareAsyncEvent() { Mutex::Autolock autoLock(mLock); + beginPrepareAsync_l(); +} +void AwesomePlayer::beginPrepareAsync_l() { if (mFlags & PREPARE_CANCELLED) { ALOGI("prepare was cancelled before doing anything"); abortPrepare(UNKNOWN_ERROR); @@ -2259,6 +2409,20 @@ void AwesomePlayer::finishAsyncPrepare_l() { modifyFlags(PREPARED, SET); mAsyncPrepareEvent = NULL; mPreparedCondition.broadcast(); + + if (mAudioTearDown) { + if (mPrepareResult == OK) { + if (mExtractorFlags & MediaExtractor::CAN_SEEK) { + seekTo_l(mAudioTearDownPosition); + } + + if (mAudioTearDownWasPlaying) { + modifyFlags(CACHE_UNDERRUN, CLEAR); + play_l(); + } + } + mAudioTearDown = false; + } } uint32_t AwesomePlayer::flags() const { @@ -2273,6 +2437,10 @@ void AwesomePlayer::postAudioSeekComplete() { postCheckAudioStatusEvent(0); } +void AwesomePlayer::postAudioTearDown() { + postAudioTearDownEvent(0); +} + status_t AwesomePlayer::setParameter(int key, const Parcel &request) { switch (key) { case KEY_PARAMETER_CACHE_STAT_COLLECT_FREQ_MS: @@ -2404,6 +2572,7 @@ status_t AwesomePlayer::selectAudioTrack_l( mAudioSource->stop(); } mAudioSource.clear(); + mOmxSource.clear(); mTimeSource = NULL; @@ -2660,4 +2829,52 @@ void AwesomePlayer::modifyFlags(unsigned value, FlagMode mode) { } } +void AwesomePlayer::onAudioTearDownEvent() { + + Mutex::Autolock autoLock(mLock); + if (!mAudioTearDownEventPending) { + return; + } + mAudioTearDownEventPending = false; + + ALOGV("onAudioTearDownEvent"); + + // stream info is cleared by reset_l() so copy what we need + mAudioTearDownWasPlaying = (mFlags & PLAYING); + KeyedVector<String8, String8> uriHeaders(mUriHeaders); + sp<DataSource> fileSource(mFileSource); + + mStatsLock.lock(); + String8 uri(mStats.mURI); + mStatsLock.unlock(); + + // get current position so we can start recreated stream from here + getPosition(&mAudioTearDownPosition); + + // Reset and recreate + reset_l(); + + status_t err; + + if (fileSource != NULL) { + mFileSource = fileSource; + err = setDataSource_l(fileSource); + } else { + err = setDataSource_l(uri, &uriHeaders); + } + + mFlags |= PREPARING; + if ( err != OK ) { + // This will force beingPrepareAsync_l() to notify + // a MEDIA_ERROR to the client and abort the prepare + mFlags |= PREPARE_CANCELLED; + } + + mAudioTearDown = true; + mIsAsyncPrepare = true; + + // Call prepare for the host decoding + beginPrepareAsync_l(); +} + } // namespace android diff --git a/media/libstagefright/CameraSource.cpp b/media/libstagefright/CameraSource.cpp index 5a26b06..3017fe7 100644 --- a/media/libstagefright/CameraSource.cpp +++ b/media/libstagefright/CameraSource.cpp @@ -536,7 +536,7 @@ status_t CameraSource::initWithCameraAccess( if (mSurface != NULL) { // This CHECK is good, since we just passed the lock/unlock // check earlier by calling mCamera->setParameters(). - CHECK_EQ((status_t)OK, mCamera->setPreviewTexture(mSurface)); + CHECK_EQ((status_t)OK, mCamera->setPreviewTarget(mSurface)); } // By default, do not store metadata in video buffers diff --git a/media/libstagefright/HTTPBase.cpp b/media/libstagefright/HTTPBase.cpp index d2cc6c2..5fa4b6f 100644 --- a/media/libstagefright/HTTPBase.cpp +++ b/media/libstagefright/HTTPBase.cpp @@ -30,6 +30,8 @@ #include <cutils/properties.h> #include <cutils/qtaguid.h> +#include <ConnectivityManager.h> + namespace android { HTTPBase::HTTPBase() @@ -164,4 +166,14 @@ void HTTPBase::UnRegisterSocketUserTag(int sockfd) { } } +// static +void HTTPBase::RegisterSocketUserMark(int sockfd, uid_t uid) { + ConnectivityManager::markSocketAsUser(sockfd, uid); +} + +// static +void HTTPBase::UnRegisterSocketUserMark(int sockfd) { + RegisterSocketUserMark(sockfd, geteuid()); +} + } // namespace android diff --git a/media/libstagefright/MPEG4Extractor.cpp b/media/libstagefright/MPEG4Extractor.cpp index 25c080c..cbc169b 100644 --- a/media/libstagefright/MPEG4Extractor.cpp +++ b/media/libstagefright/MPEG4Extractor.cpp @@ -343,6 +343,7 @@ MPEG4Extractor::MPEG4Extractor(const sp<DataSource> &source) mDataSource(source), mInitCheck(NO_INIT), mHasVideo(false), + mHeaderTimescale(0), mFirstTrack(NULL), mLastTrack(NULL), mFileMetaData(new MetaData), @@ -819,6 +820,7 @@ status_t MPEG4Extractor::parseChunk(off64_t *offset, int depth) { case FOURCC('i', 'l', 's', 't'): case FOURCC('s', 'i', 'n', 'f'): case FOURCC('s', 'c', 'h', 'i'): + case FOURCC('e', 'd', 't', 's'): { if (chunk_type == FOURCC('s', 't', 'b', 'l')) { ALOGV("sampleTable chunk is %d bytes long.", (size_t)chunk_size); @@ -906,6 +908,68 @@ status_t MPEG4Extractor::parseChunk(off64_t *offset, int depth) { break; } + case FOURCC('e', 'l', 's', 't'): + { + // See 14496-12 8.6.6 + uint8_t version; + if (mDataSource->readAt(data_offset, &version, 1) < 1) { + return ERROR_IO; + } + + uint32_t entry_count; + if (!mDataSource->getUInt32(data_offset + 4, &entry_count)) { + return ERROR_IO; + } + + if (entry_count != 1) { + // we only support a single entry at the moment, for gapless playback + ALOGW("ignoring edit list with %d entries", entry_count); + } else if (mHeaderTimescale == 0) { + ALOGW("ignoring edit list because timescale is 0"); + } else { + off64_t entriesoffset = data_offset + 8; + uint64_t segment_duration; + int64_t media_time; + + if (version == 1) { + if (!mDataSource->getUInt64(entriesoffset, &segment_duration) || + !mDataSource->getUInt64(entriesoffset + 8, (uint64_t*)&media_time)) { + return ERROR_IO; + } + } else if (version == 0) { + uint32_t sd; + int32_t mt; + if (!mDataSource->getUInt32(entriesoffset, &sd) || + !mDataSource->getUInt32(entriesoffset + 4, (uint32_t*)&mt)) { + return ERROR_IO; + } + segment_duration = sd; + media_time = mt; + } else { + return ERROR_IO; + } + + uint64_t halfscale = mHeaderTimescale / 2; + segment_duration = (segment_duration * 1000000 + halfscale)/ mHeaderTimescale; + media_time = (media_time * 1000000 + halfscale) / mHeaderTimescale; + + int64_t duration; + int32_t samplerate; + if (mLastTrack->meta->findInt64(kKeyDuration, &duration) && + mLastTrack->meta->findInt32(kKeySampleRate, &samplerate)) { + + int64_t delay = (media_time * samplerate + 500000) / 1000000; + mLastTrack->meta->setInt32(kKeyEncoderDelay, delay); + + int64_t paddingus = duration - (segment_duration + media_time); + int64_t paddingsamples = (paddingus * samplerate + 500000) / 1000000; + mLastTrack->meta->setInt32(kKeyEncoderPadding, paddingsamples); + } + } + *offset += chunk_size; + break; + } + case FOURCC('f', 'r', 'm', 'a'): { uint32_t original_fourcc; @@ -1566,24 +1630,26 @@ status_t MPEG4Extractor::parseChunk(off64_t *offset, int depth) { case FOURCC('m', 'v', 'h', 'd'): { - if (chunk_data_size < 12) { + if (chunk_data_size < 24) { return ERROR_MALFORMED; } - uint8_t header[12]; + uint8_t header[24]; if (mDataSource->readAt( data_offset, header, sizeof(header)) < (ssize_t)sizeof(header)) { return ERROR_IO; } - int64_t creationTime; + uint64_t creationTime; if (header[0] == 1) { creationTime = U64_AT(&header[4]); + mHeaderTimescale = U32_AT(&header[20]); } else if (header[0] != 0) { return ERROR_MALFORMED; } else { creationTime = U32_AT(&header[4]); + mHeaderTimescale = U32_AT(&header[12]); } String8 s; @@ -1877,13 +1943,13 @@ status_t MPEG4Extractor::parseTrackHeader( mtime = U64_AT(&buffer[12]); id = U32_AT(&buffer[20]); duration = U64_AT(&buffer[28]); - } else { - CHECK_EQ((unsigned)version, 0u); - + } else if (version == 0) { ctime = U32_AT(&buffer[4]); mtime = U32_AT(&buffer[8]); id = U32_AT(&buffer[12]); duration = U32_AT(&buffer[20]); + } else { + return ERROR_UNSUPPORTED; } mLastTrack->meta->setInt32(kKeyTrackID, id); diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp index 674e6e5..e299caf 100644 --- a/media/libstagefright/MediaCodec.cpp +++ b/media/libstagefright/MediaCodec.cpp @@ -31,6 +31,7 @@ #include <media/stagefright/foundation/hexdump.h> #include <media/stagefright/ACodec.h> #include <media/stagefright/BufferProducerWrapper.h> +#include <media/stagefright/MediaCodecList.h> #include <media/stagefright/MediaDefs.h> #include <media/stagefright/MediaErrors.h> #include <media/stagefright/MetaData.h> @@ -104,8 +105,24 @@ status_t MediaCodec::init(const char *name, bool nameIsType, bool encoder) { bool needDedicatedLooper = false; if (nameIsType && !strncasecmp(name, "video/", 6)) { needDedicatedLooper = true; - } else if (!nameIsType && !strncmp(name, "OMX.TI.DUCATI1.VIDEO.", 21)) { - needDedicatedLooper = true; + } else { + AString tmp = name; + if (tmp.endsWith(".secure")) { + tmp.erase(tmp.size() - 7, 7); + } + const MediaCodecList *mcl = MediaCodecList::getInstance(); + ssize_t codecIdx = mcl->findCodecByName(tmp.c_str()); + if (codecIdx >= 0) { + Vector<AString> types; + if (mcl->getSupportedTypes(codecIdx, &types) == OK) { + for (int i = 0; i < types.size(); i++) { + if (types[i].startsWith("video/")) { + needDedicatedLooper = true; + break; + } + } + } + } } if (needDedicatedLooper) { diff --git a/media/libstagefright/MediaCodecList.cpp b/media/libstagefright/MediaCodecList.cpp index d24337f..6248e90 100644 --- a/media/libstagefright/MediaCodecList.cpp +++ b/media/libstagefright/MediaCodecList.cpp @@ -509,7 +509,8 @@ status_t MediaCodecList::getSupportedTypes( status_t MediaCodecList::getCodecCapabilities( size_t index, const char *type, Vector<ProfileLevel> *profileLevels, - Vector<uint32_t> *colorFormats) const { + Vector<uint32_t> *colorFormats, + uint32_t *flags) const { profileLevels->clear(); colorFormats->clear(); @@ -547,6 +548,8 @@ status_t MediaCodecList::getCodecCapabilities( colorFormats->push(caps.mColorFormats.itemAt(i)); } + *flags = caps.mFlags; + return OK; } diff --git a/media/libstagefright/MediaDefs.cpp b/media/libstagefright/MediaDefs.cpp index 5d8029c..b5d4e44 100644 --- a/media/libstagefright/MediaDefs.cpp +++ b/media/libstagefright/MediaDefs.cpp @@ -20,7 +20,8 @@ namespace android { const char *MEDIA_MIMETYPE_IMAGE_JPEG = "image/jpeg"; -const char *MEDIA_MIMETYPE_VIDEO_VPX = "video/x-vnd.on2.vp8"; +const char *MEDIA_MIMETYPE_VIDEO_VP8 = "video/x-vnd.on2.vp8"; +const char *MEDIA_MIMETYPE_VIDEO_VP9 = "video/x-vnd.on2.vp9"; const char *MEDIA_MIMETYPE_VIDEO_AVC = "video/avc"; const char *MEDIA_MIMETYPE_VIDEO_MPEG4 = "video/mp4v-es"; const char *MEDIA_MIMETYPE_VIDEO_H263 = "video/3gpp"; diff --git a/media/libstagefright/MediaMuxer.cpp b/media/libstagefright/MediaMuxer.cpp index 94ce5de..d87e910 100644 --- a/media/libstagefright/MediaMuxer.cpp +++ b/media/libstagefright/MediaMuxer.cpp @@ -103,6 +103,16 @@ status_t MediaMuxer::setOrientationHint(int degrees) { return OK; } +status_t MediaMuxer::setLocation(int latitude, int longitude) { + Mutex::Autolock autoLock(mMuxerLock); + if (mState != INITIALIZED) { + ALOGE("setLocation() must be called before start()."); + return INVALID_OPERATION; + } + ALOGV("Setting location: latitude = %d, longitude = %d", latitude, longitude); + return mWriter->setGeoData(latitude, longitude); +} + status_t MediaMuxer::start() { Mutex::Autolock autoLock(mMuxerLock); if (mState == INITIALIZED) { diff --git a/media/libstagefright/OMXClient.cpp b/media/libstagefright/OMXClient.cpp index 1822f07..9f9352d 100644 --- a/media/libstagefright/OMXClient.cpp +++ b/media/libstagefright/OMXClient.cpp @@ -69,6 +69,10 @@ struct MuxOMX : public IOMX { virtual status_t storeMetaDataInBuffers( node_id node, OMX_U32 port_index, OMX_BOOL enable); + virtual status_t prepareForAdaptivePlayback( + node_id node, OMX_U32 port_index, OMX_BOOL enable, + OMX_U32 maxFrameWidth, OMX_U32 maxFrameHeight); + virtual status_t enableGraphicBuffers( node_id node, OMX_U32 port_index, OMX_BOOL enable); @@ -83,6 +87,10 @@ struct MuxOMX : public IOMX { node_id node, OMX_U32 port_index, const sp<GraphicBuffer> &graphicBuffer, buffer_id *buffer); + virtual status_t updateGraphicBufferInMeta( + node_id node, OMX_U32 port_index, + const sp<GraphicBuffer> &graphicBuffer, buffer_id buffer); + virtual status_t createInputSurface( node_id node, OMX_U32 port_index, sp<IGraphicBufferProducer> *bufferProducer); @@ -113,6 +121,13 @@ struct MuxOMX : public IOMX { const char *parameter_name, OMX_INDEXTYPE *index); + virtual status_t setInternalOption( + node_id node, + OMX_U32 port_index, + InternalOptionType type, + const void *data, + size_t size); + private: mutable Mutex mLock; @@ -257,6 +272,13 @@ status_t MuxOMX::storeMetaDataInBuffers( return getOMX(node)->storeMetaDataInBuffers(node, port_index, enable); } +status_t MuxOMX::prepareForAdaptivePlayback( + node_id node, OMX_U32 port_index, OMX_BOOL enable, + OMX_U32 maxFrameWidth, OMX_U32 maxFrameHeight) { + return getOMX(node)->prepareForAdaptivePlayback( + node, port_index, enable, maxFrameWidth, maxFrameHeight); +} + status_t MuxOMX::enableGraphicBuffers( node_id node, OMX_U32 port_index, OMX_BOOL enable) { return getOMX(node)->enableGraphicBuffers(node, port_index, enable); @@ -280,6 +302,13 @@ status_t MuxOMX::useGraphicBuffer( node, port_index, graphicBuffer, buffer); } +status_t MuxOMX::updateGraphicBufferInMeta( + node_id node, OMX_U32 port_index, + const sp<GraphicBuffer> &graphicBuffer, buffer_id buffer) { + return getOMX(node)->updateGraphicBufferInMeta( + node, port_index, graphicBuffer, buffer); +} + status_t MuxOMX::createInputSurface( node_id node, OMX_U32 port_index, sp<IGraphicBufferProducer> *bufferProducer) { @@ -331,6 +360,15 @@ status_t MuxOMX::getExtensionIndex( return getOMX(node)->getExtensionIndex(node, parameter_name, index); } +status_t MuxOMX::setInternalOption( + node_id node, + OMX_U32 port_index, + InternalOptionType type, + const void *data, + size_t size) { + return getOMX(node)->setInternalOption(node, port_index, type, data, size); +} + OMXClient::OMXClient() { } diff --git a/media/libstagefright/OMXCodec.cpp b/media/libstagefright/OMXCodec.cpp index 9d349a1..7f56af8 100644 --- a/media/libstagefright/OMXCodec.cpp +++ b/media/libstagefright/OMXCodec.cpp @@ -1195,8 +1195,10 @@ status_t OMXCodec::setVideoOutputFormat( compressionFormat = OMX_VIDEO_CodingMPEG4; } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_H263, mime)) { compressionFormat = OMX_VIDEO_CodingH263; - } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_VPX, mime)) { - compressionFormat = OMX_VIDEO_CodingVPX; + } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_VP8, mime)) { + compressionFormat = OMX_VIDEO_CodingVP8; + } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_VP9, mime)) { + compressionFormat = OMX_VIDEO_CodingVP9; } else if (!strcasecmp(MEDIA_MIMETYPE_VIDEO_MPEG2, mime)) { compressionFormat = OMX_VIDEO_CodingMPEG2; } else { @@ -1388,8 +1390,10 @@ void OMXCodec::setComponentRole( "video_decoder.mpeg4", "video_encoder.mpeg4" }, { MEDIA_MIMETYPE_VIDEO_H263, "video_decoder.h263", "video_encoder.h263" }, - { MEDIA_MIMETYPE_VIDEO_VPX, - "video_decoder.vpx", "video_encoder.vpx" }, + { MEDIA_MIMETYPE_VIDEO_VP8, + "video_decoder.vp8", "video_encoder.vp8" }, + { MEDIA_MIMETYPE_VIDEO_VP9, + "video_decoder.vp9", "video_encoder.vp9" }, { MEDIA_MIMETYPE_AUDIO_RAW, "audio_decoder.raw", "audio_encoder.raw" }, { MEDIA_MIMETYPE_AUDIO_FLAC, @@ -4563,7 +4567,7 @@ status_t QueryCodec( CodecCapabilities *caps) { if (strncmp(componentName, "OMX.", 4)) { // Not an OpenMax component but a software codec. - + caps->mFlags = 0; caps->mComponentName = componentName; return OK; } @@ -4578,6 +4582,7 @@ status_t QueryCodec( OMXCodec::setComponentRole(omx, node, isEncoder, mime); + caps->mFlags = 0; caps->mComponentName = componentName; OMX_VIDEO_PARAM_PROFILELEVELTYPE param; @@ -4615,6 +4620,16 @@ status_t QueryCodec( caps->mColorFormats.push(portFormat.eColorFormat); } + if (!isEncoder && !strncmp(mime, "video/", 6)) { + if (omx->storeMetaDataInBuffers( + node, 1 /* port index */, OMX_TRUE) == OK || + omx->prepareForAdaptivePlayback( + node, 1 /* port index */, OMX_TRUE, + 1280 /* width */, 720 /* height */) == OK) { + caps->mFlags |= CodecCapabilities::kFlagSupportsAdaptivePlayback; + } + } + CHECK_EQ(omx->freeNode(node), (status_t)OK); return OK; diff --git a/media/libstagefright/SurfaceMediaSource.cpp b/media/libstagefright/SurfaceMediaSource.cpp index 409038a..6b934d4 100644 --- a/media/libstagefright/SurfaceMediaSource.cpp +++ b/media/libstagefright/SurfaceMediaSource.cpp @@ -21,7 +21,7 @@ #include <media/stagefright/MediaDefs.h> #include <media/stagefright/MetaData.h> #include <OMX_IVCommon.h> -#include <MetadataBufferType.h> +#include <media/hardware/MetadataBufferType.h> #include <ui/GraphicBuffer.h> #include <gui/ISurfaceComposer.h> @@ -54,9 +54,8 @@ SurfaceMediaSource::SurfaceMediaSource(uint32_t bufferWidth, uint32_t bufferHeig ALOGE("Invalid dimensions %dx%d", bufferWidth, bufferHeight); } - mBufferQueue = new BufferQueue(true); + mBufferQueue = new BufferQueue(); mBufferQueue->setDefaultBufferSize(bufferWidth, bufferHeight); - mBufferQueue->setSynchronousMode(true); mBufferQueue->setConsumerUsageBits(GRALLOC_USAGE_HW_VIDEO_ENCODER | GRALLOC_USAGE_HW_TEXTURE); @@ -66,12 +65,10 @@ SurfaceMediaSource::SurfaceMediaSource(uint32_t bufferWidth, uint32_t bufferHeig // reference once the ctor ends, as that would cause the refcount of 'this' // dropping to 0 at the end of the ctor. Since all we need is a wp<...> // that's what we create. - wp<BufferQueue::ConsumerListener> listener; - sp<BufferQueue::ConsumerListener> proxy; - listener = static_cast<BufferQueue::ConsumerListener*>(this); - proxy = new BufferQueue::ProxyConsumerListener(listener); + wp<ConsumerListener> listener = static_cast<ConsumerListener*>(this); + sp<BufferQueue::ProxyConsumerListener> proxy = new BufferQueue::ProxyConsumerListener(listener); - status_t err = mBufferQueue->consumerConnect(proxy); + status_t err = mBufferQueue->consumerConnect(proxy, false); if (err != NO_ERROR) { ALOGE("SurfaceMediaSource: error connecting to BufferQueue: %s (%d)", strerror(-err), err); @@ -108,7 +105,7 @@ void SurfaceMediaSource::dump(String8& result, const char* prefix, Mutex::Autolock lock(mMutex); result.append(buffer); - mBufferQueue->dump(result); + mBufferQueue->dump(result, ""); } status_t SurfaceMediaSource::setFrameRate(int32_t fps) @@ -293,7 +290,7 @@ status_t SurfaceMediaSource::read( MediaBuffer **buffer, // wait here till the frames come in from the client side while (mStarted) { - status_t err = mBufferQueue->acquireBuffer(&item); + status_t err = mBufferQueue->acquireBuffer(&item, 0); if (err == BufferQueue::NO_BUFFER_AVAILABLE) { // wait for a buffer to be queued mFrameAvailableCondition.wait(mMutex); @@ -305,8 +302,9 @@ status_t SurfaceMediaSource::read( MediaBuffer **buffer, // First time seeing the buffer? Added it to the SMS slot if (item.mGraphicBuffer != NULL) { - mBufferSlot[item.mBuf] = item.mGraphicBuffer; + mSlots[item.mBuf].mGraphicBuffer = item.mGraphicBuffer; } + mSlots[item.mBuf].mFrameNumber = item.mFrameNumber; // check for the timing of this buffer if (mNumFramesReceived == 0 && !mUseAbsoluteTimestamps) { @@ -315,7 +313,8 @@ status_t SurfaceMediaSource::read( MediaBuffer **buffer, if (mStartTimeNs > 0) { if (item.mTimestamp < mStartTimeNs) { // This frame predates start of record, discard - mBufferQueue->releaseBuffer(item.mBuf, EGL_NO_DISPLAY, + mBufferQueue->releaseBuffer( + item.mBuf, item.mFrameNumber, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE); continue; } @@ -345,17 +344,18 @@ status_t SurfaceMediaSource::read( MediaBuffer **buffer, // First time seeing the buffer? Added it to the SMS slot if (item.mGraphicBuffer != NULL) { - mBufferSlot[mCurrentSlot] = item.mGraphicBuffer; + mSlots[item.mBuf].mGraphicBuffer = item.mGraphicBuffer; } + mSlots[item.mBuf].mFrameNumber = item.mFrameNumber; - mCurrentBuffers.push_back(mBufferSlot[mCurrentSlot]); + mCurrentBuffers.push_back(mSlots[mCurrentSlot].mGraphicBuffer); int64_t prevTimeStamp = mCurrentTimestamp; mCurrentTimestamp = item.mTimestamp; mNumFramesEncoded++; // Pass the data to the MediaBuffer. Pass in only the metadata - passMetadataBuffer(buffer, mBufferSlot[mCurrentSlot]->handle); + passMetadataBuffer(buffer, mSlots[mCurrentSlot].mGraphicBuffer->handle); (*buffer)->setObserver(this); (*buffer)->add_ref(); @@ -405,15 +405,16 @@ void SurfaceMediaSource::signalBufferReturned(MediaBuffer *buffer) { } for (int id = 0; id < BufferQueue::NUM_BUFFER_SLOTS; id++) { - if (mBufferSlot[id] == NULL) { + if (mSlots[id].mGraphicBuffer == NULL) { continue; } - if (bufferHandle == mBufferSlot[id]->handle) { + if (bufferHandle == mSlots[id].mGraphicBuffer->handle) { ALOGV("Slot %d returned, matches handle = %p", id, - mBufferSlot[id]->handle); + mSlots[id].mGraphicBuffer->handle); - mBufferQueue->releaseBuffer(id, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, + mBufferQueue->releaseBuffer(id, mSlots[id].mFrameNumber, + EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE); buffer->setObserver(0); @@ -469,7 +470,7 @@ void SurfaceMediaSource::onBuffersReleased() { mFrameAvailableCondition.signal(); for (int i = 0; i < BufferQueue::NUM_BUFFER_SLOTS; i++) { - mBufferSlot[i] = 0; + mSlots[i].mGraphicBuffer = 0; } } diff --git a/media/libstagefright/TimedEventQueue.cpp b/media/libstagefright/TimedEventQueue.cpp index 7e9c4bf..6a16bb4 100644 --- a/media/libstagefright/TimedEventQueue.cpp +++ b/media/libstagefright/TimedEventQueue.cpp @@ -31,17 +31,26 @@ #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/ALooper.h> +#include <binder/IServiceManager.h> +#include <powermanager/PowerManager.h> +#include <binder/IPCThreadState.h> +#include <utils/CallStack.h> namespace android { TimedEventQueue::TimedEventQueue() : mNextEventID(1), mRunning(false), - mStopped(false) { + mStopped(false), + mDeathRecipient(new PMDeathRecipient(this)) { } TimedEventQueue::~TimedEventQueue() { stop(); + if (mPowerManager != 0) { + sp<IBinder> binder = mPowerManager->asBinder(); + binder->unlinkToDeath(mDeathRecipient); + } } void TimedEventQueue::start() { @@ -76,6 +85,11 @@ void TimedEventQueue::stop(bool flush) { void *dummy; pthread_join(mThread, &dummy); + // some events may be left in the queue if we did not flush and the wake lock + // must be released. + if (!mQueue.empty()) { + releaseWakeLock_l(); + } mQueue.clear(); mRunning = false; @@ -117,6 +131,9 @@ TimedEventQueue::event_id TimedEventQueue::postTimedEvent( mQueueHeadChangedCondition.signal(); } + if (mQueue.empty()) { + acquireWakeLock_l(); + } mQueue.insert(it, item); mQueueNotEmptyCondition.signal(); @@ -172,7 +189,9 @@ void TimedEventQueue::cancelEvents( (*it).event->setEventID(0); it = mQueue.erase(it); - + if (mQueue.empty()) { + releaseWakeLock_l(); + } if (stopAfterFirstMatch) { return; } @@ -280,7 +299,9 @@ sp<TimedEventQueue::Event> TimedEventQueue::removeEventFromQueue_l( event->setEventID(0); mQueue.erase(it); - + if (mQueue.empty()) { + releaseWakeLock_l(); + } return event; } } @@ -290,5 +311,60 @@ sp<TimedEventQueue::Event> TimedEventQueue::removeEventFromQueue_l( return NULL; } +void TimedEventQueue::acquireWakeLock_l() +{ + if (mWakeLockToken != 0) { + return; + } + if (mPowerManager == 0) { + // use checkService() to avoid blocking if power service is not up yet + sp<IBinder> binder = + defaultServiceManager()->checkService(String16("power")); + if (binder == 0) { + ALOGW("cannot connect to the power manager service"); + } else { + mPowerManager = interface_cast<IPowerManager>(binder); + binder->linkToDeath(mDeathRecipient); + } + } + if (mPowerManager != 0) { + sp<IBinder> binder = new BBinder(); + int64_t token = IPCThreadState::self()->clearCallingIdentity(); + status_t status = mPowerManager->acquireWakeLock(POWERMANAGER_PARTIAL_WAKE_LOCK, + binder, + String16("TimedEventQueue"), + String16("media")); + IPCThreadState::self()->restoreCallingIdentity(token); + if (status == NO_ERROR) { + mWakeLockToken = binder; + } + } +} + +void TimedEventQueue::releaseWakeLock_l() +{ + if (mWakeLockToken == 0) { + return; + } + if (mPowerManager != 0) { + int64_t token = IPCThreadState::self()->clearCallingIdentity(); + mPowerManager->releaseWakeLock(mWakeLockToken, 0); + IPCThreadState::self()->restoreCallingIdentity(token); + } + mWakeLockToken.clear(); +} + +void TimedEventQueue::clearPowerManager() +{ + Mutex::Autolock _l(mLock); + releaseWakeLock_l(); + mPowerManager.clear(); +} + +void TimedEventQueue::PMDeathRecipient::binderDied(const wp<IBinder>& who) +{ + mQueue->clearPowerManager(); +} + } // namespace android diff --git a/media/libstagefright/Utils.cpp b/media/libstagefright/Utils.cpp index b0df379..4db8e80 100644 --- a/media/libstagefright/Utils.cpp +++ b/media/libstagefright/Utils.cpp @@ -26,7 +26,12 @@ #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> #include <media/stagefright/MetaData.h> +#include <media/stagefright/MediaDefs.h> +#include <media/AudioSystem.h> +#include <media/MediaPlayerInterface.h> +#include <hardware/audio.h> #include <media/stagefright/Utils.h> +#include <media/AudioParameter.h> namespace android { @@ -471,5 +476,132 @@ AString MakeUserAgent() { return ua; } +status_t sendMetaDataToHal(sp<MediaPlayerBase::AudioSink>& sink, + const sp<MetaData>& meta) +{ + int32_t sampleRate = 0; + int32_t bitRate = 0; + int32_t channelMask = 0; + int32_t delaySamples = 0; + int32_t paddingSamples = 0; + + AudioParameter param = AudioParameter(); + + if (meta->findInt32(kKeySampleRate, &sampleRate)) { + param.addInt(String8(AUDIO_OFFLOAD_CODEC_SAMPLE_RATE), sampleRate); + } + if (meta->findInt32(kKeyChannelMask, &channelMask)) { + param.addInt(String8(AUDIO_OFFLOAD_CODEC_NUM_CHANNEL), channelMask); + } + if (meta->findInt32(kKeyBitRate, &bitRate)) { + param.addInt(String8(AUDIO_OFFLOAD_CODEC_AVG_BIT_RATE), bitRate); + } + if (meta->findInt32(kKeyEncoderDelay, &delaySamples)) { + param.addInt(String8(AUDIO_OFFLOAD_CODEC_DELAY_SAMPLES), delaySamples); + } + if (meta->findInt32(kKeyEncoderPadding, &paddingSamples)) { + param.addInt(String8(AUDIO_OFFLOAD_CODEC_PADDING_SAMPLES), paddingSamples); + } + + ALOGV("sendMetaDataToHal: bitRate %d, sampleRate %d, chanMask %d," + "delaySample %d, paddingSample %d", bitRate, sampleRate, + channelMask, delaySamples, paddingSamples); + + sink->setParameters(param.toString()); + return OK; +} + +struct mime_conv_t { + const char* mime; + audio_format_t format; +}; + +static const struct mime_conv_t mimeLookup[] = { + { MEDIA_MIMETYPE_AUDIO_MPEG, AUDIO_FORMAT_MP3 }, + { MEDIA_MIMETYPE_AUDIO_RAW, AUDIO_FORMAT_PCM_16_BIT }, + { MEDIA_MIMETYPE_AUDIO_AMR_NB, AUDIO_FORMAT_AMR_NB }, + { MEDIA_MIMETYPE_AUDIO_AMR_WB, AUDIO_FORMAT_AMR_WB }, + { MEDIA_MIMETYPE_AUDIO_AAC, AUDIO_FORMAT_AAC }, + { MEDIA_MIMETYPE_AUDIO_VORBIS, AUDIO_FORMAT_VORBIS }, + { 0, AUDIO_FORMAT_INVALID } +}; + +status_t mapMimeToAudioFormat( audio_format_t& format, const char* mime ) +{ +const struct mime_conv_t* p = &mimeLookup[0]; + while (p->mime != NULL) { + if (0 == strcasecmp(mime, p->mime)) { + format = p->format; + return OK; + } + ++p; + } + + return BAD_VALUE; +} + +bool canOffloadStream(const sp<MetaData>& meta, bool hasVideo, bool isStreaming) +{ + const char *mime; + CHECK(meta->findCString(kKeyMIMEType, &mime)); + + audio_offload_info_t info = AUDIO_INFO_INITIALIZER; + + info.format = AUDIO_FORMAT_INVALID; + if (mapMimeToAudioFormat(info.format, mime) != OK) { + ALOGE(" Couldn't map mime type \"%s\" to a valid AudioSystem::audio_format !", mime); + return false; + } else { + ALOGV("Mime type \"%s\" mapped to audio_format %d", mime, info.format); + } + + if (AUDIO_FORMAT_INVALID == info.format) { + // can't offload if we don't know what the source format is + ALOGE("mime type \"%s\" not a known audio format", mime); + return false; + } + + int32_t srate = -1; + if (!meta->findInt32(kKeySampleRate, &srate)) { + ALOGV("track of type '%s' does not publish sample rate", mime); + } + info.sample_rate = srate; + + int32_t cmask = 0; + if (!meta->findInt32(kKeyChannelMask, &cmask)) { + ALOGV("track of type '%s' does not publish channel mask", mime); + + // Try a channel count instead + int32_t channelCount; + if (!meta->findInt32(kKeyChannelCount, &channelCount)) { + ALOGV("track of type '%s' does not publish channel count", mime); + } else { + cmask = audio_channel_out_mask_from_count(channelCount); + } + } + info.channel_mask = cmask; + + int64_t duration = 0; + if (!meta->findInt64(kKeyDuration, &duration)) { + ALOGV("track of type '%s' does not publish duration", mime); + } + info.duration_us = duration; + + int32_t brate = -1; + if (!meta->findInt32(kKeyBitRate, &brate)) { + ALOGV("track of type '%s' does not publish bitrate", mime); + } + info.bit_rate = brate; + + + info.stream_type = AUDIO_STREAM_MUSIC; + info.has_video = hasVideo; + info.is_streaming = isStreaming; + + // Check if offload is possible for given format, stream type, sample rate, + // bit rate, duration, video and streaming + return AudioSystem::isOffloadSupported(info); +} + } // namespace android diff --git a/media/libstagefright/codecs/aacdec/SoftAAC2.cpp b/media/libstagefright/codecs/aacdec/SoftAAC2.cpp index cf50dc9..1b20cbb 100644 --- a/media/libstagefright/codecs/aacdec/SoftAAC2.cpp +++ b/media/libstagefright/codecs/aacdec/SoftAAC2.cpp @@ -604,6 +604,9 @@ void SoftAAC2::onReset() { // To make the codec behave the same before and after a reset, we need to invalidate the // streaminfo struct. This does that: mStreamInfo->sampleRate = 0; + + mSignalledError = false; + mOutputPortSettingsChange = NONE; } void SoftAAC2::onPortEnableCompleted(OMX_U32 portIndex, bool enabled) { diff --git a/media/libstagefright/codecs/aacenc/SampleCode/Android.mk b/media/libstagefright/codecs/aacenc/SampleCode/Android.mk index 01016e7..d06dcf6 100644 --- a/media/libstagefright/codecs/aacenc/SampleCode/Android.mk +++ b/media/libstagefright/codecs/aacenc/SampleCode/Android.mk @@ -5,7 +5,7 @@ LOCAL_SRC_FILES := \ AAC_E_SAMPLES.c \ ../../common/cmnMemory.c -LOCAL_MODULE_TAGS := debug +LOCAL_MODULE_TAGS := optional LOCAL_MODULE := AACEncTest diff --git a/media/libstagefright/codecs/aacenc/SoftAACEncoder2.cpp b/media/libstagefright/codecs/aacenc/SoftAACEncoder2.cpp index 5749733..ff2b503 100644 --- a/media/libstagefright/codecs/aacenc/SoftAACEncoder2.cpp +++ b/media/libstagefright/codecs/aacenc/SoftAACEncoder2.cpp @@ -292,6 +292,10 @@ static AUDIO_OBJECT_TYPE getAOTFromProfile(OMX_U32 profile) { return AOT_AAC_LC; } else if (profile == OMX_AUDIO_AACObjectHE) { return AOT_SBR; + } else if (profile == OMX_AUDIO_AACObjectHE_PS) { + return AOT_PS; + } else if (profile == OMX_AUDIO_AACObjectLD) { + return AOT_ER_AAC_LD; } else if (profile == OMX_AUDIO_AACObjectELD) { return AOT_ER_AAC_ELD; } else { diff --git a/media/libstagefright/codecs/amrnb/dec/SoftAMR.cpp b/media/libstagefright/codecs/amrnb/dec/SoftAMR.cpp index 4d4212f..3320688 100644 --- a/media/libstagefright/codecs/amrnb/dec/SoftAMR.cpp +++ b/media/libstagefright/codecs/amrnb/dec/SoftAMR.cpp @@ -457,6 +457,11 @@ void SoftAMR::onPortEnableCompleted(OMX_U32 portIndex, bool enabled) { } } +void SoftAMR::onReset() { + mSignalledError = false; + mOutputPortSettingsChange = NONE; +} + } // namespace android android::SoftOMXComponent *createSoftOMXComponent( diff --git a/media/libstagefright/codecs/amrnb/dec/SoftAMR.h b/media/libstagefright/codecs/amrnb/dec/SoftAMR.h index 9a596e5..758d6ac 100644 --- a/media/libstagefright/codecs/amrnb/dec/SoftAMR.h +++ b/media/libstagefright/codecs/amrnb/dec/SoftAMR.h @@ -40,6 +40,7 @@ protected: virtual void onQueueFilled(OMX_U32 portIndex); virtual void onPortFlushCompleted(OMX_U32 portIndex); virtual void onPortEnableCompleted(OMX_U32 portIndex, bool enabled); + virtual void onReset(); private: enum { diff --git a/media/libstagefright/codecs/amrwbenc/SampleCode/Android.mk b/media/libstagefright/codecs/amrwbenc/SampleCode/Android.mk index db34d08..c203f77 100644 --- a/media/libstagefright/codecs/amrwbenc/SampleCode/Android.mk +++ b/media/libstagefright/codecs/amrwbenc/SampleCode/Android.mk @@ -5,7 +5,7 @@ LOCAL_SRC_FILES := \ AMRWB_E_SAMPLE.c \ ../../common/cmnMemory.c -LOCAL_MODULE_TAGS := debug +LOCAL_MODULE_TAGS := optional LOCAL_MODULE := AMRWBEncTest LOCAL_ARM_MODE := arm diff --git a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp index 020cc0a..fb2a430 100644 --- a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp +++ b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp @@ -48,42 +48,32 @@ static const CodecProfileLevel kH263ProfileLevels[] = { { OMX_VIDEO_H263ProfileISWV2, OMX_VIDEO_H263Level45 }, }; -template<class T> -static void InitOMXParams(T *params) { - params->nSize = sizeof(T); - params->nVersion.s.nVersionMajor = 1; - params->nVersion.s.nVersionMinor = 0; - params->nVersion.s.nRevision = 0; - params->nVersion.s.nStep = 0; -} - SoftMPEG4::SoftMPEG4( const char *name, + const char *componentRole, + OMX_VIDEO_CODINGTYPE codingType, + const CodecProfileLevel *profileLevels, + size_t numProfileLevels, const OMX_CALLBACKTYPE *callbacks, OMX_PTR appData, OMX_COMPONENTTYPE **component) - : SimpleSoftOMXComponent(name, callbacks, appData, component), - mMode(MODE_MPEG4), + : SoftVideoDecoderOMXComponent( + name, componentRole, codingType, profileLevels, numProfileLevels, + 352 /* width */, 288 /* height */, callbacks, appData, component), + mMode(codingType == OMX_VIDEO_CodingH263 ? MODE_H263 : MODE_MPEG4), mHandle(new tagvideoDecControls), mInputBufferCount(0), - mWidth(352), - mHeight(288), - mCropLeft(0), - mCropTop(0), - mCropRight(mWidth - 1), - mCropBottom(mHeight - 1), mSignalledError(false), mInitialized(false), mFramesConfigured(false), mNumSamplesOutput(0), - mOutputPortSettingsChange(NONE) { - if (!strcmp(name, "OMX.google.h263.decoder")) { - mMode = MODE_H263; - } else { - CHECK(!strcmp(name, "OMX.google.mpeg4.decoder")); - } - - initPorts(); + mPvTime(0) { + initPorts( + kNumInputBuffers, + 8192 /* inputBufferSize */, + kNumOutputBuffers, + (mMode == MODE_MPEG4) + ? MEDIA_MIMETYPE_VIDEO_MPEG4 : MEDIA_MIMETYPE_VIDEO_H263); CHECK_EQ(initDecoder(), (status_t)OK); } @@ -96,219 +86,11 @@ SoftMPEG4::~SoftMPEG4() { mHandle = NULL; } -void SoftMPEG4::initPorts() { - OMX_PARAM_PORTDEFINITIONTYPE def; - InitOMXParams(&def); - - def.nPortIndex = 0; - def.eDir = OMX_DirInput; - def.nBufferCountMin = kNumInputBuffers; - def.nBufferCountActual = def.nBufferCountMin; - def.nBufferSize = 8192; - def.bEnabled = OMX_TRUE; - def.bPopulated = OMX_FALSE; - def.eDomain = OMX_PortDomainVideo; - def.bBuffersContiguous = OMX_FALSE; - def.nBufferAlignment = 1; - - def.format.video.cMIMEType = - (mMode == MODE_MPEG4) - ? const_cast<char *>(MEDIA_MIMETYPE_VIDEO_MPEG4) - : const_cast<char *>(MEDIA_MIMETYPE_VIDEO_H263); - - def.format.video.pNativeRender = NULL; - def.format.video.nFrameWidth = mWidth; - def.format.video.nFrameHeight = mHeight; - def.format.video.nStride = def.format.video.nFrameWidth; - def.format.video.nSliceHeight = def.format.video.nFrameHeight; - def.format.video.nBitrate = 0; - def.format.video.xFramerate = 0; - def.format.video.bFlagErrorConcealment = OMX_FALSE; - - def.format.video.eCompressionFormat = - mMode == MODE_MPEG4 ? OMX_VIDEO_CodingMPEG4 : OMX_VIDEO_CodingH263; - - def.format.video.eColorFormat = OMX_COLOR_FormatUnused; - def.format.video.pNativeWindow = NULL; - - addPort(def); - - def.nPortIndex = 1; - def.eDir = OMX_DirOutput; - def.nBufferCountMin = kNumOutputBuffers; - def.nBufferCountActual = def.nBufferCountMin; - def.bEnabled = OMX_TRUE; - def.bPopulated = OMX_FALSE; - def.eDomain = OMX_PortDomainVideo; - def.bBuffersContiguous = OMX_FALSE; - def.nBufferAlignment = 2; - - def.format.video.cMIMEType = const_cast<char *>(MEDIA_MIMETYPE_VIDEO_RAW); - def.format.video.pNativeRender = NULL; - def.format.video.nFrameWidth = mWidth; - def.format.video.nFrameHeight = mHeight; - def.format.video.nStride = def.format.video.nFrameWidth; - def.format.video.nSliceHeight = def.format.video.nFrameHeight; - def.format.video.nBitrate = 0; - def.format.video.xFramerate = 0; - def.format.video.bFlagErrorConcealment = OMX_FALSE; - def.format.video.eCompressionFormat = OMX_VIDEO_CodingUnused; - def.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar; - def.format.video.pNativeWindow = NULL; - - def.nBufferSize = - (def.format.video.nFrameWidth * def.format.video.nFrameHeight * 3) / 2; - - addPort(def); -} - status_t SoftMPEG4::initDecoder() { memset(mHandle, 0, sizeof(tagvideoDecControls)); return OK; } -OMX_ERRORTYPE SoftMPEG4::internalGetParameter( - OMX_INDEXTYPE index, OMX_PTR params) { - switch (index) { - case OMX_IndexParamVideoPortFormat: - { - OMX_VIDEO_PARAM_PORTFORMATTYPE *formatParams = - (OMX_VIDEO_PARAM_PORTFORMATTYPE *)params; - - if (formatParams->nPortIndex > 1) { - return OMX_ErrorUndefined; - } - - if (formatParams->nIndex != 0) { - return OMX_ErrorNoMore; - } - - if (formatParams->nPortIndex == 0) { - formatParams->eCompressionFormat = - (mMode == MODE_MPEG4) - ? OMX_VIDEO_CodingMPEG4 : OMX_VIDEO_CodingH263; - - formatParams->eColorFormat = OMX_COLOR_FormatUnused; - formatParams->xFramerate = 0; - } else { - CHECK_EQ(formatParams->nPortIndex, 1u); - - formatParams->eCompressionFormat = OMX_VIDEO_CodingUnused; - formatParams->eColorFormat = OMX_COLOR_FormatYUV420Planar; - formatParams->xFramerate = 0; - } - - return OMX_ErrorNone; - } - - case OMX_IndexParamVideoProfileLevelQuerySupported: - { - OMX_VIDEO_PARAM_PROFILELEVELTYPE *profileLevel = - (OMX_VIDEO_PARAM_PROFILELEVELTYPE *) params; - - if (profileLevel->nPortIndex != 0) { // Input port only - ALOGE("Invalid port index: %ld", profileLevel->nPortIndex); - return OMX_ErrorUnsupportedIndex; - } - - size_t index = profileLevel->nProfileIndex; - if (mMode == MODE_H263) { - size_t nProfileLevels = - sizeof(kH263ProfileLevels) / sizeof(kH263ProfileLevels[0]); - if (index >= nProfileLevels) { - return OMX_ErrorNoMore; - } - - profileLevel->eProfile = kH263ProfileLevels[index].mProfile; - profileLevel->eLevel = kH263ProfileLevels[index].mLevel; - } else { - size_t nProfileLevels = - sizeof(kM4VProfileLevels) / sizeof(kM4VProfileLevels[0]); - if (index >= nProfileLevels) { - return OMX_ErrorNoMore; - } - - profileLevel->eProfile = kM4VProfileLevels[index].mProfile; - profileLevel->eLevel = kM4VProfileLevels[index].mLevel; - } - return OMX_ErrorNone; - } - - default: - return SimpleSoftOMXComponent::internalGetParameter(index, params); - } -} - -OMX_ERRORTYPE SoftMPEG4::internalSetParameter( - OMX_INDEXTYPE index, const OMX_PTR params) { - switch (index) { - case OMX_IndexParamStandardComponentRole: - { - const OMX_PARAM_COMPONENTROLETYPE *roleParams = - (const OMX_PARAM_COMPONENTROLETYPE *)params; - - if (mMode == MODE_MPEG4) { - if (strncmp((const char *)roleParams->cRole, - "video_decoder.mpeg4", - OMX_MAX_STRINGNAME_SIZE - 1)) { - return OMX_ErrorUndefined; - } - } else { - if (strncmp((const char *)roleParams->cRole, - "video_decoder.h263", - OMX_MAX_STRINGNAME_SIZE - 1)) { - return OMX_ErrorUndefined; - } - } - - return OMX_ErrorNone; - } - - case OMX_IndexParamVideoPortFormat: - { - OMX_VIDEO_PARAM_PORTFORMATTYPE *formatParams = - (OMX_VIDEO_PARAM_PORTFORMATTYPE *)params; - - if (formatParams->nPortIndex > 1) { - return OMX_ErrorUndefined; - } - - if (formatParams->nIndex != 0) { - return OMX_ErrorNoMore; - } - - return OMX_ErrorNone; - } - - default: - return SimpleSoftOMXComponent::internalSetParameter(index, params); - } -} - -OMX_ERRORTYPE SoftMPEG4::getConfig( - OMX_INDEXTYPE index, OMX_PTR params) { - switch (index) { - case OMX_IndexConfigCommonOutputCrop: - { - OMX_CONFIG_RECTTYPE *rectParams = (OMX_CONFIG_RECTTYPE *)params; - - if (rectParams->nPortIndex != 1) { - return OMX_ErrorUndefined; - } - - rectParams->nLeft = mCropLeft; - rectParams->nTop = mCropTop; - rectParams->nWidth = mCropRight - mCropLeft + 1; - rectParams->nHeight = mCropBottom - mCropTop + 1; - - return OMX_ErrorNone; - } - - default: - return OMX_ErrorUnsupportedIndex; - } -} - void SoftMPEG4::onQueueFilled(OMX_U32 portIndex) { if (mSignalledError || mOutputPortSettingsChange != NONE) { return; @@ -415,9 +197,14 @@ void SoftMPEG4::onQueueFilled(OMX_U32 portIndex) { uint32_t useExtTimestamp = (inHeader->nOffset == 0); - // decoder deals in ms, OMX in us. - uint32_t timestamp = - useExtTimestamp ? (inHeader->nTimeStamp + 500) / 1000 : 0xFFFFFFFF; + // decoder deals in ms (int32_t), OMX in us (int64_t) + // so use fake timestamp instead + uint32_t timestamp = 0xFFFFFFFF; + if (useExtTimestamp) { + mPvToOmxTimeMap.add(mPvTime, inHeader->nTimeStamp); + timestamp = mPvTime; + mPvTime++; + } int32_t bufferSize = inHeader->nFilledLen; int32_t tmp = bufferSize; @@ -441,7 +228,8 @@ void SoftMPEG4::onQueueFilled(OMX_U32 portIndex) { } // decoder deals in ms, OMX in us. - outHeader->nTimeStamp = timestamp * 1000; + outHeader->nTimeStamp = mPvToOmxTimeMap.valueFor(timestamp); + mPvToOmxTimeMap.removeItem(timestamp); inHeader->nOffset += bufferSize; inHeader->nFilledLen = 0; @@ -482,11 +270,11 @@ void SoftMPEG4::onQueueFilled(OMX_U32 portIndex) { } bool SoftMPEG4::portSettingsChanged() { - int32_t disp_width, disp_height; - PVGetVideoDimensions(mHandle, &disp_width, &disp_height); + uint32_t disp_width, disp_height; + PVGetVideoDimensions(mHandle, (int32 *)&disp_width, (int32 *)&disp_height); - int32_t buf_width, buf_height; - PVGetBufferDimensions(mHandle, &buf_width, &buf_height); + uint32_t buf_width, buf_height; + PVGetBufferDimensions(mHandle, (int32 *)&buf_width, (int32 *)&buf_height); CHECK_LE(disp_width, buf_width); CHECK_LE(disp_height, buf_height); @@ -494,12 +282,12 @@ bool SoftMPEG4::portSettingsChanged() { ALOGV("disp_width = %d, disp_height = %d, buf_width = %d, buf_height = %d", disp_width, disp_height, buf_width, buf_height); - if (mCropRight != disp_width - 1 - || mCropBottom != disp_height - 1) { + if (mCropWidth != disp_width + || mCropHeight != disp_height) { mCropLeft = 0; mCropTop = 0; - mCropRight = disp_width - 1; - mCropBottom = disp_height - 1; + mCropWidth = disp_width; + mCropHeight = disp_height; notify(OMX_EventPortSettingsChanged, 1, @@ -545,45 +333,22 @@ void SoftMPEG4::onPortFlushCompleted(OMX_U32 portIndex) { } } -void SoftMPEG4::onPortEnableCompleted(OMX_U32 portIndex, bool enabled) { - if (portIndex != 1) { - return; - } - - switch (mOutputPortSettingsChange) { - case NONE: - break; - - case AWAITING_DISABLED: - { - CHECK(!enabled); - mOutputPortSettingsChange = AWAITING_ENABLED; - break; - } - - default: - { - CHECK_EQ((int)mOutputPortSettingsChange, (int)AWAITING_ENABLED); - CHECK(enabled); - mOutputPortSettingsChange = NONE; - break; - } +void SoftMPEG4::onReset() { + SoftVideoDecoderOMXComponent::onReset(); + mPvToOmxTimeMap.clear(); + mSignalledError = false; + mFramesConfigured = false; + if (mInitialized) { + PVCleanUpVideoDecoder(mHandle); + mInitialized = false; } } void SoftMPEG4::updatePortDefinitions() { - OMX_PARAM_PORTDEFINITIONTYPE *def = &editPortInfo(0)->mDef; - def->format.video.nFrameWidth = mWidth; - def->format.video.nFrameHeight = mHeight; - def->format.video.nStride = def->format.video.nFrameWidth; - def->format.video.nSliceHeight = def->format.video.nFrameHeight; - - def = &editPortInfo(1)->mDef; - def->format.video.nFrameWidth = mWidth; - def->format.video.nFrameHeight = mHeight; - def->format.video.nStride = def->format.video.nFrameWidth; - def->format.video.nSliceHeight = def->format.video.nFrameHeight; + SoftVideoDecoderOMXComponent::updatePortDefinitions(); + /* We have to align our width and height - this should affect stride! */ + OMX_PARAM_PORTDEFINITIONTYPE *def = &editPortInfo(kOutputPortIndex)->mDef; def->nBufferSize = (((def->format.video.nFrameWidth + 15) & -16) * ((def->format.video.nFrameHeight + 15) & -16) * 3) / 2; @@ -594,6 +359,19 @@ void SoftMPEG4::updatePortDefinitions() { android::SoftOMXComponent *createSoftOMXComponent( const char *name, const OMX_CALLBACKTYPE *callbacks, OMX_PTR appData, OMX_COMPONENTTYPE **component) { - return new android::SoftMPEG4(name, callbacks, appData, component); + using namespace android; + if (!strcmp(name, "OMX.google.h263.decoder")) { + return new android::SoftMPEG4( + name, "video_decoder.h263", OMX_VIDEO_CodingH263, + kH263ProfileLevels, ARRAY_SIZE(kH263ProfileLevels), + callbacks, appData, component); + } else if (!strcmp(name, "OMX.google.mpeg4.decoder")) { + return new android::SoftMPEG4( + name, "video_decoder.mpeg4", OMX_VIDEO_CodingMPEG4, + kM4VProfileLevels, ARRAY_SIZE(kM4VProfileLevels), + callbacks, appData, component); + } else { + CHECK(!"Unknown component"); + } } diff --git a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.h b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.h index dff08a7..de14aaf 100644 --- a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.h +++ b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.h @@ -18,14 +18,18 @@ #define SOFT_MPEG4_H_ -#include "SimpleSoftOMXComponent.h" +#include "SoftVideoDecoderOMXComponent.h" struct tagvideoDecControls; namespace android { -struct SoftMPEG4 : public SimpleSoftOMXComponent { +struct SoftMPEG4 : public SoftVideoDecoderOMXComponent { SoftMPEG4(const char *name, + const char *componentRole, + OMX_VIDEO_CODINGTYPE codingType, + const CodecProfileLevel *profileLevels, + size_t numProfileLevels, const OMX_CALLBACKTYPE *callbacks, OMX_PTR appData, OMX_COMPONENTTYPE **component); @@ -33,17 +37,9 @@ struct SoftMPEG4 : public SimpleSoftOMXComponent { protected: virtual ~SoftMPEG4(); - virtual OMX_ERRORTYPE internalGetParameter( - OMX_INDEXTYPE index, OMX_PTR params); - - virtual OMX_ERRORTYPE internalSetParameter( - OMX_INDEXTYPE index, const OMX_PTR params); - - virtual OMX_ERRORTYPE getConfig(OMX_INDEXTYPE index, OMX_PTR params); - virtual void onQueueFilled(OMX_U32 portIndex); virtual void onPortFlushCompleted(OMX_U32 portIndex); - virtual void onPortEnableCompleted(OMX_U32 portIndex, bool enabled); + virtual void onReset(); private: enum { @@ -54,32 +50,23 @@ private: enum { MODE_MPEG4, MODE_H263, - } mMode; tagvideoDecControls *mHandle; size_t mInputBufferCount; - int32_t mWidth, mHeight; - int32_t mCropLeft, mCropTop, mCropRight, mCropBottom; - bool mSignalledError; bool mInitialized; bool mFramesConfigured; int32_t mNumSamplesOutput; + int32_t mPvTime; + KeyedVector<int32_t, OMX_TICKS> mPvToOmxTimeMap; - enum { - NONE, - AWAITING_DISABLED, - AWAITING_ENABLED - } mOutputPortSettingsChange; - - void initPorts(); status_t initDecoder(); - void updatePortDefinitions(); + virtual void updatePortDefinitions(); bool portSettingsChanged(); DISALLOW_EVIL_CONSTRUCTORS(SoftMPEG4); diff --git a/media/libstagefright/codecs/mp3dec/SoftMP3.cpp b/media/libstagefright/codecs/mp3dec/SoftMP3.cpp index 9f25536..7c382fb 100644 --- a/media/libstagefright/codecs/mp3dec/SoftMP3.cpp +++ b/media/libstagefright/codecs/mp3dec/SoftMP3.cpp @@ -361,6 +361,8 @@ void SoftMP3::onPortEnableCompleted(OMX_U32 portIndex, bool enabled) { void SoftMP3::onReset() { pvmp3_InitDecoder(mConfig, mDecoderBuf); mIsFirst = true; + mSignalledError = false; + mOutputPortSettingsChange = NONE; } } // namespace android diff --git a/media/libstagefright/codecs/on2/dec/SoftVPX.cpp b/media/libstagefright/codecs/on2/dec/SoftVPX.cpp index a400b4c..476e986 100644 --- a/media/libstagefright/codecs/on2/dec/SoftVPX.cpp +++ b/media/libstagefright/codecs/on2/dec/SoftVPX.cpp @@ -29,26 +29,23 @@ namespace android { -template<class T> -static void InitOMXParams(T *params) { - params->nSize = sizeof(T); - params->nVersion.s.nVersionMajor = 1; - params->nVersion.s.nVersionMinor = 0; - params->nVersion.s.nRevision = 0; - params->nVersion.s.nStep = 0; -} - SoftVPX::SoftVPX( const char *name, + const char *componentRole, + OMX_VIDEO_CODINGTYPE codingType, const OMX_CALLBACKTYPE *callbacks, OMX_PTR appData, OMX_COMPONENTTYPE **component) - : SimpleSoftOMXComponent(name, callbacks, appData, component), - mCtx(NULL), - mWidth(320), - mHeight(240), - mOutputPortSettingsChange(NONE) { - initPorts(); + : SoftVideoDecoderOMXComponent( + name, componentRole, codingType, + NULL /* profileLevels */, 0 /* numProfileLevels */, + 320 /* width */, 240 /* height */, callbacks, appData, component), + mMode(codingType == OMX_VIDEO_CodingVP8 ? MODE_VP8 : MODE_VP9), + mCtx(NULL) { + initPorts(kNumBuffers, 768 * 1024 /* inputBufferSize */, + kNumBuffers, + codingType == OMX_VIDEO_CodingVP8 ? MEDIA_MIMETYPE_VIDEO_VP8 : MEDIA_MIMETYPE_VIDEO_VP9); + CHECK_EQ(initDecoder(), (status_t)OK); } @@ -58,65 +55,6 @@ SoftVPX::~SoftVPX() { mCtx = NULL; } -void SoftVPX::initPorts() { - OMX_PARAM_PORTDEFINITIONTYPE def; - InitOMXParams(&def); - - def.nPortIndex = 0; - def.eDir = OMX_DirInput; - def.nBufferCountMin = kNumBuffers; - def.nBufferCountActual = def.nBufferCountMin; - def.nBufferSize = 768 * 1024; - def.bEnabled = OMX_TRUE; - def.bPopulated = OMX_FALSE; - def.eDomain = OMX_PortDomainVideo; - def.bBuffersContiguous = OMX_FALSE; - def.nBufferAlignment = 1; - - def.format.video.cMIMEType = const_cast<char *>(MEDIA_MIMETYPE_VIDEO_VPX); - def.format.video.pNativeRender = NULL; - def.format.video.nFrameWidth = mWidth; - def.format.video.nFrameHeight = mHeight; - def.format.video.nStride = def.format.video.nFrameWidth; - def.format.video.nSliceHeight = def.format.video.nFrameHeight; - def.format.video.nBitrate = 0; - def.format.video.xFramerate = 0; - def.format.video.bFlagErrorConcealment = OMX_FALSE; - def.format.video.eCompressionFormat = OMX_VIDEO_CodingVPX; - def.format.video.eColorFormat = OMX_COLOR_FormatUnused; - def.format.video.pNativeWindow = NULL; - - addPort(def); - - def.nPortIndex = 1; - def.eDir = OMX_DirOutput; - def.nBufferCountMin = kNumBuffers; - def.nBufferCountActual = def.nBufferCountMin; - def.bEnabled = OMX_TRUE; - def.bPopulated = OMX_FALSE; - def.eDomain = OMX_PortDomainVideo; - def.bBuffersContiguous = OMX_FALSE; - def.nBufferAlignment = 2; - - def.format.video.cMIMEType = const_cast<char *>(MEDIA_MIMETYPE_VIDEO_RAW); - def.format.video.pNativeRender = NULL; - def.format.video.nFrameWidth = mWidth; - def.format.video.nFrameHeight = mHeight; - def.format.video.nStride = def.format.video.nFrameWidth; - def.format.video.nSliceHeight = def.format.video.nFrameHeight; - def.format.video.nBitrate = 0; - def.format.video.xFramerate = 0; - def.format.video.bFlagErrorConcealment = OMX_FALSE; - def.format.video.eCompressionFormat = OMX_VIDEO_CodingUnused; - def.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar; - def.format.video.pNativeWindow = NULL; - - def.nBufferSize = - (def.format.video.nFrameWidth * def.format.video.nFrameHeight * 3) / 2; - - addPort(def); -} - static int GetCPUCoreCount() { int cpuCoreCount = 1; #if defined(_SC_NPROCESSORS_ONLN) @@ -137,7 +75,9 @@ status_t SoftVPX::initDecoder() { memset(&cfg, 0, sizeof(vpx_codec_dec_cfg_t)); cfg.threads = GetCPUCoreCount(); if ((vpx_err = vpx_codec_dec_init( - (vpx_codec_ctx_t *)mCtx, &vpx_codec_vp8_dx_algo, &cfg, 0))) { + (vpx_codec_ctx_t *)mCtx, + mMode == MODE_VP8 ? &vpx_codec_vp8_dx_algo : &vpx_codec_vp9_dx_algo, + &cfg, 0))) { ALOGE("on2 decoder failed to initialize. (%d)", vpx_err); return UNKNOWN_ERROR; } @@ -145,80 +85,6 @@ status_t SoftVPX::initDecoder() { return OK; } -OMX_ERRORTYPE SoftVPX::internalGetParameter( - OMX_INDEXTYPE index, OMX_PTR params) { - switch (index) { - case OMX_IndexParamVideoPortFormat: - { - OMX_VIDEO_PARAM_PORTFORMATTYPE *formatParams = - (OMX_VIDEO_PARAM_PORTFORMATTYPE *)params; - - if (formatParams->nPortIndex > 1) { - return OMX_ErrorUndefined; - } - - if (formatParams->nIndex != 0) { - return OMX_ErrorNoMore; - } - - if (formatParams->nPortIndex == 0) { - formatParams->eCompressionFormat = OMX_VIDEO_CodingVPX; - formatParams->eColorFormat = OMX_COLOR_FormatUnused; - formatParams->xFramerate = 0; - } else { - CHECK_EQ(formatParams->nPortIndex, 1u); - - formatParams->eCompressionFormat = OMX_VIDEO_CodingUnused; - formatParams->eColorFormat = OMX_COLOR_FormatYUV420Planar; - formatParams->xFramerate = 0; - } - - return OMX_ErrorNone; - } - - default: - return SimpleSoftOMXComponent::internalGetParameter(index, params); - } -} - -OMX_ERRORTYPE SoftVPX::internalSetParameter( - OMX_INDEXTYPE index, const OMX_PTR params) { - switch (index) { - case OMX_IndexParamStandardComponentRole: - { - const OMX_PARAM_COMPONENTROLETYPE *roleParams = - (const OMX_PARAM_COMPONENTROLETYPE *)params; - - if (strncmp((const char *)roleParams->cRole, - "video_decoder.vpx", - OMX_MAX_STRINGNAME_SIZE - 1)) { - return OMX_ErrorUndefined; - } - - return OMX_ErrorNone; - } - - case OMX_IndexParamVideoPortFormat: - { - OMX_VIDEO_PARAM_PORTFORMATTYPE *formatParams = - (OMX_VIDEO_PARAM_PORTFORMATTYPE *)params; - - if (formatParams->nPortIndex > 1) { - return OMX_ErrorUndefined; - } - - if (formatParams->nIndex != 0) { - return OMX_ErrorNoMore; - } - - return OMX_ErrorNone; - } - - default: - return SimpleSoftOMXComponent::internalSetParameter(index, params); - } -} - void SoftVPX::onQueueFilled(OMX_U32 portIndex) { if (mOutputPortSettingsChange != NONE) { return; @@ -226,6 +92,7 @@ void SoftVPX::onQueueFilled(OMX_U32 portIndex) { List<BufferInfo *> &inQueue = getPortQueue(0); List<BufferInfo *> &outQueue = getPortQueue(1); + bool EOSseen = false; while (!inQueue.empty() && !outQueue.empty()) { BufferInfo *inInfo = *inQueue.begin(); @@ -235,17 +102,20 @@ void SoftVPX::onQueueFilled(OMX_U32 portIndex) { OMX_BUFFERHEADERTYPE *outHeader = outInfo->mHeader; if (inHeader->nFlags & OMX_BUFFERFLAG_EOS) { - inQueue.erase(inQueue.begin()); - inInfo->mOwnedByUs = false; - notifyEmptyBufferDone(inHeader); - - outHeader->nFilledLen = 0; - outHeader->nFlags = OMX_BUFFERFLAG_EOS; - - outQueue.erase(outQueue.begin()); - outInfo->mOwnedByUs = false; - notifyFillBufferDone(outHeader); - return; + EOSseen = true; + if (inHeader->nFilledLen == 0) { + inQueue.erase(inQueue.begin()); + inInfo->mOwnedByUs = false; + notifyEmptyBufferDone(inHeader); + + outHeader->nFilledLen = 0; + outHeader->nFlags = OMX_BUFFERFLAG_EOS; + + outQueue.erase(outQueue.begin()); + outInfo->mOwnedByUs = false; + notifyFillBufferDone(outHeader); + return; + } } if (vpx_codec_decode( @@ -266,8 +136,8 @@ void SoftVPX::onQueueFilled(OMX_U32 portIndex) { if (img != NULL) { CHECK_EQ(img->fmt, IMG_FMT_I420); - int32_t width = img->d_w; - int32_t height = img->d_h; + uint32_t width = img->d_w; + uint32_t height = img->d_h; if (width != mWidth || height != mHeight) { mWidth = width; @@ -282,7 +152,7 @@ void SoftVPX::onQueueFilled(OMX_U32 portIndex) { outHeader->nOffset = 0; outHeader->nFilledLen = (width * height * 3) / 2; - outHeader->nFlags = 0; + outHeader->nFlags = EOSseen ? OMX_BUFFERFLAG_EOS : 0; outHeader->nTimeStamp = inHeader->nTimeStamp; const uint8_t *srcLine = (const uint8_t *)img->planes[PLANE_Y]; @@ -325,58 +195,20 @@ void SoftVPX::onQueueFilled(OMX_U32 portIndex) { } } -void SoftVPX::onPortFlushCompleted(OMX_U32 portIndex) { -} - -void SoftVPX::onPortEnableCompleted(OMX_U32 portIndex, bool enabled) { - if (portIndex != 1) { - return; - } - - switch (mOutputPortSettingsChange) { - case NONE: - break; - - case AWAITING_DISABLED: - { - CHECK(!enabled); - mOutputPortSettingsChange = AWAITING_ENABLED; - break; - } - - default: - { - CHECK_EQ((int)mOutputPortSettingsChange, (int)AWAITING_ENABLED); - CHECK(enabled); - mOutputPortSettingsChange = NONE; - break; - } - } -} - -void SoftVPX::updatePortDefinitions() { - OMX_PARAM_PORTDEFINITIONTYPE *def = &editPortInfo(0)->mDef; - def->format.video.nFrameWidth = mWidth; - def->format.video.nFrameHeight = mHeight; - def->format.video.nStride = def->format.video.nFrameWidth; - def->format.video.nSliceHeight = def->format.video.nFrameHeight; - - def = &editPortInfo(1)->mDef; - def->format.video.nFrameWidth = mWidth; - def->format.video.nFrameHeight = mHeight; - def->format.video.nStride = def->format.video.nFrameWidth; - def->format.video.nSliceHeight = def->format.video.nFrameHeight; - - def->nBufferSize = - (def->format.video.nFrameWidth - * def->format.video.nFrameHeight * 3) / 2; -} - } // namespace android android::SoftOMXComponent *createSoftOMXComponent( const char *name, const OMX_CALLBACKTYPE *callbacks, OMX_PTR appData, OMX_COMPONENTTYPE **component) { - return new android::SoftVPX(name, callbacks, appData, component); + if (!strcmp(name, "OMX.google.vp8.decoder")) { + return new android::SoftVPX( + name, "video_decoder.vp8", OMX_VIDEO_CodingVP8, + callbacks, appData, component); + } else if (!strcmp(name, "OMX.google.vp9.decoder")) { + return new android::SoftVPX( + name, "video_decoder.vp9", OMX_VIDEO_CodingVP9, + callbacks, appData, component); + } else { + CHECK(!"Unknown component"); + } } - diff --git a/media/libstagefright/codecs/on2/dec/SoftVPX.h b/media/libstagefright/codecs/on2/dec/SoftVPX.h index 3e814a2..cd5eb28 100644 --- a/media/libstagefright/codecs/on2/dec/SoftVPX.h +++ b/media/libstagefright/codecs/on2/dec/SoftVPX.h @@ -18,12 +18,14 @@ #define SOFT_VPX_H_ -#include "SimpleSoftOMXComponent.h" +#include "SoftVideoDecoderOMXComponent.h" namespace android { -struct SoftVPX : public SimpleSoftOMXComponent { +struct SoftVPX : public SoftVideoDecoderOMXComponent { SoftVPX(const char *name, + const char *componentRole, + OMX_VIDEO_CODINGTYPE codingType, const OMX_CALLBACKTYPE *callbacks, OMX_PTR appData, OMX_COMPONENTTYPE **component); @@ -31,36 +33,21 @@ struct SoftVPX : public SimpleSoftOMXComponent { protected: virtual ~SoftVPX(); - virtual OMX_ERRORTYPE internalGetParameter( - OMX_INDEXTYPE index, OMX_PTR params); - - virtual OMX_ERRORTYPE internalSetParameter( - OMX_INDEXTYPE index, const OMX_PTR params); - virtual void onQueueFilled(OMX_U32 portIndex); - virtual void onPortFlushCompleted(OMX_U32 portIndex); - virtual void onPortEnableCompleted(OMX_U32 portIndex, bool enabled); private: enum { kNumBuffers = 4 }; - void *mCtx; - - int32_t mWidth; - int32_t mHeight; - enum { - NONE, - AWAITING_DISABLED, - AWAITING_ENABLED - } mOutputPortSettingsChange; + MODE_VP8, + MODE_VP9 + } mMode; - void initPorts(); - status_t initDecoder(); + void *mCtx; - void updatePortDefinitions(); + status_t initDecoder(); DISALLOW_EVIL_CONSTRUCTORS(SoftVPX); }; diff --git a/media/libstagefright/codecs/on2/enc/Android.mk b/media/libstagefright/codecs/on2/enc/Android.mk index a92d376..4060a0a 100644 --- a/media/libstagefright/codecs/on2/enc/Android.mk +++ b/media/libstagefright/codecs/on2/enc/Android.mk @@ -12,11 +12,16 @@ LOCAL_C_INCLUDES := \ frameworks/av/media/libstagefright/include \ frameworks/native/include/media/openmax \ +ifeq ($(TARGET_DEVICE), manta) + LOCAL_CFLAGS += -DSURFACE_IS_BGR32 +endif + LOCAL_STATIC_LIBRARIES := \ libvpx LOCAL_SHARED_LIBRARIES := \ libstagefright libstagefright_omx libstagefright_foundation libutils liblog \ + libhardware \ LOCAL_MODULE := libstagefright_soft_vpxenc LOCAL_MODULE_TAGS := optional diff --git a/media/libstagefright/codecs/on2/enc/SoftVPXEncoder.cpp b/media/libstagefright/codecs/on2/enc/SoftVPXEncoder.cpp index e25637a..8375cac 100644 --- a/media/libstagefright/codecs/on2/enc/SoftVPXEncoder.cpp +++ b/media/libstagefright/codecs/on2/enc/SoftVPXEncoder.cpp @@ -20,6 +20,8 @@ #include <utils/Log.h> +#include <media/hardware/HardwareAPI.h> +#include <media/hardware/MetadataBufferType.h> #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/MediaDefs.h> @@ -81,6 +83,52 @@ inline static void ConvertSemiPlanarToPlanar(uint8_t *inyuv, } } +static void ConvertRGB32ToPlanar( + const uint8_t *src, uint8_t *dstY, int32_t width, int32_t height) { + CHECK((width & 1) == 0); + CHECK((height & 1) == 0); + + uint8_t *dstU = dstY + width * height; + uint8_t *dstV = dstU + (width / 2) * (height / 2); + + for (int32_t y = 0; y < height; ++y) { + for (int32_t x = 0; x < width; ++x) { +#ifdef SURFACE_IS_BGR32 + unsigned blue = src[4 * x]; + unsigned green = src[4 * x + 1]; + unsigned red= src[4 * x + 2]; +#else + unsigned red= src[4 * x]; + unsigned green = src[4 * x + 1]; + unsigned blue = src[4 * x + 2]; +#endif + + unsigned luma = + ((red * 66 + green * 129 + blue * 25) >> 8) + 16; + + dstY[x] = luma; + + if ((x & 1) == 0 && (y & 1) == 0) { + unsigned U = + ((-red * 38 - green * 74 + blue * 112) >> 8) + 128; + + unsigned V = + ((red * 112 - green * 94 - blue * 18) >> 8) + 128; + + dstU[x / 2] = U; + dstV[x / 2] = V; + } + } + + if ((y & 1) == 0) { + dstU += width / 2; + dstV += width / 2; + } + + src += 4 * width; + dstY += width; + } +} SoftVPXEncoder::SoftVPXEncoder(const char *name, const OMX_CALLBACKTYPE *callbacks, @@ -93,14 +141,17 @@ SoftVPXEncoder::SoftVPXEncoder(const char *name, mWidth(176), mHeight(144), mBitrate(192000), // in bps + mBitrateUpdated(false), mBitrateControlMode(VPX_VBR), // variable bitrate mFrameDurationUs(33333), // Defaults to 30 fps mDCTPartitions(0), mErrorResilience(OMX_FALSE), mColorFormat(OMX_COLOR_FormatYUV420Planar), mLevel(OMX_VIDEO_VP8Level_Version0), - mConversionBuffer(NULL) { - + mConversionBuffer(NULL), + mInputDataIsMeta(false), + mGrallocModule(NULL), + mKeyFrameRequested(false) { initPorts(); } @@ -165,8 +216,8 @@ void SoftVPXEncoder::initPorts() { outputPort.eDir = OMX_DirOutput; outputPort.nBufferAlignment = kOutputBufferAlignment; outputPort.format.video.cMIMEType = - const_cast<char *>(MEDIA_MIMETYPE_VIDEO_VPX); - outputPort.format.video.eCompressionFormat = OMX_VIDEO_CodingVPX; + const_cast<char *>(MEDIA_MIMETYPE_VIDEO_VP8); + outputPort.format.video.eCompressionFormat = OMX_VIDEO_CodingVP8; outputPort.format.video.eColorFormat = OMX_COLOR_FormatUnused; outputPort.format.video.pNativeWindow = NULL; outputPort.nBufferSize = 256 * 1024; // arbitrary @@ -247,7 +298,7 @@ status_t SoftVPXEncoder::initEncoder() { return UNKNOWN_ERROR; } - if (mColorFormat == OMX_COLOR_FormatYUV420SemiPlanar) { + if (mColorFormat == OMX_COLOR_FormatYUV420SemiPlanar || mInputDataIsMeta) { if (mConversionBuffer == NULL) { mConversionBuffer = (uint8_t *)malloc(mWidth * mHeight * 3 / 2); if (mConversionBuffer == NULL) { @@ -315,7 +366,7 @@ OMX_ERRORTYPE SoftVPXEncoder::internalGetParameter(OMX_INDEXTYPE index, formatParams->xFramerate = (1000000/mFrameDurationUs) << 16; return OMX_ErrorNone; } else if (formatParams->nPortIndex == kOutputPortIndex) { - formatParams->eCompressionFormat = OMX_VIDEO_CodingVPX; + formatParams->eCompressionFormat = OMX_VIDEO_CodingVP8; formatParams->eColorFormat = OMX_COLOR_FormatUnused; formatParams->xFramerate = 0; return OMX_ErrorNone; @@ -427,9 +478,17 @@ OMX_ERRORTYPE SoftVPXEncoder::internalSetParameter(OMX_INDEXTYPE index, (const OMX_VIDEO_PARAM_BITRATETYPE *)param); case OMX_IndexParamPortDefinition: - return internalSetPortParams( + { + OMX_ERRORTYPE err = internalSetPortParams( (const OMX_PARAM_PORTDEFINITIONTYPE *)param); + if (err != OMX_ErrorNone) { + return err; + } + + return SimpleSoftOMXComponent::internalSetParameter(index, param); + } + case OMX_IndexParamVideoPortFormat: return internalSetFormatParams( (const OMX_VIDEO_PARAM_PORTFORMATTYPE *)param); @@ -442,11 +501,63 @@ OMX_ERRORTYPE SoftVPXEncoder::internalSetParameter(OMX_INDEXTYPE index, return internalSetProfileLevel( (const OMX_VIDEO_PARAM_PROFILELEVELTYPE *)param); + case OMX_IndexVendorStartUnused: + { + // storeMetaDataInBuffers + const StoreMetaDataInBuffersParams *storeParam = + (const StoreMetaDataInBuffersParams *)param; + + if (storeParam->nPortIndex != kInputPortIndex) { + return OMX_ErrorBadPortIndex; + } + + mInputDataIsMeta = (storeParam->bStoreMetaData == OMX_TRUE); + + return OMX_ErrorNone; + } + default: return SimpleSoftOMXComponent::internalSetParameter(index, param); } } +OMX_ERRORTYPE SoftVPXEncoder::setConfig( + OMX_INDEXTYPE index, const OMX_PTR _params) { + switch (index) { + case OMX_IndexConfigVideoIntraVOPRefresh: + { + OMX_CONFIG_INTRAREFRESHVOPTYPE *params = + (OMX_CONFIG_INTRAREFRESHVOPTYPE *)_params; + + if (params->nPortIndex != kOutputPortIndex) { + return OMX_ErrorBadPortIndex; + } + + mKeyFrameRequested = params->IntraRefreshVOP; + return OMX_ErrorNone; + } + + case OMX_IndexConfigVideoBitrate: + { + OMX_VIDEO_CONFIG_BITRATETYPE *params = + (OMX_VIDEO_CONFIG_BITRATETYPE *)_params; + + if (params->nPortIndex != kOutputPortIndex) { + return OMX_ErrorBadPortIndex; + } + + if (mBitrate != params->nEncodeBitrate) { + mBitrate = params->nEncodeBitrate; + mBitrateUpdated = true; + } + return OMX_ErrorNone; + } + + default: + return SimpleSoftOMXComponent::setConfig(index, _params); + } +} + OMX_ERRORTYPE SoftVPXEncoder::internalSetProfileLevel( const OMX_VIDEO_PARAM_PROFILELEVELTYPE* profileAndLevel) { if (profileAndLevel->nPortIndex != kOutputPortIndex) { @@ -507,13 +618,17 @@ OMX_ERRORTYPE SoftVPXEncoder::internalSetFormatParams( format->eColorFormat == OMX_COLOR_FormatYUV420SemiPlanar || format->eColorFormat == OMX_COLOR_FormatAndroidOpaque) { mColorFormat = format->eColorFormat; + + OMX_PARAM_PORTDEFINITIONTYPE *def = &editPortInfo(kInputPortIndex)->mDef; + def->format.video.eColorFormat = mColorFormat; + return OMX_ErrorNone; } else { ALOGE("Unsupported color format %i", format->eColorFormat); return OMX_ErrorUnsupportedSetting; } } else if (format->nPortIndex == kOutputPortIndex) { - if (format->eCompressionFormat == OMX_VIDEO_CodingVPX) { + if (format->eCompressionFormat == OMX_VIDEO_CodingVP8) { return OMX_ErrorNone; } else { return OMX_ErrorUnsupportedSetting; @@ -529,7 +644,7 @@ OMX_ERRORTYPE SoftVPXEncoder::internalSetRoleParams( const char* roleText = (const char*)role->cRole; const size_t roleTextMaxSize = OMX_MAX_STRINGNAME_SIZE - 1; - if (strncmp(roleText, "video_encoder.vpx", roleTextMaxSize)) { + if (strncmp(roleText, "video_encoder.vp8", roleTextMaxSize)) { ALOGE("Unsupported component role"); return OMX_ErrorBadParameter; } @@ -552,11 +667,17 @@ OMX_ERRORTYPE SoftVPXEncoder::internalSetPortParams( if (port->format.video.eColorFormat == OMX_COLOR_FormatYUV420Planar || port->format.video.eColorFormat == OMX_COLOR_FormatYUV420SemiPlanar || port->format.video.eColorFormat == OMX_COLOR_FormatAndroidOpaque) { - mColorFormat = port->format.video.eColorFormat; + mColorFormat = port->format.video.eColorFormat; } else { return OMX_ErrorUnsupportedSetting; } + OMX_PARAM_PORTDEFINITIONTYPE *def = &editPortInfo(kInputPortIndex)->mDef; + def->format.video.nFrameWidth = mWidth; + def->format.video.nFrameHeight = mHeight; + def->format.video.xFramerate = port->format.video.xFramerate; + def->format.video.eColorFormat = mColorFormat; + return OMX_ErrorNone; } else if (port->nPortIndex == kOutputPortIndex) { mBitrate = port->format.video.nBitrate; @@ -625,24 +746,78 @@ void SoftVPXEncoder::onQueueFilled(OMX_U32 portIndex) { return; } - uint8_t* source = inputBufferHeader->pBuffer + inputBufferHeader->nOffset; + uint8_t *source = + inputBufferHeader->pBuffer + inputBufferHeader->nOffset; + + if (mInputDataIsMeta) { + CHECK_GE(inputBufferHeader->nFilledLen, + 4 + sizeof(buffer_handle_t)); + + uint32_t bufferType = *(uint32_t *)source; + CHECK_EQ(bufferType, kMetadataBufferTypeGrallocSource); + + if (mGrallocModule == NULL) { + CHECK_EQ(0, hw_get_module( + GRALLOC_HARDWARE_MODULE_ID, &mGrallocModule)); + } + + const gralloc_module_t *grmodule = + (const gralloc_module_t *)mGrallocModule; + + buffer_handle_t handle = *(buffer_handle_t *)(source + 4); + + void *bits; + CHECK_EQ(0, + grmodule->lock( + grmodule, handle, + GRALLOC_USAGE_SW_READ_OFTEN + | GRALLOC_USAGE_SW_WRITE_NEVER, + 0, 0, mWidth, mHeight, &bits)); + + ConvertRGB32ToPlanar( + (const uint8_t *)bits, mConversionBuffer, mWidth, mHeight); + + source = mConversionBuffer; + + CHECK_EQ(0, grmodule->unlock(grmodule, handle)); + } else if (mColorFormat == OMX_COLOR_FormatYUV420SemiPlanar) { + ConvertSemiPlanarToPlanar( + source, mConversionBuffer, mWidth, mHeight); - // NOTE: As much as nothing is known about color format - // when it is denoted as AndroidOpaque, it is at least - // assumed to be planar. - if (mColorFormat == OMX_COLOR_FormatYUV420SemiPlanar) { - ConvertSemiPlanarToPlanar(source, mConversionBuffer, mWidth, mHeight); source = mConversionBuffer; } vpx_image_t raw_frame; vpx_img_wrap(&raw_frame, VPX_IMG_FMT_I420, mWidth, mHeight, kInputBufferAlignment, source); - codec_return = vpx_codec_encode(mCodecContext, - &raw_frame, - inputBufferHeader->nTimeStamp, // in timebase units - mFrameDurationUs, // frame duration in timebase units - 0, // frame flags - VPX_DL_REALTIME); // encoding deadline + + vpx_enc_frame_flags_t flags = 0; + if (mKeyFrameRequested) { + flags |= VPX_EFLAG_FORCE_KF; + mKeyFrameRequested = false; + } + + if (mBitrateUpdated) { + mCodecConfiguration->rc_target_bitrate = mBitrate/1000; + vpx_codec_err_t res = vpx_codec_enc_config_set(mCodecContext, + mCodecConfiguration); + if (res != VPX_CODEC_OK) { + ALOGE("vp8 encoder failed to update bitrate: %s", + vpx_codec_err_to_string(res)); + notify(OMX_EventError, + OMX_ErrorUndefined, + 0, // Extra notification data + NULL); // Notification data pointer + } + mBitrateUpdated = false; + } + + codec_return = vpx_codec_encode( + mCodecContext, + &raw_frame, + inputBufferHeader->nTimeStamp, // in timebase units + mFrameDurationUs, // frame duration in timebase units + flags, // frame flags + VPX_DL_REALTIME); // encoding deadline if (codec_return != VPX_CODEC_OK) { ALOGE("vpx encoder failed to encode frame"); notify(OMX_EventError, @@ -660,6 +835,8 @@ void SoftVPXEncoder::onQueueFilled(OMX_U32 portIndex) { if (encoded_packet->kind == VPX_CODEC_CX_FRAME_PKT) { outputBufferHeader->nTimeStamp = encoded_packet->data.frame.pts; outputBufferHeader->nFlags = 0; + if (encoded_packet->data.frame.flags & VPX_FRAME_IS_KEY) + outputBufferHeader->nFlags |= OMX_BUFFERFLAG_SYNCFRAME; outputBufferHeader->nOffset = 0; outputBufferHeader->nFilledLen = encoded_packet->data.frame.sz; memcpy(outputBufferHeader->pBuffer, @@ -676,6 +853,17 @@ void SoftVPXEncoder::onQueueFilled(OMX_U32 portIndex) { notifyEmptyBufferDone(inputBufferHeader); } } + +OMX_ERRORTYPE SoftVPXEncoder::getExtensionIndex( + const char *name, OMX_INDEXTYPE *index) { + if (!strcmp(name, "OMX.google.android.index.storeMetaDataInBuffers")) { + *index = OMX_IndexVendorStartUnused; + return OMX_ErrorNone; + } + + return SimpleSoftOMXComponent::getExtensionIndex(name, index); +} + } // namespace android diff --git a/media/libstagefright/codecs/on2/enc/SoftVPXEncoder.h b/media/libstagefright/codecs/on2/enc/SoftVPXEncoder.h index 3bc05c0..076830f 100644 --- a/media/libstagefright/codecs/on2/enc/SoftVPXEncoder.h +++ b/media/libstagefright/codecs/on2/enc/SoftVPXEncoder.h @@ -23,6 +23,8 @@ #include <OMX_VideoExt.h> #include <OMX_IndexExt.h> +#include <hardware/gralloc.h> + #include "vpx/vpx_encoder.h" #include "vpx/vpx_codec.h" #include "vpx/vp8cx.h" @@ -57,14 +59,13 @@ namespace android { // - OMX timestamps are in microseconds, therefore // encoder timebase is fixed to 1/1000000 -class SoftVPXEncoder : public SimpleSoftOMXComponent { - public: +struct SoftVPXEncoder : public SimpleSoftOMXComponent { SoftVPXEncoder(const char *name, const OMX_CALLBACKTYPE *callbacks, OMX_PTR appData, OMX_COMPONENTTYPE **component); - protected: +protected: virtual ~SoftVPXEncoder(); // Returns current values for requested OMX @@ -77,13 +78,19 @@ class SoftVPXEncoder : public SimpleSoftOMXComponent { virtual OMX_ERRORTYPE internalSetParameter( OMX_INDEXTYPE index, const OMX_PTR param); + virtual OMX_ERRORTYPE setConfig( + OMX_INDEXTYPE index, const OMX_PTR params); + // OMX callback when buffers available // Note that both an input and output buffer // is expected to be available to carry out // encoding of the frame virtual void onQueueFilled(OMX_U32 portIndex); - private: + virtual OMX_ERRORTYPE getExtensionIndex( + const char *name, OMX_INDEXTYPE *index); + +private: // number of buffers allocated per port static const uint32_t kNumBuffers = 4; @@ -121,7 +128,10 @@ class SoftVPXEncoder : public SimpleSoftOMXComponent { int32_t mHeight; // Target bitrate set for the encoder, in bits per second. - int32_t mBitrate; + uint32_t mBitrate; + + // If a request for a change it bitrate has been received. + bool mBitrateUpdated; // Bitrate control mode, either constant or variable vpx_rc_mode mBitrateControlMode; @@ -156,6 +166,11 @@ class SoftVPXEncoder : public SimpleSoftOMXComponent { // indeed YUV420SemiPlanar. uint8_t* mConversionBuffer; + bool mInputDataIsMeta; + const hw_module_t *mGrallocModule; + + bool mKeyFrameRequested; + // Initializes input and output OMX ports with sensible // default values. void initPorts(); @@ -175,7 +190,7 @@ class SoftVPXEncoder : public SimpleSoftOMXComponent { const OMX_VIDEO_PARAM_PORTFORMATTYPE* format); // Verifies the component role tried to be set to this OMX component is - // strictly video_encoder.vpx + // strictly video_encoder.vp8 OMX_ERRORTYPE internalSetRoleParams( const OMX_PARAM_COMPONENTROLETYPE* role); diff --git a/media/libstagefright/codecs/on2/h264dec/Android.mk b/media/libstagefright/codecs/on2/h264dec/Android.mk index 2539f98..655b2ab 100644 --- a/media/libstagefright/codecs/on2/h264dec/Android.mk +++ b/media/libstagefright/codecs/on2/h264dec/Android.mk @@ -119,7 +119,7 @@ LOCAL_C_INCLUDES := $(LOCAL_PATH)/inc LOCAL_SHARED_LIBRARIES := libstagefright_soft_h264dec -LOCAL_MODULE_TAGS := debug +LOCAL_MODULE_TAGS := optional LOCAL_MODULE := decoder diff --git a/media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp b/media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp index 6e36651..7ddb13c 100644 --- a/media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp +++ b/media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp @@ -47,38 +47,28 @@ static const CodecProfileLevel kProfileLevels[] = { { OMX_VIDEO_AVCProfileBaseline, OMX_VIDEO_AVCLevel51 }, }; -template<class T> -static void InitOMXParams(T *params) { - params->nSize = sizeof(T); - params->nVersion.s.nVersionMajor = 1; - params->nVersion.s.nVersionMinor = 0; - params->nVersion.s.nRevision = 0; - params->nVersion.s.nStep = 0; -} - SoftAVC::SoftAVC( const char *name, const OMX_CALLBACKTYPE *callbacks, OMX_PTR appData, OMX_COMPONENTTYPE **component) - : SimpleSoftOMXComponent(name, callbacks, appData, component), + : SoftVideoDecoderOMXComponent( + name, "video_decoder.avc", OMX_VIDEO_CodingAVC, + kProfileLevels, ARRAY_SIZE(kProfileLevels), + 320 /* width */, 240 /* height */, callbacks, appData, component), mHandle(NULL), mInputBufferCount(0), - mWidth(320), - mHeight(240), mPictureSize(mWidth * mHeight * 3 / 2), - mCropLeft(0), - mCropTop(0), - mCropWidth(mWidth), - mCropHeight(mHeight), mFirstPicture(NULL), mFirstPictureId(-1), mPicId(0), mHeadersDecoded(false), mEOSStatus(INPUT_DATA_AVAILABLE), - mOutputPortSettingsChange(NONE), mSignalledError(false) { - initPorts(); + initPorts( + kNumInputBuffers, 8192 /* inputBufferSize */, + kNumOutputBuffers, MEDIA_MIMETYPE_VIDEO_AVC); + CHECK_EQ(initDecoder(), (status_t)OK); } @@ -100,65 +90,6 @@ SoftAVC::~SoftAVC() { delete[] mFirstPicture; } -void SoftAVC::initPorts() { - OMX_PARAM_PORTDEFINITIONTYPE def; - InitOMXParams(&def); - - def.nPortIndex = kInputPortIndex; - def.eDir = OMX_DirInput; - def.nBufferCountMin = kNumInputBuffers; - def.nBufferCountActual = def.nBufferCountMin; - def.nBufferSize = 8192; - def.bEnabled = OMX_TRUE; - def.bPopulated = OMX_FALSE; - def.eDomain = OMX_PortDomainVideo; - def.bBuffersContiguous = OMX_FALSE; - def.nBufferAlignment = 1; - - def.format.video.cMIMEType = const_cast<char *>(MEDIA_MIMETYPE_VIDEO_AVC); - def.format.video.pNativeRender = NULL; - def.format.video.nFrameWidth = mWidth; - def.format.video.nFrameHeight = mHeight; - def.format.video.nStride = def.format.video.nFrameWidth; - def.format.video.nSliceHeight = def.format.video.nFrameHeight; - def.format.video.nBitrate = 0; - def.format.video.xFramerate = 0; - def.format.video.bFlagErrorConcealment = OMX_FALSE; - def.format.video.eCompressionFormat = OMX_VIDEO_CodingAVC; - def.format.video.eColorFormat = OMX_COLOR_FormatUnused; - def.format.video.pNativeWindow = NULL; - - addPort(def); - - def.nPortIndex = kOutputPortIndex; - def.eDir = OMX_DirOutput; - def.nBufferCountMin = kNumOutputBuffers; - def.nBufferCountActual = def.nBufferCountMin; - def.bEnabled = OMX_TRUE; - def.bPopulated = OMX_FALSE; - def.eDomain = OMX_PortDomainVideo; - def.bBuffersContiguous = OMX_FALSE; - def.nBufferAlignment = 2; - - def.format.video.cMIMEType = const_cast<char *>(MEDIA_MIMETYPE_VIDEO_RAW); - def.format.video.pNativeRender = NULL; - def.format.video.nFrameWidth = mWidth; - def.format.video.nFrameHeight = mHeight; - def.format.video.nStride = def.format.video.nFrameWidth; - def.format.video.nSliceHeight = def.format.video.nFrameHeight; - def.format.video.nBitrate = 0; - def.format.video.xFramerate = 0; - def.format.video.bFlagErrorConcealment = OMX_FALSE; - def.format.video.eCompressionFormat = OMX_VIDEO_CodingUnused; - def.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar; - def.format.video.pNativeWindow = NULL; - - def.nBufferSize = - (def.format.video.nFrameWidth * def.format.video.nFrameHeight * 3) / 2; - - addPort(def); -} - status_t SoftAVC::initDecoder() { // Force decoder to output buffers in display order. if (H264SwDecInit(&mHandle, 0) == H264SWDEC_OK) { @@ -167,126 +98,6 @@ status_t SoftAVC::initDecoder() { return UNKNOWN_ERROR; } -OMX_ERRORTYPE SoftAVC::internalGetParameter( - OMX_INDEXTYPE index, OMX_PTR params) { - switch (index) { - case OMX_IndexParamVideoPortFormat: - { - OMX_VIDEO_PARAM_PORTFORMATTYPE *formatParams = - (OMX_VIDEO_PARAM_PORTFORMATTYPE *)params; - - if (formatParams->nPortIndex > kOutputPortIndex) { - return OMX_ErrorUndefined; - } - - if (formatParams->nIndex != 0) { - return OMX_ErrorNoMore; - } - - if (formatParams->nPortIndex == kInputPortIndex) { - formatParams->eCompressionFormat = OMX_VIDEO_CodingAVC; - formatParams->eColorFormat = OMX_COLOR_FormatUnused; - formatParams->xFramerate = 0; - } else { - CHECK(formatParams->nPortIndex == kOutputPortIndex); - - formatParams->eCompressionFormat = OMX_VIDEO_CodingUnused; - formatParams->eColorFormat = OMX_COLOR_FormatYUV420Planar; - formatParams->xFramerate = 0; - } - - return OMX_ErrorNone; - } - - case OMX_IndexParamVideoProfileLevelQuerySupported: - { - OMX_VIDEO_PARAM_PROFILELEVELTYPE *profileLevel = - (OMX_VIDEO_PARAM_PROFILELEVELTYPE *) params; - - if (profileLevel->nPortIndex != kInputPortIndex) { - ALOGE("Invalid port index: %ld", profileLevel->nPortIndex); - return OMX_ErrorUnsupportedIndex; - } - - size_t index = profileLevel->nProfileIndex; - size_t nProfileLevels = - sizeof(kProfileLevels) / sizeof(kProfileLevels[0]); - if (index >= nProfileLevels) { - return OMX_ErrorNoMore; - } - - profileLevel->eProfile = kProfileLevels[index].mProfile; - profileLevel->eLevel = kProfileLevels[index].mLevel; - return OMX_ErrorNone; - } - - default: - return SimpleSoftOMXComponent::internalGetParameter(index, params); - } -} - -OMX_ERRORTYPE SoftAVC::internalSetParameter( - OMX_INDEXTYPE index, const OMX_PTR params) { - switch (index) { - case OMX_IndexParamStandardComponentRole: - { - const OMX_PARAM_COMPONENTROLETYPE *roleParams = - (const OMX_PARAM_COMPONENTROLETYPE *)params; - - if (strncmp((const char *)roleParams->cRole, - "video_decoder.avc", - OMX_MAX_STRINGNAME_SIZE - 1)) { - return OMX_ErrorUndefined; - } - - return OMX_ErrorNone; - } - - case OMX_IndexParamVideoPortFormat: - { - OMX_VIDEO_PARAM_PORTFORMATTYPE *formatParams = - (OMX_VIDEO_PARAM_PORTFORMATTYPE *)params; - - if (formatParams->nPortIndex > kOutputPortIndex) { - return OMX_ErrorUndefined; - } - - if (formatParams->nIndex != 0) { - return OMX_ErrorNoMore; - } - - return OMX_ErrorNone; - } - - default: - return SimpleSoftOMXComponent::internalSetParameter(index, params); - } -} - -OMX_ERRORTYPE SoftAVC::getConfig( - OMX_INDEXTYPE index, OMX_PTR params) { - switch (index) { - case OMX_IndexConfigCommonOutputCrop: - { - OMX_CONFIG_RECTTYPE *rectParams = (OMX_CONFIG_RECTTYPE *)params; - - if (rectParams->nPortIndex != 1) { - return OMX_ErrorUndefined; - } - - rectParams->nLeft = mCropLeft; - rectParams->nTop = mCropTop; - rectParams->nWidth = mCropWidth; - rectParams->nHeight = mCropHeight; - - return OMX_ErrorNone; - } - - default: - return OMX_ErrorUnsupportedIndex; - } -} - void SoftAVC::onQueueFilled(OMX_U32 portIndex) { if (mSignalledError || mOutputPortSettingsChange != NONE) { return; @@ -298,13 +109,21 @@ void SoftAVC::onQueueFilled(OMX_U32 portIndex) { List<BufferInfo *> &inQueue = getPortQueue(kInputPortIndex); List<BufferInfo *> &outQueue = getPortQueue(kOutputPortIndex); + + if (mHeadersDecoded) { + // Dequeue any already decoded output frames to free up space + // in the output queue. + + drainAllOutputBuffers(false /* eos */); + } + H264SwDecRet ret = H264SWDEC_PIC_RDY; bool portSettingsChanged = false; while ((mEOSStatus != INPUT_DATA_AVAILABLE || !inQueue.empty()) && outQueue.size() == kNumOutputBuffers) { if (mEOSStatus == INPUT_EOS_SEEN) { - drainAllOutputBuffers(); + drainAllOutputBuffers(true /* eos */); return; } @@ -392,15 +211,7 @@ void SoftAVC::onQueueFilled(OMX_U32 portIndex) { mFirstPictureId = -1; } - while (!outQueue.empty() && - mHeadersDecoded && - H264SwDecNextPicture(mHandle, &decodedPicture, 0) - == H264SWDEC_PIC_RDY) { - - int32_t picId = decodedPicture.picId; - uint8_t *data = (uint8_t *) decodedPicture.pOutputPicture; - drainOneOutputBuffer(picId, data); - } + drainAllOutputBuffers(false /* eos */); } } @@ -409,8 +220,6 @@ bool SoftAVC::handlePortSettingChangeEvent(const H264SwDecInfo *info) { mWidth = info->picWidth; mHeight = info->picHeight; mPictureSize = mWidth * mHeight * 3 / 2; - mCropWidth = mWidth; - mCropHeight = mHeight; updatePortDefinitions(); notify(OMX_EventPortSettingsChanged, 1, 0, NULL); mOutputPortSettingsChange = AWAITING_DISABLED; @@ -463,43 +272,38 @@ void SoftAVC::drainOneOutputBuffer(int32_t picId, uint8_t* data) { notifyFillBufferDone(outHeader); } -bool SoftAVC::drainAllOutputBuffers() { +void SoftAVC::drainAllOutputBuffers(bool eos) { List<BufferInfo *> &outQueue = getPortQueue(kOutputPortIndex); H264SwDecPicture decodedPicture; + if (mHeadersDecoded) { + while (!outQueue.empty() + && H264SWDEC_PIC_RDY == H264SwDecNextPicture( + mHandle, &decodedPicture, eos /* flush */)) { + int32_t picId = decodedPicture.picId; + uint8_t *data = (uint8_t *) decodedPicture.pOutputPicture; + drainOneOutputBuffer(picId, data); + } + } + + if (!eos) { + return; + } + while (!outQueue.empty()) { BufferInfo *outInfo = *outQueue.begin(); outQueue.erase(outQueue.begin()); OMX_BUFFERHEADERTYPE *outHeader = outInfo->mHeader; - if (mHeadersDecoded && - H264SWDEC_PIC_RDY == - H264SwDecNextPicture(mHandle, &decodedPicture, 1 /* flush */)) { - int32_t picId = decodedPicture.picId; - CHECK(mPicToHeaderMap.indexOfKey(picId) >= 0); - - memcpy(outHeader->pBuffer + outHeader->nOffset, - decodedPicture.pOutputPicture, - mPictureSize); - - OMX_BUFFERHEADERTYPE *header = mPicToHeaderMap.valueFor(picId); - outHeader->nTimeStamp = header->nTimeStamp; - outHeader->nFlags = header->nFlags; - outHeader->nFilledLen = mPictureSize; - mPicToHeaderMap.removeItem(picId); - delete header; - } else { - outHeader->nTimeStamp = 0; - outHeader->nFilledLen = 0; - outHeader->nFlags = OMX_BUFFERFLAG_EOS; - mEOSStatus = OUTPUT_FRAMES_FLUSHED; - } + outHeader->nTimeStamp = 0; + outHeader->nFilledLen = 0; + outHeader->nFlags = OMX_BUFFERFLAG_EOS; outInfo->mOwnedByUs = false; notifyFillBufferDone(outHeader); - } - return true; + mEOSStatus = OUTPUT_FRAMES_FLUSHED; + } } void SoftAVC::onPortFlushCompleted(OMX_U32 portIndex) { @@ -508,44 +312,9 @@ void SoftAVC::onPortFlushCompleted(OMX_U32 portIndex) { } } -void SoftAVC::onPortEnableCompleted(OMX_U32 portIndex, bool enabled) { - switch (mOutputPortSettingsChange) { - case NONE: - break; - - case AWAITING_DISABLED: - { - CHECK(!enabled); - mOutputPortSettingsChange = AWAITING_ENABLED; - break; - } - - default: - { - CHECK_EQ((int)mOutputPortSettingsChange, (int)AWAITING_ENABLED); - CHECK(enabled); - mOutputPortSettingsChange = NONE; - break; - } - } -} - -void SoftAVC::updatePortDefinitions() { - OMX_PARAM_PORTDEFINITIONTYPE *def = &editPortInfo(0)->mDef; - def->format.video.nFrameWidth = mWidth; - def->format.video.nFrameHeight = mHeight; - def->format.video.nStride = def->format.video.nFrameWidth; - def->format.video.nSliceHeight = def->format.video.nFrameHeight; - - def = &editPortInfo(1)->mDef; - def->format.video.nFrameWidth = mWidth; - def->format.video.nFrameHeight = mHeight; - def->format.video.nStride = def->format.video.nFrameWidth; - def->format.video.nSliceHeight = def->format.video.nFrameHeight; - - def->nBufferSize = - (def->format.video.nFrameWidth - * def->format.video.nFrameHeight * 3) / 2; +void SoftAVC::onReset() { + SoftVideoDecoderOMXComponent::onReset(); + mSignalledError = false; } } // namespace android diff --git a/media/libstagefright/codecs/on2/h264dec/SoftAVC.h b/media/libstagefright/codecs/on2/h264dec/SoftAVC.h index 879b014..ee69926 100644 --- a/media/libstagefright/codecs/on2/h264dec/SoftAVC.h +++ b/media/libstagefright/codecs/on2/h264dec/SoftAVC.h @@ -18,7 +18,7 @@ #define SOFT_AVC_H_ -#include "SimpleSoftOMXComponent.h" +#include "SoftVideoDecoderOMXComponent.h" #include <utils/KeyedVector.h> #include "H264SwDecApi.h" @@ -26,7 +26,7 @@ namespace android { -struct SoftAVC : public SimpleSoftOMXComponent { +struct SoftAVC : public SoftVideoDecoderOMXComponent { SoftAVC(const char *name, const OMX_CALLBACKTYPE *callbacks, OMX_PTR appData, @@ -35,22 +35,12 @@ struct SoftAVC : public SimpleSoftOMXComponent { protected: virtual ~SoftAVC(); - virtual OMX_ERRORTYPE internalGetParameter( - OMX_INDEXTYPE index, OMX_PTR params); - - virtual OMX_ERRORTYPE internalSetParameter( - OMX_INDEXTYPE index, const OMX_PTR params); - - virtual OMX_ERRORTYPE getConfig(OMX_INDEXTYPE index, OMX_PTR params); - virtual void onQueueFilled(OMX_U32 portIndex); virtual void onPortFlushCompleted(OMX_U32 portIndex); - virtual void onPortEnableCompleted(OMX_U32 portIndex, bool enabled); + virtual void onReset(); private: enum { - kInputPortIndex = 0, - kOutputPortIndex = 1, kNumInputBuffers = 8, kNumOutputBuffers = 2, }; @@ -65,9 +55,7 @@ private: size_t mInputBufferCount; - uint32_t mWidth, mHeight, mPictureSize; - uint32_t mCropLeft, mCropTop; - uint32_t mCropWidth, mCropHeight; + uint32_t mPictureSize; uint8_t *mFirstPicture; int32_t mFirstPictureId; @@ -81,19 +69,10 @@ private: EOSStatus mEOSStatus; - enum OutputPortSettingChange { - NONE, - AWAITING_DISABLED, - AWAITING_ENABLED - }; - OutputPortSettingChange mOutputPortSettingsChange; - bool mSignalledError; - void initPorts(); status_t initDecoder(); - void updatePortDefinitions(); - bool drainAllOutputBuffers(); + void drainAllOutputBuffers(bool eos); void drainOneOutputBuffer(int32_t picId, uint8_t *data); void saveFirstOutputBuffer(int32_t pidId, uint8_t *data); bool handleCropRectEvent(const CropParams* crop); diff --git a/media/libstagefright/codecs/vorbis/dec/SoftVorbis.cpp b/media/libstagefright/codecs/vorbis/dec/SoftVorbis.cpp index 4115324..51bb958 100644 --- a/media/libstagefright/codecs/vorbis/dec/SoftVorbis.cpp +++ b/media/libstagefright/codecs/vorbis/dec/SoftVorbis.cpp @@ -424,6 +424,8 @@ void SoftVorbis::onReset() { delete mVi; mVi = NULL; } + + mOutputPortSettingsChange = NONE; } void SoftVorbis::onPortEnableCompleted(OMX_U32 portIndex, bool enabled) { diff --git a/media/libstagefright/foundation/AHierarchicalStateMachine.cpp b/media/libstagefright/foundation/AHierarchicalStateMachine.cpp index 40c5a3c..f7a00d8 100644 --- a/media/libstagefright/foundation/AHierarchicalStateMachine.cpp +++ b/media/libstagefright/foundation/AHierarchicalStateMachine.cpp @@ -14,6 +14,10 @@ * limitations under the License. */ +//#define LOG_NDEBUG 0 +#define LOG_TAG "AHierarchicalStateMachine" +#include <utils/Log.h> + #include <media/stagefright/foundation/AHierarchicalStateMachine.h> #include <media/stagefright/foundation/ADebug.h> diff --git a/media/libstagefright/foundation/ALooper.cpp b/media/libstagefright/foundation/ALooper.cpp index 22777a2..ebf9d8d 100644 --- a/media/libstagefright/foundation/ALooper.cpp +++ b/media/libstagefright/foundation/ALooper.cpp @@ -72,6 +72,10 @@ ALooper::ALooper() ALooper::~ALooper() { stop(); + + // Since this looper is "dead" (or as good as dead by now), + // have ALooperRoster unregister any handlers still registered for it. + gLooperRoster.unregisterStaleHandlers(); } void ALooper::setName(const char *name) { diff --git a/media/libstagefright/foundation/ALooperRoster.cpp b/media/libstagefright/foundation/ALooperRoster.cpp index ad10d2b..0c181ff 100644 --- a/media/libstagefright/foundation/ALooperRoster.cpp +++ b/media/libstagefright/foundation/ALooperRoster.cpp @@ -71,6 +71,20 @@ void ALooperRoster::unregisterHandler(ALooper::handler_id handlerID) { mHandlers.removeItemsAt(index); } +void ALooperRoster::unregisterStaleHandlers() { + Mutex::Autolock autoLock(mLock); + + for (size_t i = mHandlers.size(); i-- > 0;) { + const HandlerInfo &info = mHandlers.valueAt(i); + + sp<ALooper> looper = info.mLooper.promote(); + if (looper == NULL) { + ALOGV("Unregistering stale handler %d", mHandlers.keyAt(i)); + mHandlers.removeItemsAt(i); + } + } +} + status_t ALooperRoster::postMessage( const sp<AMessage> &msg, int64_t delayUs) { Mutex::Autolock autoLock(mLock); diff --git a/media/libstagefright/wifi-display/ANetworkSession.cpp b/media/libstagefright/foundation/ANetworkSession.cpp index 938d601..e629588 100644 --- a/media/libstagefright/wifi-display/ANetworkSession.cpp +++ b/media/libstagefright/foundation/ANetworkSession.cpp @@ -34,10 +34,21 @@ #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> #include <media/stagefright/foundation/hexdump.h> -#include <media/stagefright/Utils.h> namespace android { +static uint16_t U16_AT(const uint8_t *ptr) { + return ptr[0] << 8 | ptr[1]; +} + +static uint32_t U32_AT(const uint8_t *ptr) { + return ptr[0] << 24 | ptr[1] << 16 | ptr[2] << 8 | ptr[3]; +} + +static uint64_t U64_AT(const uint8_t *ptr) { + return ((uint64_t)U32_AT(ptr)) << 32 | U32_AT(ptr + 4); +} + static const size_t kMaxUDPSize = 1500; static const int32_t kMaxUDPRetries = 200; @@ -56,6 +67,12 @@ private: }; struct ANetworkSession::Session : public RefBase { + enum Mode { + MODE_RTSP, + MODE_DATAGRAM, + MODE_WEBSOCKET, + }; + enum State { CONNECTING, CONNECTED, @@ -85,7 +102,9 @@ struct ANetworkSession::Session : public RefBase { status_t sendRequest( const void *data, ssize_t size, bool timeValid, int64_t timeUs); - void setIsRTSPConnection(bool yesno); + void setMode(Mode mode); + + status_t switchToWebSocketMode(); protected: virtual ~Session(); @@ -102,7 +121,7 @@ private: int32_t mSessionID; State mState; - bool mIsRTSPConnection; + Mode mMode; int mSocket; sp<AMessage> mNotify; bool mSawReceiveFailure, mSawSendFailure; @@ -145,7 +164,7 @@ ANetworkSession::Session::Session( const sp<AMessage> ¬ify) : mSessionID(sessionID), mState(state), - mIsRTSPConnection(false), + mMode(MODE_DATAGRAM), mSocket(s), mNotify(notify), mSawReceiveFailure(false), @@ -209,8 +228,18 @@ int ANetworkSession::Session::socket() const { return mSocket; } -void ANetworkSession::Session::setIsRTSPConnection(bool yesno) { - mIsRTSPConnection = yesno; +void ANetworkSession::Session::setMode(Mode mode) { + mMode = mode; +} + +status_t ANetworkSession::Session::switchToWebSocketMode() { + if (mState != CONNECTED || mMode != MODE_RTSP) { + return INVALID_OPERATION; + } + + mMode = MODE_WEBSOCKET; + + return OK; } sp<AMessage> ANetworkSession::Session::getNotificationMessage() const { @@ -238,6 +267,8 @@ bool ANetworkSession::Session::wantsToWrite() { status_t ANetworkSession::Session::readMore() { if (mState == DATAGRAM) { + CHECK_EQ(mMode, MODE_DATAGRAM); + status_t err; do { sp<ABuffer> buf = new ABuffer(kMaxUDPSize); @@ -326,7 +357,7 @@ status_t ANetworkSession::Session::readMore() { err = -ECONNRESET; } - if (!mIsRTSPConnection) { + if (mMode == MODE_DATAGRAM) { // TCP stream carrying 16-bit length-prefixed datagrams. while (mInBuffer.size() >= 2) { @@ -350,7 +381,7 @@ status_t ANetworkSession::Session::readMore() { mInBuffer.erase(0, packetSize + 2); } - } else { + } else if (mMode == MODE_RTSP) { for (;;) { size_t length; @@ -417,6 +448,69 @@ status_t ANetworkSession::Session::readMore() { break; } } + } else { + CHECK_EQ(mMode, MODE_WEBSOCKET); + + const uint8_t *data = (const uint8_t *)mInBuffer.c_str(); + // hexdump(data, mInBuffer.size()); + + while (mInBuffer.size() >= 2) { + size_t offset = 2; + + unsigned payloadLen = data[1] & 0x7f; + if (payloadLen == 126) { + if (offset + 2 > mInBuffer.size()) { + break; + } + + payloadLen = U16_AT(&data[offset]); + offset += 2; + } else if (payloadLen == 127) { + if (offset + 8 > mInBuffer.size()) { + break; + } + + payloadLen = U64_AT(&data[offset]); + offset += 8; + } + + uint32_t mask = 0; + if (data[1] & 0x80) { + // MASK==1 + if (offset + 4 > mInBuffer.size()) { + break; + } + + mask = U32_AT(&data[offset]); + offset += 4; + } + + if (offset + payloadLen > mInBuffer.size()) { + break; + } + + // We have the full message. + + sp<ABuffer> packet = new ABuffer(payloadLen); + memcpy(packet->data(), &data[offset], payloadLen); + + if (mask != 0) { + for (size_t i = 0; i < payloadLen; ++i) { + packet->data()[i] = + data[offset + i] + ^ ((mask >> (8 * (3 - (i % 4)))) & 0xff); + } + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("sessionID", mSessionID); + notify->setInt32("reason", kWhatWebSocketMessage); + notify->setBuffer("data", packet); + notify->setInt32("headerByte", data[0]); + notify->post(); + + mInBuffer.erase(0, offset + payloadLen); + } } if (err != OK) { @@ -608,13 +702,61 @@ status_t ANetworkSession::Session::sendRequest( sp<ABuffer> buffer; - if (mState == CONNECTED && !mIsRTSPConnection) { + if (mState == CONNECTED && mMode == MODE_DATAGRAM) { CHECK_LE(size, 65535); buffer = new ABuffer(size + 2); buffer->data()[0] = size >> 8; buffer->data()[1] = size & 0xff; memcpy(buffer->data() + 2, data, size); + } else if (mState == CONNECTED && mMode == MODE_WEBSOCKET) { + static const bool kUseMask = false; // Chromium doesn't like it. + + size_t numHeaderBytes = 2 + (kUseMask ? 4 : 0); + if (size > 65535) { + numHeaderBytes += 8; + } else if (size > 125) { + numHeaderBytes += 2; + } + + buffer = new ABuffer(numHeaderBytes + size); + buffer->data()[0] = 0x81; // FIN==1 | opcode=1 (text) + buffer->data()[1] = kUseMask ? 0x80 : 0x00; + + if (size > 65535) { + buffer->data()[1] |= 127; + buffer->data()[2] = 0x00; + buffer->data()[3] = 0x00; + buffer->data()[4] = 0x00; + buffer->data()[5] = 0x00; + buffer->data()[6] = (size >> 24) & 0xff; + buffer->data()[7] = (size >> 16) & 0xff; + buffer->data()[8] = (size >> 8) & 0xff; + buffer->data()[9] = size & 0xff; + } else if (size > 125) { + buffer->data()[1] |= 126; + buffer->data()[2] = (size >> 8) & 0xff; + buffer->data()[3] = size & 0xff; + } else { + buffer->data()[1] |= size; + } + + if (kUseMask) { + uint32_t mask = rand(); + + buffer->data()[numHeaderBytes - 4] = (mask >> 24) & 0xff; + buffer->data()[numHeaderBytes - 3] = (mask >> 16) & 0xff; + buffer->data()[numHeaderBytes - 2] = (mask >> 8) & 0xff; + buffer->data()[numHeaderBytes - 1] = mask & 0xff; + + for (size_t i = 0; i < (size_t)size; ++i) { + buffer->data()[numHeaderBytes + i] = + ((const uint8_t *)data)[i] + ^ ((mask >> (8 * (3 - (i % 4)))) & 0xff); + } + } else { + memcpy(buffer->data() + numHeaderBytes, data, size); + } } else { buffer = new ABuffer(size); memcpy(buffer->data(), data, size); @@ -1001,9 +1143,9 @@ status_t ANetworkSession::createClientOrServer( notify); if (mode == kModeCreateTCPDatagramSessionActive) { - session->setIsRTSPConnection(false); + session->setMode(Session::MODE_DATAGRAM); } else if (mode == kModeCreateRTSPClient) { - session->setIsRTSPConnection(true); + session->setMode(Session::MODE_RTSP); } mSessions.add(session->sessionID(), session); @@ -1080,6 +1222,19 @@ status_t ANetworkSession::sendRequest( return err; } +status_t ANetworkSession::switchToWebSocketMode(int32_t sessionID) { + Mutex::Autolock autoLock(mLock); + + ssize_t index = mSessions.indexOfKey(sessionID); + + if (index < 0) { + return -ENOENT; + } + + const sp<Session> session = mSessions.valueAt(index); + return session->switchToWebSocketMode(); +} + void ANetworkSession::interrupt() { static const char dummy = 0; @@ -1213,8 +1368,10 @@ void ANetworkSession::threadLoop() { clientSocket, session->getNotificationMessage()); - clientSession->setIsRTSPConnection( - session->isRTSPServer()); + clientSession->setMode( + session->isRTSPServer() + ? Session::MODE_RTSP + : Session::MODE_DATAGRAM); sessionsToAdd.push_back(clientSession); } diff --git a/media/libstagefright/foundation/Android.mk b/media/libstagefright/foundation/Android.mk index d65e213..ad2dab5 100644 --- a/media/libstagefright/foundation/Android.mk +++ b/media/libstagefright/foundation/Android.mk @@ -10,7 +10,9 @@ LOCAL_SRC_FILES:= \ ALooper.cpp \ ALooperRoster.cpp \ AMessage.cpp \ + ANetworkSession.cpp \ AString.cpp \ + ParsedMessage.cpp \ base64.cpp \ hexdump.cpp diff --git a/media/libstagefright/wifi-display/ParsedMessage.cpp b/media/libstagefright/foundation/ParsedMessage.cpp index c0e60c3..049c9ad 100644 --- a/media/libstagefright/wifi-display/ParsedMessage.cpp +++ b/media/libstagefright/foundation/ParsedMessage.cpp @@ -19,6 +19,7 @@ #include <ctype.h> #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/hexdump.h> namespace android { @@ -89,6 +90,7 @@ ssize_t ParsedMessage::parse(const char *data, size_t size, bool noMoreData) { ssize_t lastDictIndex = -1; size_t offset = 0; + bool headersComplete = false; while (offset < size) { size_t lineEndOffset = offset; while (lineEndOffset + 1 < size @@ -113,6 +115,8 @@ ssize_t ParsedMessage::parse(const char *data, size_t size, bool noMoreData) { } if (lineEndOffset == offset) { + // An empty line separates headers from body. + headersComplete = true; offset += 2; break; } @@ -146,12 +150,17 @@ ssize_t ParsedMessage::parse(const char *data, size_t size, bool noMoreData) { offset = lineEndOffset + 2; } + if (!headersComplete && (!noMoreData || offset == 0)) { + // We either saw the empty line separating headers from body + // or we saw at least the status line and know that no more data + // is going to follow. + return -1; + } + for (size_t i = 0; i < mDict.size(); ++i) { mDict.editValueAt(i).trim(); } - // Found the end of headers. - int32_t contentLength; if (!findInt32("content-length", &contentLength) || contentLength < 0) { contentLength = 0; @@ -168,13 +177,17 @@ ssize_t ParsedMessage::parse(const char *data, size_t size, bool noMoreData) { return totalLength; } -void ParsedMessage::getRequestField(size_t index, AString *field) const { +bool ParsedMessage::getRequestField(size_t index, AString *field) const { AString line; CHECK(findString("_", &line)); size_t prevOffset = 0; size_t offset = 0; for (size_t i = 0; i <= index; ++i) { + if (offset >= line.size()) { + return false; + } + ssize_t spacePos = line.find(" ", offset); if (spacePos < 0) { @@ -186,11 +199,16 @@ void ParsedMessage::getRequestField(size_t index, AString *field) const { } field->setTo(line, prevOffset, offset - prevOffset - 1); + + return true; } bool ParsedMessage::getStatusCode(int32_t *statusCode) const { AString statusCodeString; - getRequestField(1, &statusCodeString); + if (!getRequestField(1, &statusCodeString)) { + *statusCode = 0; + return false; + } char *end; *statusCode = strtol(statusCodeString.c_str(), &end, 10); diff --git a/media/libstagefright/httplive/Android.mk b/media/libstagefright/httplive/Android.mk index a3fa7a3..f3529f9 100644 --- a/media/libstagefright/httplive/Android.mk +++ b/media/libstagefright/httplive/Android.mk @@ -6,16 +6,26 @@ LOCAL_SRC_FILES:= \ LiveDataSource.cpp \ LiveSession.cpp \ M3UParser.cpp \ + PlaylistFetcher.cpp \ LOCAL_C_INCLUDES:= \ $(TOP)/frameworks/av/media/libstagefright \ $(TOP)/frameworks/native/include/media/openmax \ $(TOP)/external/openssl/include +LOCAL_SHARED_LIBRARIES := \ + libbinder \ + libcrypto \ + libcutils \ + libmedia \ + libstagefright \ + libstagefright_foundation \ + libutils \ + LOCAL_MODULE:= libstagefright_httplive ifeq ($(TARGET_ARCH),arm) LOCAL_CFLAGS += -Wno-psabi endif -include $(BUILD_STATIC_LIBRARY) +include $(BUILD_SHARED_LIBRARY) diff --git a/media/libstagefright/httplive/LiveSession.cpp b/media/libstagefright/httplive/LiveSession.cpp index 505bdb3..bd12ddc 100644 --- a/media/libstagefright/httplive/LiveSession.cpp +++ b/media/libstagefright/httplive/LiveSession.cpp @@ -18,12 +18,13 @@ #define LOG_TAG "LiveSession" #include <utils/Log.h> -#include "include/LiveSession.h" +#include "LiveSession.h" -#include "LiveDataSource.h" +#include "M3UParser.h" +#include "PlaylistFetcher.h" -#include "include/M3UParser.h" #include "include/HTTPBase.h" +#include "mpeg2ts/AnotherPacketSource.h" #include <cutils/properties.h> #include <media/stagefright/foundation/hexdump.h> @@ -33,6 +34,8 @@ #include <media/stagefright/DataSource.h> #include <media/stagefright/FileSource.h> #include <media/stagefright/MediaErrors.h> +#include <media/stagefright/MetaData.h> +#include <media/stagefright/Utils.h> #include <ctype.h> #include <openssl/aes.h> @@ -47,37 +50,115 @@ LiveSession::LiveSession( mUIDValid(uidValid), mUID(uid), mInPreparationPhase(true), - mDataSource(new LiveDataSource), mHTTPDataSource( HTTPBase::Create( (mFlags & kFlagIncognito) ? HTTPBase::kFlagIncognito : 0)), mPrevBandwidthIndex(-1), - mLastPlaylistFetchTimeUs(-1), - mSeqNumber(-1), - mSeekTimeUs(-1), - mNumRetries(0), - mStartOfPlayback(true), - mDurationUs(-1), - mDurationFixed(false), - mSeekDone(false), - mDisconnectPending(false), - mMonitorQueueGeneration(0), - mRefreshState(INITIAL_MINIMUM_RELOAD_DELAY) { + mStreamMask(0), + mCheckBandwidthGeneration(0), + mLastDequeuedTimeUs(0ll), + mRealTimeBaseUs(0ll), + mReconfigurationInProgress(false), + mDisconnectReplyID(0) { if (mUIDValid) { mHTTPDataSource->setUID(mUID); } + + mPacketSources.add( + STREAMTYPE_AUDIO, new AnotherPacketSource(NULL /* meta */)); + + mPacketSources.add( + STREAMTYPE_VIDEO, new AnotherPacketSource(NULL /* meta */)); + + mPacketSources.add( + STREAMTYPE_SUBTITLES, new AnotherPacketSource(NULL /* meta */)); } LiveSession::~LiveSession() { } -sp<DataSource> LiveSession::getDataSource() { - return mDataSource; +status_t LiveSession::dequeueAccessUnit( + StreamType stream, sp<ABuffer> *accessUnit) { + if (!(mStreamMask & stream)) { + return UNKNOWN_ERROR; + } + + sp<AnotherPacketSource> packetSource = mPacketSources.valueFor(stream); + + status_t finalResult; + if (!packetSource->hasBufferAvailable(&finalResult)) { + return finalResult == OK ? -EAGAIN : finalResult; + } + + status_t err = packetSource->dequeueAccessUnit(accessUnit); + + const char *streamStr; + switch (stream) { + case STREAMTYPE_AUDIO: + streamStr = "audio"; + break; + case STREAMTYPE_VIDEO: + streamStr = "video"; + break; + case STREAMTYPE_SUBTITLES: + streamStr = "subs"; + break; + default: + TRESPASS(); + } + + if (err == INFO_DISCONTINUITY) { + int32_t type; + CHECK((*accessUnit)->meta()->findInt32("discontinuity", &type)); + + sp<AMessage> extra; + if (!(*accessUnit)->meta()->findMessage("extra", &extra)) { + extra.clear(); + } + + ALOGI("[%s] read discontinuity of type %d, extra = %s", + streamStr, + type, + extra == NULL ? "NULL" : extra->debugString().c_str()); + } else if (err == OK) { + if (stream == STREAMTYPE_AUDIO || stream == STREAMTYPE_VIDEO) { + int64_t timeUs; + CHECK((*accessUnit)->meta()->findInt64("timeUs", &timeUs)); + ALOGV("[%s] read buffer at time %lld us", streamStr, timeUs); + + mLastDequeuedTimeUs = timeUs; + mRealTimeBaseUs = ALooper::GetNowUs() - timeUs; + } else if (stream == STREAMTYPE_SUBTITLES) { + (*accessUnit)->meta()->setInt32( + "trackIndex", mPlaylist->getSelectedIndex()); + (*accessUnit)->meta()->setInt64("baseUs", mRealTimeBaseUs); + } + } else { + ALOGI("[%s] encountered error %d", streamStr, err); + } + + return err; +} + +status_t LiveSession::getStreamFormat(StreamType stream, sp<AMessage> *format) { + if (!(mStreamMask & stream)) { + return UNKNOWN_ERROR; + } + + sp<AnotherPacketSource> packetSource = mPacketSources.valueFor(stream); + + sp<MetaData> meta = packetSource->getFormat(); + + if (meta == NULL) { + return -EAGAIN; + } + + return convertMetaDataToMessage(meta, format); } -void LiveSession::connect( +void LiveSession::connectAsync( const char *url, const KeyedVector<String8, String8> *headers) { sp<AMessage> msg = new AMessage(kWhatConnect, id()); msg->setString("url", url); @@ -91,55 +172,190 @@ void LiveSession::connect( msg->post(); } -void LiveSession::disconnect() { - Mutex::Autolock autoLock(mLock); - mDisconnectPending = true; +status_t LiveSession::disconnect() { + sp<AMessage> msg = new AMessage(kWhatDisconnect, id()); - mHTTPDataSource->disconnect(); + sp<AMessage> response; + status_t err = msg->postAndAwaitResponse(&response); - (new AMessage(kWhatDisconnect, id()))->post(); + return err; } -void LiveSession::seekTo(int64_t timeUs) { - Mutex::Autolock autoLock(mLock); - mSeekDone = false; - +status_t LiveSession::seekTo(int64_t timeUs) { sp<AMessage> msg = new AMessage(kWhatSeek, id()); msg->setInt64("timeUs", timeUs); - msg->post(); - while (!mSeekDone) { - mCondition.wait(mLock); - } + sp<AMessage> response; + status_t err = msg->postAndAwaitResponse(&response); + + return err; } void LiveSession::onMessageReceived(const sp<AMessage> &msg) { switch (msg->what()) { case kWhatConnect: + { onConnect(msg); break; + } case kWhatDisconnect: - onDisconnect(); + { + CHECK(msg->senderAwaitsResponse(&mDisconnectReplyID)); + + if (mReconfigurationInProgress) { + break; + } + + finishDisconnect(); break; + } + + case kWhatSeek: + { + uint32_t replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + + status_t err = onSeek(msg); + + sp<AMessage> response = new AMessage; + response->setInt32("err", err); + + response->postReply(replyID); + break; + } + + case kWhatFetcherNotify: + { + int32_t what; + CHECK(msg->findInt32("what", &what)); + + switch (what) { + case PlaylistFetcher::kWhatStarted: + break; + case PlaylistFetcher::kWhatPaused: + case PlaylistFetcher::kWhatStopped: + { + if (what == PlaylistFetcher::kWhatStopped) { + AString uri; + CHECK(msg->findString("uri", &uri)); + mFetcherInfos.removeItem(uri); + } + + if (mContinuation != NULL) { + CHECK_GT(mContinuationCounter, 0); + if (--mContinuationCounter == 0) { + mContinuation->post(); + } + } + break; + } + + case PlaylistFetcher::kWhatDurationUpdate: + { + AString uri; + CHECK(msg->findString("uri", &uri)); + + int64_t durationUs; + CHECK(msg->findInt64("durationUs", &durationUs)); + + FetcherInfo *info = &mFetcherInfos.editValueFor(uri); + info->mDurationUs = durationUs; + break; + } + + case PlaylistFetcher::kWhatError: + { + status_t err; + CHECK(msg->findInt32("err", &err)); + + ALOGE("XXX Received error %d from PlaylistFetcher.", err); - case kWhatMonitorQueue: + if (mInPreparationPhase) { + postPrepared(err); + } + + mPacketSources.valueFor(STREAMTYPE_AUDIO)->signalEOS(err); + + mPacketSources.valueFor(STREAMTYPE_VIDEO)->signalEOS(err); + + mPacketSources.valueFor( + STREAMTYPE_SUBTITLES)->signalEOS(err); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatError); + notify->setInt32("err", err); + notify->post(); + break; + } + + case PlaylistFetcher::kWhatTemporarilyDoneFetching: + { + AString uri; + CHECK(msg->findString("uri", &uri)); + + FetcherInfo *info = &mFetcherInfos.editValueFor(uri); + info->mIsPrepared = true; + + if (mInPreparationPhase) { + bool allFetchersPrepared = true; + for (size_t i = 0; i < mFetcherInfos.size(); ++i) { + if (!mFetcherInfos.valueAt(i).mIsPrepared) { + allFetchersPrepared = false; + break; + } + } + + if (allFetchersPrepared) { + postPrepared(OK); + } + } + break; + } + + default: + TRESPASS(); + } + + break; + } + + case kWhatCheckBandwidth: { int32_t generation; CHECK(msg->findInt32("generation", &generation)); - if (generation != mMonitorQueueGeneration) { - // Stale event + if (generation != mCheckBandwidthGeneration) { break; } - onMonitorQueue(); + onCheckBandwidth(); break; } - case kWhatSeek: - onSeek(msg); + case kWhatChangeConfiguration: + { + onChangeConfiguration(msg); + break; + } + + case kWhatChangeConfiguration2: + { + onChangeConfiguration2(msg); + break; + } + + case kWhatChangeConfiguration3: + { + onChangeConfiguration3(msg); break; + } + + case kWhatFinishDisconnect2: + { + onFinishDisconnect2(); + break; + } default: TRESPASS(); @@ -172,48 +388,128 @@ void LiveSession::onConnect(const sp<AMessage> &msg) { headers = NULL; } +#if 1 ALOGI("onConnect <URL suppressed>"); +#else + ALOGI("onConnect %s", url.c_str()); +#endif mMasterURL = url; bool dummy; - sp<M3UParser> playlist = fetchPlaylist(url.c_str(), &dummy); + mPlaylist = fetchPlaylist(url.c_str(), NULL /* curPlaylistHash */, &dummy); - if (playlist == NULL) { + if (mPlaylist == NULL) { ALOGE("unable to fetch master playlist '%s'.", url.c_str()); - signalEOS(ERROR_IO); + postPrepared(ERROR_IO); return; } - if (playlist->isVariantPlaylist()) { - for (size_t i = 0; i < playlist->size(); ++i) { + // We trust the content provider to make a reasonable choice of preferred + // initial bandwidth by listing it first in the variant playlist. + // At startup we really don't have a good estimate on the available + // network bandwidth since we haven't tranferred any data yet. Once + // we have we can make a better informed choice. + size_t initialBandwidth = 0; + size_t initialBandwidthIndex = 0; + + if (mPlaylist->isVariantPlaylist()) { + for (size_t i = 0; i < mPlaylist->size(); ++i) { BandwidthItem item; + item.mPlaylistIndex = i; + sp<AMessage> meta; - playlist->itemAt(i, &item.mURI, &meta); + AString uri; + mPlaylist->itemAt(i, &uri, &meta); unsigned long bandwidth; CHECK(meta->findInt32("bandwidth", (int32_t *)&item.mBandwidth)); + if (initialBandwidth == 0) { + initialBandwidth = item.mBandwidth; + } + mBandwidthItems.push(item); } CHECK_GT(mBandwidthItems.size(), 0u); mBandwidthItems.sort(SortByBandwidth); + + for (size_t i = 0; i < mBandwidthItems.size(); ++i) { + if (mBandwidthItems.itemAt(i).mBandwidth == initialBandwidth) { + initialBandwidthIndex = i; + break; + } + } + } else { + // dummy item. + BandwidthItem item; + item.mPlaylistIndex = 0; + item.mBandwidth = 0; + mBandwidthItems.push(item); + } + + changeConfiguration( + 0ll /* timeUs */, initialBandwidthIndex, true /* pickTrack */); +} + +void LiveSession::finishDisconnect() { + // No reconfiguration is currently pending, make sure none will trigger + // during disconnection either. + cancelCheckBandwidthEvent(); + + for (size_t i = 0; i < mFetcherInfos.size(); ++i) { + mFetcherInfos.valueAt(i).mFetcher->stopAsync(); } - postMonitorQueue(); + sp<AMessage> msg = new AMessage(kWhatFinishDisconnect2, id()); + + mContinuationCounter = mFetcherInfos.size(); + mContinuation = msg; + + if (mContinuationCounter == 0) { + msg->post(); + } +} + +void LiveSession::onFinishDisconnect2() { + mContinuation.clear(); + + mPacketSources.valueFor(STREAMTYPE_AUDIO)->signalEOS(ERROR_END_OF_STREAM); + mPacketSources.valueFor(STREAMTYPE_VIDEO)->signalEOS(ERROR_END_OF_STREAM); + + mPacketSources.valueFor( + STREAMTYPE_SUBTITLES)->signalEOS(ERROR_END_OF_STREAM); + + sp<AMessage> response = new AMessage; + response->setInt32("err", OK); + + response->postReply(mDisconnectReplyID); + mDisconnectReplyID = 0; } -void LiveSession::onDisconnect() { - ALOGI("onDisconnect"); +sp<PlaylistFetcher> LiveSession::addFetcher(const char *uri) { + ssize_t index = mFetcherInfos.indexOfKey(uri); + + if (index >= 0) { + return NULL; + } - signalEOS(ERROR_END_OF_STREAM); + sp<AMessage> notify = new AMessage(kWhatFetcherNotify, id()); + notify->setString("uri", uri); - Mutex::Autolock autoLock(mLock); - mDisconnectPending = false; + FetcherInfo info; + info.mFetcher = new PlaylistFetcher(notify, this, uri); + info.mDurationUs = -1ll; + info.mIsPrepared = false; + looper()->registerHandler(info.mFetcher); + + mFetcherInfos.add(uri, info); + + return info.mFetcher; } status_t LiveSession::fetchFile( @@ -229,14 +525,6 @@ status_t LiveSession::fetchFile( && strncasecmp(url, "https://", 8)) { return ERROR_UNSUPPORTED; } else { - { - Mutex::Autolock autoLock(mLock); - - if (mDisconnectPending) { - return ERROR_IO; - } - } - KeyedVector<String8, String8> headers = mExtraHeaders; if (range_offset > 0 || range_length >= 0) { headers.add( @@ -315,7 +603,8 @@ status_t LiveSession::fetchFile( return OK; } -sp<M3UParser> LiveSession::fetchPlaylist(const char *url, bool *unchanged) { +sp<M3UParser> LiveSession::fetchPlaylist( + const char *url, uint8_t *curPlaylistHash, bool *unchanged) { ALOGV("fetchPlaylist '%s'", url); *unchanged = false; @@ -339,13 +628,8 @@ sp<M3UParser> LiveSession::fetchPlaylist(const char *url, bool *unchanged) { MD5_Final(hash, &m); - if (mPlaylist != NULL && !memcmp(hash, mPlaylistHash, 16)) { + if (curPlaylistHash != NULL && !memcmp(hash, curPlaylistHash, 16)) { // playlist unchanged - - if (mRefreshState != THIRD_UNCHANGED_RELOAD_ATTEMPT) { - mRefreshState = (RefreshState)(mRefreshState + 1); - } - *unchanged = true; ALOGV("Playlist unchanged, refresh state is now %d", @@ -354,9 +638,9 @@ sp<M3UParser> LiveSession::fetchPlaylist(const char *url, bool *unchanged) { return NULL; } - memcpy(mPlaylistHash, hash, sizeof(hash)); - - mRefreshState = INITIAL_MINIMUM_RELOAD_DELAY; + if (curPlaylistHash != NULL) { + memcpy(curPlaylistHash, hash, sizeof(hash)); + } #endif sp<M3UParser> playlist = @@ -371,37 +655,6 @@ sp<M3UParser> LiveSession::fetchPlaylist(const char *url, bool *unchanged) { return playlist; } -int64_t LiveSession::getSegmentStartTimeUs(int32_t seqNumber) const { - CHECK(mPlaylist != NULL); - - int32_t firstSeqNumberInPlaylist; - if (mPlaylist->meta() == NULL || !mPlaylist->meta()->findInt32( - "media-sequence", &firstSeqNumberInPlaylist)) { - firstSeqNumberInPlaylist = 0; - } - - int32_t lastSeqNumberInPlaylist = - firstSeqNumberInPlaylist + (int32_t)mPlaylist->size() - 1; - - CHECK_GE(seqNumber, firstSeqNumberInPlaylist); - CHECK_LE(seqNumber, lastSeqNumberInPlaylist); - - int64_t segmentStartUs = 0ll; - for (int32_t index = 0; - index < seqNumber - firstSeqNumberInPlaylist; ++index) { - sp<AMessage> itemMeta; - CHECK(mPlaylist->itemAt( - index, NULL /* uri */, &itemMeta)); - - int64_t itemDurationUs; - CHECK(itemMeta->findInt64("durationUs", &itemDurationUs)); - - segmentStartUs += itemDurationUs; - } - - return segmentStartUs; -} - static double uniformRand() { return (double)rand() / RAND_MAX; } @@ -412,36 +665,50 @@ size_t LiveSession::getBandwidthIndex() { } #if 1 - int32_t bandwidthBps; - if (mHTTPDataSource != NULL - && mHTTPDataSource->estimateBandwidth(&bandwidthBps)) { - ALOGV("bandwidth estimated at %.2f kbps", bandwidthBps / 1024.0f); - } else { - ALOGV("no bandwidth estimate."); - return 0; // Pick the lowest bandwidth stream by default. - } - char value[PROPERTY_VALUE_MAX]; - if (property_get("media.httplive.max-bw", value, NULL)) { + ssize_t index = -1; + if (property_get("media.httplive.bw-index", value, NULL)) { char *end; - long maxBw = strtoul(value, &end, 10); - if (end > value && *end == '\0') { - if (maxBw > 0 && bandwidthBps > maxBw) { - ALOGV("bandwidth capped to %ld bps", maxBw); - bandwidthBps = maxBw; - } + index = strtol(value, &end, 10); + CHECK(end > value && *end == '\0'); + + if (index >= 0 && (size_t)index >= mBandwidthItems.size()) { + index = mBandwidthItems.size() - 1; } } - // Consider only 80% of the available bandwidth usable. - bandwidthBps = (bandwidthBps * 8) / 10; + if (index < 0) { + int32_t bandwidthBps; + if (mHTTPDataSource != NULL + && mHTTPDataSource->estimateBandwidth(&bandwidthBps)) { + ALOGV("bandwidth estimated at %.2f kbps", bandwidthBps / 1024.0f); + } else { + ALOGV("no bandwidth estimate."); + return 0; // Pick the lowest bandwidth stream by default. + } - // Pick the highest bandwidth stream below or equal to estimated bandwidth. + char value[PROPERTY_VALUE_MAX]; + if (property_get("media.httplive.max-bw", value, NULL)) { + char *end; + long maxBw = strtoul(value, &end, 10); + if (end > value && *end == '\0') { + if (maxBw > 0 && bandwidthBps > maxBw) { + ALOGV("bandwidth capped to %ld bps", maxBw); + bandwidthBps = maxBw; + } + } + } - size_t index = mBandwidthItems.size() - 1; - while (index > 0 && mBandwidthItems.itemAt(index).mBandwidth - > (size_t)bandwidthBps) { - --index; + // Consider only 80% of the available bandwidth usable. + bandwidthBps = (bandwidthBps * 8) / 10; + + // Pick the highest bandwidth stream below or equal to estimated bandwidth. + + index = mBandwidthItems.size() - 1; + while (index > 0 && mBandwidthItems.itemAt(index).mBandwidth + > (size_t)bandwidthBps) { + --index; + } } #elif 0 // Change bandwidth at random() @@ -452,6 +719,8 @@ size_t LiveSession::getBandwidthIndex() { // to lowest) const size_t kMinIndex = 0; + static ssize_t mPrevBandwidthIndex = -1; + size_t index; if (mPrevBandwidthIndex < 0) { index = kMinIndex; @@ -463,6 +732,7 @@ size_t LiveSession::getBandwidthIndex() { index = kMinIndex; } } + mPrevBandwidthIndex = index; #elif 0 // Pick the highest bandwidth stream below or equal to 1.2 Mbit/sec @@ -470,570 +740,405 @@ size_t LiveSession::getBandwidthIndex() { while (index > 0 && mBandwidthItems.itemAt(index).mBandwidth > 1200000) { --index; } +#elif 1 + char value[PROPERTY_VALUE_MAX]; + size_t index; + if (property_get("media.httplive.bw-index", value, NULL)) { + char *end; + index = strtoul(value, &end, 10); + CHECK(end > value && *end == '\0'); + + if (index >= mBandwidthItems.size()) { + index = mBandwidthItems.size() - 1; + } + } else { + index = 0; + } #else size_t index = mBandwidthItems.size() - 1; // Highest bandwidth stream #endif + CHECK_GE(index, 0); + return index; } -bool LiveSession::timeToRefreshPlaylist(int64_t nowUs) const { - if (mPlaylist == NULL) { - CHECK_EQ((int)mRefreshState, (int)INITIAL_MINIMUM_RELOAD_DELAY); - return true; - } - - int32_t targetDurationSecs; - CHECK(mPlaylist->meta()->findInt32("target-duration", &targetDurationSecs)); - - int64_t targetDurationUs = targetDurationSecs * 1000000ll; - - int64_t minPlaylistAgeUs; - - switch (mRefreshState) { - case INITIAL_MINIMUM_RELOAD_DELAY: - { - size_t n = mPlaylist->size(); - if (n > 0) { - sp<AMessage> itemMeta; - CHECK(mPlaylist->itemAt(n - 1, NULL /* uri */, &itemMeta)); - - int64_t itemDurationUs; - CHECK(itemMeta->findInt64("durationUs", &itemDurationUs)); - - minPlaylistAgeUs = itemDurationUs; - break; - } - - // fall through - } - - case FIRST_UNCHANGED_RELOAD_ATTEMPT: - { - minPlaylistAgeUs = targetDurationUs / 2; - break; - } - - case SECOND_UNCHANGED_RELOAD_ATTEMPT: - { - minPlaylistAgeUs = (targetDurationUs * 3) / 2; - break; - } - - case THIRD_UNCHANGED_RELOAD_ATTEMPT: - { - minPlaylistAgeUs = targetDurationUs * 3; - break; - } +status_t LiveSession::onSeek(const sp<AMessage> &msg) { + int64_t timeUs; + CHECK(msg->findInt64("timeUs", &timeUs)); - default: - TRESPASS(); - break; + if (!mReconfigurationInProgress) { + changeConfiguration(timeUs, getBandwidthIndex()); } - return mLastPlaylistFetchTimeUs + minPlaylistAgeUs <= nowUs; + return OK; } -void LiveSession::onDownloadNext() { - size_t bandwidthIndex = getBandwidthIndex(); - -rinse_repeat: - int64_t nowUs = ALooper::GetNowUs(); - - if (mLastPlaylistFetchTimeUs < 0 - || (ssize_t)bandwidthIndex != mPrevBandwidthIndex - || (!mPlaylist->isComplete() && timeToRefreshPlaylist(nowUs))) { - AString url; - if (mBandwidthItems.size() > 0) { - url = mBandwidthItems.editItemAt(bandwidthIndex).mURI; - } else { - url = mMasterURL; - } - - if ((ssize_t)bandwidthIndex != mPrevBandwidthIndex) { - // If we switch bandwidths, do not pay any heed to whether - // playlists changed since the last time... - mPlaylist.clear(); - } - - bool unchanged; - sp<M3UParser> playlist = fetchPlaylist(url.c_str(), &unchanged); - if (playlist == NULL) { - if (unchanged) { - // We succeeded in fetching the playlist, but it was - // unchanged from the last time we tried. - } else { - ALOGE("failed to load playlist at url '%s'", url.c_str()); - signalEOS(ERROR_IO); - - return; - } - } else { - mPlaylist = playlist; - } - - if (!mDurationFixed) { - Mutex::Autolock autoLock(mLock); - - if (!mPlaylist->isComplete() && !mPlaylist->isEvent()) { - mDurationUs = -1; - mDurationFixed = true; - } else { - mDurationUs = 0; - for (size_t i = 0; i < mPlaylist->size(); ++i) { - sp<AMessage> itemMeta; - CHECK(mPlaylist->itemAt( - i, NULL /* uri */, &itemMeta)); - - int64_t itemDurationUs; - CHECK(itemMeta->findInt64("durationUs", &itemDurationUs)); - - mDurationUs += itemDurationUs; - } +status_t LiveSession::getDuration(int64_t *durationUs) const { + int64_t maxDurationUs = 0ll; + for (size_t i = 0; i < mFetcherInfos.size(); ++i) { + int64_t fetcherDurationUs = mFetcherInfos.valueAt(i).mDurationUs; - mDurationFixed = mPlaylist->isComplete(); - } + if (fetcherDurationUs >= 0ll && fetcherDurationUs > maxDurationUs) { + maxDurationUs = fetcherDurationUs; } - - mLastPlaylistFetchTimeUs = ALooper::GetNowUs(); } - int32_t firstSeqNumberInPlaylist; - if (mPlaylist->meta() == NULL || !mPlaylist->meta()->findInt32( - "media-sequence", &firstSeqNumberInPlaylist)) { - firstSeqNumberInPlaylist = 0; - } + *durationUs = maxDurationUs; - bool seekDiscontinuity = false; - bool explicitDiscontinuity = false; - bool bandwidthChanged = false; + return OK; +} - if (mSeekTimeUs >= 0) { - if (mPlaylist->isComplete() || mPlaylist->isEvent()) { - size_t index = 0; - int64_t segmentStartUs = 0; - while (index < mPlaylist->size()) { - sp<AMessage> itemMeta; - CHECK(mPlaylist->itemAt( - index, NULL /* uri */, &itemMeta)); +bool LiveSession::isSeekable() const { + int64_t durationUs; + return getDuration(&durationUs) == OK && durationUs >= 0; +} - int64_t itemDurationUs; - CHECK(itemMeta->findInt64("durationUs", &itemDurationUs)); +bool LiveSession::hasDynamicDuration() const { + return false; +} - if (mSeekTimeUs < segmentStartUs + itemDurationUs) { - break; - } +status_t LiveSession::getTrackInfo(Parcel *reply) const { + return mPlaylist->getTrackInfo(reply); +} - segmentStartUs += itemDurationUs; - ++index; - } +status_t LiveSession::selectTrack(size_t index, bool select) { + status_t err = mPlaylist->selectTrack(index, select); + if (err == OK) { + (new AMessage(kWhatChangeConfiguration, id()))->post(); + } + return err; +} - if (index < mPlaylist->size()) { - int32_t newSeqNumber = firstSeqNumberInPlaylist + index; +void LiveSession::changeConfiguration( + int64_t timeUs, size_t bandwidthIndex, bool pickTrack) { + CHECK(!mReconfigurationInProgress); + mReconfigurationInProgress = true; - ALOGI("seeking to seq no %d", newSeqNumber); + mPrevBandwidthIndex = bandwidthIndex; - mSeqNumber = newSeqNumber; + ALOGV("changeConfiguration => timeUs:%lld us, bwIndex:%d, pickTrack:%d", + timeUs, bandwidthIndex, pickTrack); - mDataSource->reset(); + if (pickTrack) { + mPlaylist->pickRandomMediaItems(); + } - // reseting the data source will have had the - // side effect of discarding any previously queued - // bandwidth change discontinuity. - // Therefore we'll need to treat these seek - // discontinuities as involving a bandwidth change - // even if they aren't directly. - seekDiscontinuity = true; - bandwidthChanged = true; - } - } + CHECK_LT(bandwidthIndex, mBandwidthItems.size()); + const BandwidthItem &item = mBandwidthItems.itemAt(bandwidthIndex); - mSeekTimeUs = -1; + uint32_t streamMask = 0; - Mutex::Autolock autoLock(mLock); - mSeekDone = true; - mCondition.broadcast(); + AString audioURI; + if (mPlaylist->getAudioURI(item.mPlaylistIndex, &audioURI)) { + streamMask |= STREAMTYPE_AUDIO; } - const int32_t lastSeqNumberInPlaylist = - firstSeqNumberInPlaylist + (int32_t)mPlaylist->size() - 1; - - if (mSeqNumber < 0) { - if (mPlaylist->isComplete()) { - mSeqNumber = firstSeqNumberInPlaylist; - } else { - // If this is a live session, start 3 segments from the end. - mSeqNumber = lastSeqNumberInPlaylist - 3; - if (mSeqNumber < firstSeqNumberInPlaylist) { - mSeqNumber = firstSeqNumberInPlaylist; - } - } + AString videoURI; + if (mPlaylist->getVideoURI(item.mPlaylistIndex, &videoURI)) { + streamMask |= STREAMTYPE_VIDEO; } - if (mSeqNumber < firstSeqNumberInPlaylist - || mSeqNumber > lastSeqNumberInPlaylist) { - if (mPrevBandwidthIndex != (ssize_t)bandwidthIndex) { - // Go back to the previous bandwidth. - - ALOGI("new bandwidth does not have the sequence number " - "we're looking for, switching back to previous bandwidth"); + AString subtitleURI; + if (mPlaylist->getSubtitleURI(item.mPlaylistIndex, &subtitleURI)) { + streamMask |= STREAMTYPE_SUBTITLES; + } - mLastPlaylistFetchTimeUs = -1; - bandwidthIndex = mPrevBandwidthIndex; - goto rinse_repeat; - } + // Step 1, stop and discard fetchers that are no longer needed. + // Pause those that we'll reuse. + for (size_t i = 0; i < mFetcherInfos.size(); ++i) { + const AString &uri = mFetcherInfos.keyAt(i); - if (!mPlaylist->isComplete() && mNumRetries < kMaxNumRetries) { - ++mNumRetries; + bool discardFetcher = true; - if (mSeqNumber > lastSeqNumberInPlaylist) { - mLastPlaylistFetchTimeUs = -1; - postMonitorQueue(3000000ll); - return; + // If we're seeking all current fetchers are discarded. + if (timeUs < 0ll) { + if (((streamMask & STREAMTYPE_AUDIO) && uri == audioURI) + || ((streamMask & STREAMTYPE_VIDEO) && uri == videoURI) + || ((streamMask & STREAMTYPE_SUBTITLES) && uri == subtitleURI)) { + discardFetcher = false; } + } - // we've missed the boat, let's start from the lowest sequence - // number available and signal a discontinuity. - - ALOGI("We've missed the boat, restarting playback."); - mSeqNumber = lastSeqNumberInPlaylist; - explicitDiscontinuity = true; - - // fall through + if (discardFetcher) { + mFetcherInfos.valueAt(i).mFetcher->stopAsync(); } else { - ALOGE("Cannot find sequence number %d in playlist " - "(contains %d - %d)", - mSeqNumber, firstSeqNumberInPlaylist, - firstSeqNumberInPlaylist + mPlaylist->size() - 1); - - signalEOS(ERROR_END_OF_STREAM); - return; + mFetcherInfos.valueAt(i).mFetcher->pauseAsync(); } } - mNumRetries = 0; - - AString uri; - sp<AMessage> itemMeta; - CHECK(mPlaylist->itemAt( - mSeqNumber - firstSeqNumberInPlaylist, - &uri, - &itemMeta)); - - int32_t val; - if (itemMeta->findInt32("discontinuity", &val) && val != 0) { - explicitDiscontinuity = true; + sp<AMessage> msg = new AMessage(kWhatChangeConfiguration2, id()); + msg->setInt32("streamMask", streamMask); + msg->setInt64("timeUs", timeUs); + if (streamMask & STREAMTYPE_AUDIO) { + msg->setString("audioURI", audioURI.c_str()); } - - int64_t range_offset, range_length; - if (!itemMeta->findInt64("range-offset", &range_offset) - || !itemMeta->findInt64("range-length", &range_length)) { - range_offset = 0; - range_length = -1; + if (streamMask & STREAMTYPE_VIDEO) { + msg->setString("videoURI", videoURI.c_str()); } - - ALOGV("fetching segment %d from (%d .. %d)", - mSeqNumber, firstSeqNumberInPlaylist, lastSeqNumberInPlaylist); - - sp<ABuffer> buffer; - status_t err = fetchFile(uri.c_str(), &buffer, range_offset, range_length); - if (err != OK) { - ALOGE("failed to fetch .ts segment at url '%s'", uri.c_str()); - signalEOS(err); - return; + if (streamMask & STREAMTYPE_SUBTITLES) { + msg->setString("subtitleURI", subtitleURI.c_str()); } - CHECK(buffer != NULL); - - err = decryptBuffer(mSeqNumber - firstSeqNumberInPlaylist, buffer); + // Every time a fetcher acknowledges the stopAsync or pauseAsync request + // we'll decrement mContinuationCounter, once it reaches zero, i.e. all + // fetchers have completed their asynchronous operation, we'll post + // mContinuation, which then is handled below in onChangeConfiguration2. + mContinuationCounter = mFetcherInfos.size(); + mContinuation = msg; - if (err != OK) { - ALOGE("decryptBuffer failed w/ error %d", err); - - signalEOS(err); - return; + if (mContinuationCounter == 0) { + msg->post(); } +} - if (buffer->size() == 0 || buffer->data()[0] != 0x47) { - // Not a transport stream??? - - ALOGE("This doesn't look like a transport stream..."); - - mBandwidthItems.removeAt(bandwidthIndex); +void LiveSession::onChangeConfiguration(const sp<AMessage> &msg) { + if (!mReconfigurationInProgress) { + changeConfiguration(-1ll /* timeUs */, getBandwidthIndex()); + } else { + msg->post(1000000ll); // retry in 1 sec + } +} - if (mBandwidthItems.isEmpty()) { - signalEOS(ERROR_UNSUPPORTED); - return; - } +void LiveSession::onChangeConfiguration2(const sp<AMessage> &msg) { + mContinuation.clear(); - ALOGI("Retrying with a different bandwidth stream."); + // All fetchers are either suspended or have been removed now. - mLastPlaylistFetchTimeUs = -1; - bandwidthIndex = getBandwidthIndex(); - mPrevBandwidthIndex = bandwidthIndex; - mSeqNumber = -1; + uint32_t streamMask; + CHECK(msg->findInt32("streamMask", (int32_t *)&streamMask)); - goto rinse_repeat; + AString audioURI, videoURI, subtitleURI; + if (streamMask & STREAMTYPE_AUDIO) { + CHECK(msg->findString("audioURI", &audioURI)); + ALOGV("audioURI = '%s'", audioURI.c_str()); } - - if ((size_t)mPrevBandwidthIndex != bandwidthIndex) { - bandwidthChanged = true; + if (streamMask & STREAMTYPE_VIDEO) { + CHECK(msg->findString("videoURI", &videoURI)); + ALOGV("videoURI = '%s'", videoURI.c_str()); } - - if (mPrevBandwidthIndex < 0) { - // Don't signal a bandwidth change at the very beginning of - // playback. - bandwidthChanged = false; + if (streamMask & STREAMTYPE_SUBTITLES) { + CHECK(msg->findString("subtitleURI", &subtitleURI)); + ALOGV("subtitleURI = '%s'", subtitleURI.c_str()); } - if (mStartOfPlayback) { - seekDiscontinuity = true; - mStartOfPlayback = false; + // Determine which decoders to shutdown on the player side, + // a decoder has to be shutdown if either + // 1) its streamtype was active before but now longer isn't. + // or + // 2) its streamtype was already active and still is but the URI + // has changed. + uint32_t changedMask = 0; + if (((mStreamMask & streamMask & STREAMTYPE_AUDIO) + && !(audioURI == mAudioURI)) + || (mStreamMask & ~streamMask & STREAMTYPE_AUDIO)) { + changedMask |= STREAMTYPE_AUDIO; + } + if (((mStreamMask & streamMask & STREAMTYPE_VIDEO) + && !(videoURI == mVideoURI)) + || (mStreamMask & ~streamMask & STREAMTYPE_VIDEO)) { + changedMask |= STREAMTYPE_VIDEO; } - if (seekDiscontinuity || explicitDiscontinuity || bandwidthChanged) { - // Signal discontinuity. - - ALOGI("queueing discontinuity (seek=%d, explicit=%d, bandwidthChanged=%d)", - seekDiscontinuity, explicitDiscontinuity, bandwidthChanged); + if (changedMask == 0) { + // If nothing changed as far as the audio/video decoders + // are concerned we can proceed. + onChangeConfiguration3(msg); + return; + } - sp<ABuffer> tmp = new ABuffer(188); - memset(tmp->data(), 0, tmp->size()); + // Something changed, inform the player which will shutdown the + // corresponding decoders and will post the reply once that's done. + // Handling the reply will continue executing below in + // onChangeConfiguration3. + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatStreamsChanged); + notify->setInt32("changedMask", changedMask); - // signal a 'hard' discontinuity for explicit or bandwidthChanged. - uint8_t type = (explicitDiscontinuity || bandwidthChanged) ? 1 : 0; + msg->setWhat(kWhatChangeConfiguration3); + msg->setTarget(id()); - if (mPlaylist->isComplete() || mPlaylist->isEvent()) { - // If this was a live event this made no sense since - // we don't have access to all the segment before the current - // one. - int64_t segmentStartTimeUs = getSegmentStartTimeUs(mSeqNumber); - memcpy(tmp->data() + 2, &segmentStartTimeUs, sizeof(segmentStartTimeUs)); + notify->setMessage("reply", msg); + notify->post(); +} - type |= 2; - } +void LiveSession::onChangeConfiguration3(const sp<AMessage> &msg) { + // All remaining fetchers are still suspended, the player has shutdown + // any decoders that needed it. - tmp->data()[1] = type; + uint32_t streamMask; + CHECK(msg->findInt32("streamMask", (int32_t *)&streamMask)); - mDataSource->queueBuffer(tmp); + AString audioURI, videoURI, subtitleURI; + if (streamMask & STREAMTYPE_AUDIO) { + CHECK(msg->findString("audioURI", &audioURI)); + } + if (streamMask & STREAMTYPE_VIDEO) { + CHECK(msg->findString("videoURI", &videoURI)); + } + if (streamMask & STREAMTYPE_SUBTITLES) { + CHECK(msg->findString("subtitleURI", &subtitleURI)); } - mDataSource->queueBuffer(buffer); + int64_t timeUs; + CHECK(msg->findInt64("timeUs", &timeUs)); - mPrevBandwidthIndex = bandwidthIndex; - ++mSeqNumber; + if (timeUs < 0ll) { + timeUs = mLastDequeuedTimeUs; + } + mRealTimeBaseUs = ALooper::GetNowUs() - timeUs; - postMonitorQueue(); -} + mStreamMask = streamMask; + mAudioURI = audioURI; + mVideoURI = videoURI; + mSubtitleURI = subtitleURI; -void LiveSession::signalEOS(status_t err) { - if (mInPreparationPhase && mNotify != NULL) { - sp<AMessage> notify = mNotify->dup(); + // Resume all existing fetchers and assign them packet sources. + for (size_t i = 0; i < mFetcherInfos.size(); ++i) { + const AString &uri = mFetcherInfos.keyAt(i); - notify->setInt32( - "what", - err == ERROR_END_OF_STREAM - ? kWhatPrepared : kWhatPreparationFailed); + uint32_t resumeMask = 0; - if (err != ERROR_END_OF_STREAM) { - notify->setInt32("err", err); + sp<AnotherPacketSource> audioSource; + if ((streamMask & STREAMTYPE_AUDIO) && uri == audioURI) { + audioSource = mPacketSources.valueFor(STREAMTYPE_AUDIO); + resumeMask |= STREAMTYPE_AUDIO; } - notify->post(); - - mInPreparationPhase = false; - } - - mDataSource->queueEOS(err); -} - -void LiveSession::onMonitorQueue() { - if (mSeekTimeUs >= 0 - || mDataSource->countQueuedBuffers() < kMaxNumQueuedFragments) { - onDownloadNext(); - } else { - if (mInPreparationPhase) { - if (mNotify != NULL) { - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatPrepared); - notify->post(); - } - - mInPreparationPhase = false; + sp<AnotherPacketSource> videoSource; + if ((streamMask & STREAMTYPE_VIDEO) && uri == videoURI) { + videoSource = mPacketSources.valueFor(STREAMTYPE_VIDEO); + resumeMask |= STREAMTYPE_VIDEO; } - postMonitorQueue(1000000ll); - } -} + sp<AnotherPacketSource> subtitleSource; + if ((streamMask & STREAMTYPE_SUBTITLES) && uri == subtitleURI) { + subtitleSource = mPacketSources.valueFor(STREAMTYPE_SUBTITLES); + resumeMask |= STREAMTYPE_SUBTITLES; + } -status_t LiveSession::decryptBuffer( - size_t playlistIndex, const sp<ABuffer> &buffer) { - sp<AMessage> itemMeta; - bool found = false; - AString method; + CHECK_NE(resumeMask, 0u); - for (ssize_t i = playlistIndex; i >= 0; --i) { - AString uri; - CHECK(mPlaylist->itemAt(i, &uri, &itemMeta)); + ALOGV("resuming fetchers for mask 0x%08x", resumeMask); - if (itemMeta->findString("cipher-method", &method)) { - found = true; - break; - } - } + streamMask &= ~resumeMask; - if (!found) { - method = "NONE"; + mFetcherInfos.valueAt(i).mFetcher->startAsync( + audioSource, videoSource, subtitleSource); } - if (method == "NONE") { - return OK; - } else if (!(method == "AES-128")) { - ALOGE("Unsupported cipher method '%s'", method.c_str()); - return ERROR_UNSUPPORTED; - } + // streamMask now only contains the types that need a new fetcher created. - AString keyURI; - if (!itemMeta->findString("cipher-uri", &keyURI)) { - ALOGE("Missing key uri"); - return ERROR_MALFORMED; + if (streamMask != 0) { + ALOGV("creating new fetchers for mask 0x%08x", streamMask); } - ssize_t index = mAESKeyForURI.indexOfKey(keyURI); - - sp<ABuffer> key; - if (index >= 0) { - key = mAESKeyForURI.valueAt(index); - } else { - key = new ABuffer(16); - - sp<HTTPBase> keySource = - HTTPBase::Create( - (mFlags & kFlagIncognito) - ? HTTPBase::kFlagIncognito - : 0); + while (streamMask != 0) { + StreamType streamType = (StreamType)(streamMask & ~(streamMask - 1)); - if (mUIDValid) { - keySource->setUID(mUID); + AString uri; + switch (streamType) { + case STREAMTYPE_AUDIO: + uri = audioURI; + break; + case STREAMTYPE_VIDEO: + uri = videoURI; + break; + case STREAMTYPE_SUBTITLES: + uri = subtitleURI; + break; + default: + TRESPASS(); } - status_t err = - keySource->connect( - keyURI.c_str(), - mExtraHeaders.isEmpty() ? NULL : &mExtraHeaders); - - if (err == OK) { - size_t offset = 0; - while (offset < 16) { - ssize_t n = keySource->readAt( - offset, key->data() + offset, 16 - offset); - if (n <= 0) { - err = ERROR_IO; - break; - } + sp<PlaylistFetcher> fetcher = addFetcher(uri.c_str()); + CHECK(fetcher != NULL); - offset += n; - } - } + sp<AnotherPacketSource> audioSource; + if ((streamMask & STREAMTYPE_AUDIO) && uri == audioURI) { + audioSource = mPacketSources.valueFor(STREAMTYPE_AUDIO); + audioSource->clear(); - if (err != OK) { - ALOGE("failed to fetch cipher key from '%s'.", keyURI.c_str()); - return ERROR_IO; + streamMask &= ~STREAMTYPE_AUDIO; } - mAESKeyForURI.add(keyURI, key); - } + sp<AnotherPacketSource> videoSource; + if ((streamMask & STREAMTYPE_VIDEO) && uri == videoURI) { + videoSource = mPacketSources.valueFor(STREAMTYPE_VIDEO); + videoSource->clear(); - AES_KEY aes_key; - if (AES_set_decrypt_key(key->data(), 128, &aes_key) != 0) { - ALOGE("failed to set AES decryption key."); - return UNKNOWN_ERROR; - } - - unsigned char aes_ivec[16]; - - AString iv; - if (itemMeta->findString("cipher-iv", &iv)) { - if ((!iv.startsWith("0x") && !iv.startsWith("0X")) - || iv.size() != 16 * 2 + 2) { - ALOGE("malformed cipher IV '%s'.", iv.c_str()); - return ERROR_MALFORMED; + streamMask &= ~STREAMTYPE_VIDEO; } - memset(aes_ivec, 0, sizeof(aes_ivec)); - for (size_t i = 0; i < 16; ++i) { - char c1 = tolower(iv.c_str()[2 + 2 * i]); - char c2 = tolower(iv.c_str()[3 + 2 * i]); - if (!isxdigit(c1) || !isxdigit(c2)) { - ALOGE("malformed cipher IV '%s'.", iv.c_str()); - return ERROR_MALFORMED; - } - uint8_t nibble1 = isdigit(c1) ? c1 - '0' : c1 - 'a' + 10; - uint8_t nibble2 = isdigit(c2) ? c2 - '0' : c2 - 'a' + 10; + sp<AnotherPacketSource> subtitleSource; + if ((streamMask & STREAMTYPE_SUBTITLES) && uri == subtitleURI) { + subtitleSource = mPacketSources.valueFor(STREAMTYPE_SUBTITLES); + subtitleSource->clear(); - aes_ivec[i] = nibble1 << 4 | nibble2; + streamMask &= ~STREAMTYPE_SUBTITLES; } - } else { - memset(aes_ivec, 0, sizeof(aes_ivec)); - aes_ivec[15] = mSeqNumber & 0xff; - aes_ivec[14] = (mSeqNumber >> 8) & 0xff; - aes_ivec[13] = (mSeqNumber >> 16) & 0xff; - aes_ivec[12] = (mSeqNumber >> 24) & 0xff; + + fetcher->startAsync(audioSource, videoSource, subtitleSource, timeUs); } - AES_cbc_encrypt( - buffer->data(), buffer->data(), buffer->size(), - &aes_key, aes_ivec, AES_DECRYPT); + // All fetchers have now been started, the configuration change + // has completed. - // hexdump(buffer->data(), buffer->size()); + scheduleCheckBandwidthEvent(); - size_t n = buffer->size(); - CHECK_GT(n, 0u); + ALOGV("XXX configuration change completed."); - size_t pad = buffer->data()[n - 1]; + mReconfigurationInProgress = false; - CHECK_GT(pad, 0u); - CHECK_LE(pad, 16u); - CHECK_GE((size_t)n, pad); - for (size_t i = 0; i < pad; ++i) { - CHECK_EQ((unsigned)buffer->data()[n - 1 - i], pad); + if (mDisconnectReplyID != 0) { + finishDisconnect(); } +} - n -= pad; - - buffer->setRange(buffer->offset(), n); - - return OK; +void LiveSession::scheduleCheckBandwidthEvent() { + sp<AMessage> msg = new AMessage(kWhatCheckBandwidth, id()); + msg->setInt32("generation", mCheckBandwidthGeneration); + msg->post(10000000ll); } -void LiveSession::postMonitorQueue(int64_t delayUs) { - sp<AMessage> msg = new AMessage(kWhatMonitorQueue, id()); - msg->setInt32("generation", ++mMonitorQueueGeneration); - msg->post(delayUs); +void LiveSession::cancelCheckBandwidthEvent() { + ++mCheckBandwidthGeneration; } -void LiveSession::onSeek(const sp<AMessage> &msg) { - int64_t timeUs; - CHECK(msg->findInt64("timeUs", &timeUs)); +void LiveSession::onCheckBandwidth() { + if (mReconfigurationInProgress) { + scheduleCheckBandwidthEvent(); + return; + } + + size_t bandwidthIndex = getBandwidthIndex(); + if (mPrevBandwidthIndex < 0 + || bandwidthIndex != (size_t)mPrevBandwidthIndex) { + changeConfiguration(-1ll /* timeUs */, bandwidthIndex); + } - mSeekTimeUs = timeUs; - postMonitorQueue(); + // Handling the kWhatCheckBandwidth even here does _not_ automatically + // schedule another one on return, only an explicit call to + // scheduleCheckBandwidthEvent will do that. + // This ensures that only one configuration change is ongoing at any + // one time, once that completes it'll schedule another check bandwidth + // event. } -status_t LiveSession::getDuration(int64_t *durationUs) const { - Mutex::Autolock autoLock(mLock); - *durationUs = mDurationUs; +void LiveSession::postPrepared(status_t err) { + CHECK(mInPreparationPhase); - return OK; -} + sp<AMessage> notify = mNotify->dup(); + if (err == OK || err == ERROR_END_OF_STREAM) { + notify->setInt32("what", kWhatPrepared); + } else { + notify->setInt32("what", kWhatPreparationFailed); + notify->setInt32("err", err); + } -bool LiveSession::isSeekable() const { - int64_t durationUs; - return getDuration(&durationUs) == OK && durationUs >= 0; -} + notify->post(); -bool LiveSession::hasDynamicDuration() const { - return !mDurationFixed; + mInPreparationPhase = false; } } // namespace android diff --git a/media/libstagefright/include/LiveSession.h b/media/libstagefright/httplive/LiveSession.h index db44a33..99b480a8 100644 --- a/media/libstagefright/include/LiveSession.h +++ b/media/libstagefright/httplive/LiveSession.h @@ -25,10 +25,13 @@ namespace android { struct ABuffer; +struct AnotherPacketSource; struct DataSource; +struct HTTPBase; struct LiveDataSource; struct M3UParser; -struct HTTPBase; +struct PlaylistFetcher; +struct Parcel; struct LiveSession : public AHandler { enum Flags { @@ -39,24 +42,34 @@ struct LiveSession : public AHandler { const sp<AMessage> ¬ify, uint32_t flags = 0, bool uidValid = false, uid_t uid = 0); - sp<DataSource> getDataSource(); + enum StreamType { + STREAMTYPE_AUDIO = 1, + STREAMTYPE_VIDEO = 2, + STREAMTYPE_SUBTITLES = 4, + }; + status_t dequeueAccessUnit(StreamType stream, sp<ABuffer> *accessUnit); + + status_t getStreamFormat(StreamType stream, sp<AMessage> *format); - void connect( + void connectAsync( const char *url, const KeyedVector<String8, String8> *headers = NULL); - void disconnect(); + status_t disconnect(); // Blocks until seek is complete. - void seekTo(int64_t timeUs); + status_t seekTo(int64_t timeUs); status_t getDuration(int64_t *durationUs) const; + status_t getTrackInfo(Parcel *reply) const; + status_t selectTrack(size_t index, bool select); bool isSeekable() const; bool hasDynamicDuration() const; - // Posted notification's "what" field will carry one of the following: enum { + kWhatStreamsChanged, + kWhatError, kWhatPrepared, kWhatPreparationFailed, }; @@ -67,23 +80,31 @@ protected: virtual void onMessageReceived(const sp<AMessage> &msg); private: - enum { - kMaxNumQueuedFragments = 3, - kMaxNumRetries = 5, - }; + friend struct PlaylistFetcher; enum { - kWhatConnect = 'conn', - kWhatDisconnect = 'disc', - kWhatMonitorQueue = 'moni', - kWhatSeek = 'seek', + kWhatConnect = 'conn', + kWhatDisconnect = 'disc', + kWhatSeek = 'seek', + kWhatFetcherNotify = 'notf', + kWhatCheckBandwidth = 'bndw', + kWhatChangeConfiguration = 'chC0', + kWhatChangeConfiguration2 = 'chC2', + kWhatChangeConfiguration3 = 'chC3', + kWhatFinishDisconnect2 = 'fin2', }; struct BandwidthItem { - AString mURI; + size_t mPlaylistIndex; unsigned long mBandwidth; }; + struct FetcherInfo { + sp<PlaylistFetcher> mFetcher; + int64_t mDurationUs; + bool mIsPrepared; + }; + sp<AMessage> mNotify; uint32_t mFlags; bool mUIDValid; @@ -91,71 +112,64 @@ private: bool mInPreparationPhase; - sp<LiveDataSource> mDataSource; - sp<HTTPBase> mHTTPDataSource; + KeyedVector<String8, String8> mExtraHeaders; AString mMasterURL; - KeyedVector<String8, String8> mExtraHeaders; Vector<BandwidthItem> mBandwidthItems; - - KeyedVector<AString, sp<ABuffer> > mAESKeyForURI; - ssize_t mPrevBandwidthIndex; - int64_t mLastPlaylistFetchTimeUs; + sp<M3UParser> mPlaylist; - int32_t mSeqNumber; - int64_t mSeekTimeUs; - int32_t mNumRetries; - bool mStartOfPlayback; - - mutable Mutex mLock; - Condition mCondition; - int64_t mDurationUs; - bool mDurationFixed; // Duration has been determined once and for all. - bool mSeekDone; - bool mDisconnectPending; - - int32_t mMonitorQueueGeneration; - - enum RefreshState { - INITIAL_MINIMUM_RELOAD_DELAY, - FIRST_UNCHANGED_RELOAD_ATTEMPT, - SECOND_UNCHANGED_RELOAD_ATTEMPT, - THIRD_UNCHANGED_RELOAD_ATTEMPT - }; - RefreshState mRefreshState; - uint8_t mPlaylistHash[16]; + KeyedVector<AString, FetcherInfo> mFetcherInfos; + AString mAudioURI, mVideoURI, mSubtitleURI; + uint32_t mStreamMask; + + KeyedVector<StreamType, sp<AnotherPacketSource> > mPacketSources; + + int32_t mCheckBandwidthGeneration; + + size_t mContinuationCounter; + sp<AMessage> mContinuation; + + int64_t mLastDequeuedTimeUs; + int64_t mRealTimeBaseUs; + + bool mReconfigurationInProgress; + uint32_t mDisconnectReplyID; + + sp<PlaylistFetcher> addFetcher(const char *uri); void onConnect(const sp<AMessage> &msg); - void onDisconnect(); - void onDownloadNext(); - void onMonitorQueue(); - void onSeek(const sp<AMessage> &msg); + status_t onSeek(const sp<AMessage> &msg); + void onFinishDisconnect2(); status_t fetchFile( const char *url, sp<ABuffer> *out, int64_t range_offset = 0, int64_t range_length = -1); - sp<M3UParser> fetchPlaylist(const char *url, bool *unchanged); + sp<M3UParser> fetchPlaylist( + const char *url, uint8_t *curPlaylistHash, bool *unchanged); + size_t getBandwidthIndex(); - status_t decryptBuffer( - size_t playlistIndex, const sp<ABuffer> &buffer); + static int SortByBandwidth(const BandwidthItem *, const BandwidthItem *); - void postMonitorQueue(int64_t delayUs = 0); + void changeConfiguration( + int64_t timeUs, size_t bandwidthIndex, bool pickTrack = false); + void onChangeConfiguration(const sp<AMessage> &msg); + void onChangeConfiguration2(const sp<AMessage> &msg); + void onChangeConfiguration3(const sp<AMessage> &msg); - bool timeToRefreshPlaylist(int64_t nowUs) const; + void scheduleCheckBandwidthEvent(); + void cancelCheckBandwidthEvent(); - static int SortByBandwidth(const BandwidthItem *, const BandwidthItem *); + void onCheckBandwidth(); - // Returns the media time in us of the segment specified by seqNumber. - // This is computed by summing the durations of all segments before it. - int64_t getSegmentStartTimeUs(int32_t seqNumber) const; + void finishDisconnect(); - void signalEOS(status_t err); + void postPrepared(status_t err); DISALLOW_EVIL_CONSTRUCTORS(LiveSession); }; diff --git a/media/libstagefright/httplive/M3UParser.cpp b/media/libstagefright/httplive/M3UParser.cpp index 68bbca2..243888c 100644 --- a/media/libstagefright/httplive/M3UParser.cpp +++ b/media/libstagefright/httplive/M3UParser.cpp @@ -18,14 +18,217 @@ #define LOG_TAG "M3UParser" #include <utils/Log.h> -#include "include/M3UParser.h" - +#include "M3UParser.h" +#include <binder/Parcel.h> +#include <cutils/properties.h> #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> #include <media/stagefright/MediaErrors.h> +#include <media/mediaplayer.h> namespace android { +struct M3UParser::MediaGroup : public RefBase { + enum Type { + TYPE_AUDIO, + TYPE_VIDEO, + TYPE_SUBS, + }; + + enum FlagBits { + FLAG_AUTOSELECT = 1, + FLAG_DEFAULT = 2, + FLAG_FORCED = 4, + FLAG_HAS_LANGUAGE = 8, + FLAG_HAS_URI = 16, + }; + + MediaGroup(Type type); + + Type type() const; + + status_t addMedia( + const char *name, + const char *uri, + const char *language, + uint32_t flags); + + bool getActiveURI(AString *uri) const; + + void pickRandomMediaItems(); + status_t selectTrack(size_t index, bool select); + void getTrackInfo(Parcel* reply) const; + size_t countTracks() const; + +protected: + virtual ~MediaGroup(); + +private: + struct Media { + AString mName; + AString mURI; + AString mLanguage; + uint32_t mFlags; + }; + + Type mType; + Vector<Media> mMediaItems; + + ssize_t mSelectedIndex; + + DISALLOW_EVIL_CONSTRUCTORS(MediaGroup); +}; + +M3UParser::MediaGroup::MediaGroup(Type type) + : mType(type), + mSelectedIndex(-1) { +} + +M3UParser::MediaGroup::~MediaGroup() { +} + +M3UParser::MediaGroup::Type M3UParser::MediaGroup::type() const { + return mType; +} + +status_t M3UParser::MediaGroup::addMedia( + const char *name, + const char *uri, + const char *language, + uint32_t flags) { + mMediaItems.push(); + Media &item = mMediaItems.editItemAt(mMediaItems.size() - 1); + + item.mName = name; + + if (uri) { + item.mURI = uri; + } + + if (language) { + item.mLanguage = language; + } + + item.mFlags = flags; + + return OK; +} + +void M3UParser::MediaGroup::pickRandomMediaItems() { +#if 1 + switch (mType) { + case TYPE_AUDIO: + { + char value[PROPERTY_VALUE_MAX]; + if (property_get("media.httplive.audio-index", value, NULL)) { + char *end; + mSelectedIndex = strtoul(value, &end, 10); + CHECK(end > value && *end == '\0'); + + if (mSelectedIndex >= mMediaItems.size()) { + mSelectedIndex = mMediaItems.size() - 1; + } + } else { + mSelectedIndex = 0; + } + break; + } + + case TYPE_VIDEO: + { + mSelectedIndex = 0; + break; + } + + case TYPE_SUBS: + { + mSelectedIndex = -1; + break; + } + + default: + TRESPASS(); + } +#else + mSelectedIndex = (rand() * mMediaItems.size()) / RAND_MAX; +#endif +} + +status_t M3UParser::MediaGroup::selectTrack(size_t index, bool select) { + if (mType != TYPE_SUBS) { + ALOGE("only select subtitile tracks for now!"); + return INVALID_OPERATION; + } + + if (select) { + if (index >= mMediaItems.size()) { + ALOGE("track %d does not exist", index); + return INVALID_OPERATION; + } + if (mSelectedIndex == index) { + ALOGE("track %d already selected", index); + return BAD_VALUE; + } + ALOGV("selected track %d", index); + mSelectedIndex = index; + } else { + if (mSelectedIndex != index) { + ALOGE("track %d is not selected", index); + return BAD_VALUE; + } + ALOGV("unselected track %d", index); + mSelectedIndex = -1; + } + + return OK; +} + +void M3UParser::MediaGroup::getTrackInfo(Parcel* reply) const { + for (size_t i = 0; i < mMediaItems.size(); ++i) { + reply->writeInt32(2); // 2 fields + + if (mType == TYPE_AUDIO) { + reply->writeInt32(MEDIA_TRACK_TYPE_AUDIO); + } else if (mType == TYPE_VIDEO) { + reply->writeInt32(MEDIA_TRACK_TYPE_VIDEO); + } else if (mType == TYPE_SUBS) { + reply->writeInt32(MEDIA_TRACK_TYPE_SUBTITLE); + } else { + reply->writeInt32(MEDIA_TRACK_TYPE_UNKNOWN); + } + + const Media &item = mMediaItems.itemAt(i); + const char *lang = item.mLanguage.empty() ? "und" : item.mLanguage.c_str(); + reply->writeString16(String16(lang)); + + if (mType == TYPE_SUBS) { + // TO-DO: pass in a MediaFormat instead + reply->writeInt32(!!(item.mFlags & MediaGroup::FLAG_AUTOSELECT)); + reply->writeInt32(!!(item.mFlags & MediaGroup::FLAG_DEFAULT)); + reply->writeInt32(!!(item.mFlags & MediaGroup::FLAG_FORCED)); + } + } +} + +size_t M3UParser::MediaGroup::countTracks() const { + return mMediaItems.size(); +} + +bool M3UParser::MediaGroup::getActiveURI(AString *uri) const { + for (size_t i = 0; i < mMediaItems.size(); ++i) { + if (mSelectedIndex >= 0 && i == (size_t)mSelectedIndex) { + const Media &item = mMediaItems.itemAt(i); + + *uri = item.mURI; + return true; + } + } + + return false; +} + +//////////////////////////////////////////////////////////////////////////////// + M3UParser::M3UParser( const char *baseURI, const void *data, size_t size) : mInitCheck(NO_INIT), @@ -33,7 +236,8 @@ M3UParser::M3UParser( mIsExtM3U(false), mIsVariantPlaylist(false), mIsComplete(false), - mIsEvent(false) { + mIsEvent(false), + mSelectedIndex(-1) { mInitCheck = parse(data, size); } @@ -92,6 +296,91 @@ bool M3UParser::itemAt(size_t index, AString *uri, sp<AMessage> *meta) { return true; } +void M3UParser::pickRandomMediaItems() { + for (size_t i = 0; i < mMediaGroups.size(); ++i) { + mMediaGroups.valueAt(i)->pickRandomMediaItems(); + } +} + +status_t M3UParser::selectTrack(size_t index, bool select) { + for (size_t i = 0, ii = index; i < mMediaGroups.size(); ++i) { + sp<MediaGroup> group = mMediaGroups.valueAt(i); + size_t tracks = group->countTracks(); + if (ii < tracks) { + status_t err = group->selectTrack(ii, select); + if (err == OK) { + mSelectedIndex = select ? index : -1; + } + return err; + } + ii -= tracks; + } + return INVALID_OPERATION; +} + +status_t M3UParser::getTrackInfo(Parcel* reply) const { + size_t trackCount = 0; + for (size_t i = 0; i < mMediaGroups.size(); ++i) { + trackCount += mMediaGroups.valueAt(i)->countTracks(); + } + reply->writeInt32(trackCount); + + for (size_t i = 0; i < mMediaGroups.size(); ++i) { + mMediaGroups.valueAt(i)->getTrackInfo(reply); + } + return OK; +} + +ssize_t M3UParser::getSelectedIndex() const { + return mSelectedIndex; +} + +bool M3UParser::getTypeURI(size_t index, const char *key, AString *uri) const { + if (!mIsVariantPlaylist) { + *uri = mBaseURI; + + // Assume media without any more specific attribute contains + // audio and video, but no subtitles. + return !strcmp("audio", key) || !strcmp("video", key); + } + + CHECK_LT(index, mItems.size()); + + sp<AMessage> meta = mItems.itemAt(index).mMeta; + + AString groupID; + if (!meta->findString(key, &groupID)) { + *uri = mItems.itemAt(index).mURI; + + // Assume media without any more specific attribute contains + // audio and video, but no subtitles. + return !strcmp("audio", key) || !strcmp("video", key); + } + + sp<MediaGroup> group = mMediaGroups.valueFor(groupID); + if (!group->getActiveURI(uri)) { + return false; + } + + if ((*uri).empty()) { + *uri = mItems.itemAt(index).mURI; + } + + return true; +} + +bool M3UParser::getAudioURI(size_t index, AString *uri) const { + return getTypeURI(index, "audio", uri); +} + +bool M3UParser::getVideoURI(size_t index, AString *uri) const { + return getTypeURI(index, "video", uri); +} + +bool M3UParser::getSubtitleURI(size_t index, AString *uri) const { + return getTypeURI(index, "subtitles", uri); +} + static bool MakeURL(const char *baseURL, const char *url, AString *out) { out->clear(); @@ -241,6 +530,8 @@ status_t M3UParser::parse(const void *_data, size_t size) { segmentRangeOffset = offset + length; } + } else if (line.startsWith("#EXT-X-MEDIA")) { + err = parseMedia(line); } if (err != OK) { @@ -322,9 +613,31 @@ status_t M3UParser::parseMetaDataDuration( return OK; } -// static +// Find the next occurence of the character "what" at or after "offset", +// but ignore occurences between quotation marks. +// Return the index of the occurrence or -1 if not found. +static ssize_t FindNextUnquoted( + const AString &line, char what, size_t offset) { + CHECK_NE((int)what, (int)'"'); + + bool quoted = false; + while (offset < line.size()) { + char c = line.c_str()[offset]; + + if (c == '"') { + quoted = !quoted; + } else if (c == what && !quoted) { + return offset; + } + + ++offset; + } + + return -1; +} + status_t M3UParser::parseStreamInf( - const AString &line, sp<AMessage> *meta) { + const AString &line, sp<AMessage> *meta) const { ssize_t colonPos = line.find(":"); if (colonPos < 0) { @@ -334,7 +647,7 @@ status_t M3UParser::parseStreamInf( size_t offset = colonPos + 1; while (offset < line.size()) { - ssize_t end = line.find(",", offset); + ssize_t end = FindNextUnquoted(line, ',', offset); if (end < 0) { end = line.size(); } @@ -371,33 +684,35 @@ status_t M3UParser::parseStreamInf( *meta = new AMessage; } (*meta)->setInt32("bandwidth", x); - } - } + } else if (!strcasecmp("audio", key.c_str()) + || !strcasecmp("video", key.c_str()) + || !strcasecmp("subtitles", key.c_str())) { + if (val.size() < 2 + || val.c_str()[0] != '"' + || val.c_str()[val.size() - 1] != '"') { + ALOGE("Expected quoted string for %s attribute, " + "got '%s' instead.", + key.c_str(), val.c_str()); + + return ERROR_MALFORMED; + } - return OK; -} + AString groupID(val, 1, val.size() - 2); + ssize_t groupIndex = mMediaGroups.indexOfKey(groupID); -// Find the next occurence of the character "what" at or after "offset", -// but ignore occurences between quotation marks. -// Return the index of the occurrence or -1 if not found. -static ssize_t FindNextUnquoted( - const AString &line, char what, size_t offset) { - CHECK_NE((int)what, (int)'"'); + if (groupIndex < 0) { + ALOGE("Undefined media group '%s' referenced in stream info.", + groupID.c_str()); - bool quoted = false; - while (offset < line.size()) { - char c = line.c_str()[offset]; + return ERROR_MALFORMED; + } - if (c == '"') { - quoted = !quoted; - } else if (c == what && !quoted) { - return offset; + key.tolower(); + (*meta)->setString(key.c_str(), groupID.c_str()); } - - ++offset; } - return -1; + return OK; } // static @@ -515,6 +830,234 @@ status_t M3UParser::parseByteRange( return OK; } +status_t M3UParser::parseMedia(const AString &line) { + ssize_t colonPos = line.find(":"); + + if (colonPos < 0) { + return ERROR_MALFORMED; + } + + bool haveGroupType = false; + MediaGroup::Type groupType = MediaGroup::TYPE_AUDIO; + + bool haveGroupID = false; + AString groupID; + + bool haveGroupLanguage = false; + AString groupLanguage; + + bool haveGroupName = false; + AString groupName; + + bool haveGroupAutoselect = false; + bool groupAutoselect = false; + + bool haveGroupDefault = false; + bool groupDefault = false; + + bool haveGroupForced = false; + bool groupForced = false; + + bool haveGroupURI = false; + AString groupURI; + + size_t offset = colonPos + 1; + + while (offset < line.size()) { + ssize_t end = FindNextUnquoted(line, ',', offset); + if (end < 0) { + end = line.size(); + } + + AString attr(line, offset, end - offset); + attr.trim(); + + offset = end + 1; + + ssize_t equalPos = attr.find("="); + if (equalPos < 0) { + continue; + } + + AString key(attr, 0, equalPos); + key.trim(); + + AString val(attr, equalPos + 1, attr.size() - equalPos - 1); + val.trim(); + + ALOGV("key=%s value=%s", key.c_str(), val.c_str()); + + if (!strcasecmp("type", key.c_str())) { + if (!strcasecmp("subtitles", val.c_str())) { + groupType = MediaGroup::TYPE_SUBS; + } else if (!strcasecmp("audio", val.c_str())) { + groupType = MediaGroup::TYPE_AUDIO; + } else if (!strcasecmp("video", val.c_str())) { + groupType = MediaGroup::TYPE_VIDEO; + } else { + ALOGE("Invalid media group type '%s'", val.c_str()); + return ERROR_MALFORMED; + } + + haveGroupType = true; + } else if (!strcasecmp("group-id", key.c_str())) { + if (val.size() < 2 + || val.c_str()[0] != '"' + || val.c_str()[val.size() - 1] != '"') { + ALOGE("Expected quoted string for GROUP-ID, got '%s' instead.", + val.c_str()); + + return ERROR_MALFORMED; + } + + groupID.setTo(val, 1, val.size() - 2); + haveGroupID = true; + } else if (!strcasecmp("language", key.c_str())) { + if (val.size() < 2 + || val.c_str()[0] != '"' + || val.c_str()[val.size() - 1] != '"') { + ALOGE("Expected quoted string for LANGUAGE, got '%s' instead.", + val.c_str()); + + return ERROR_MALFORMED; + } + + groupLanguage.setTo(val, 1, val.size() - 2); + haveGroupLanguage = true; + } else if (!strcasecmp("name", key.c_str())) { + if (val.size() < 2 + || val.c_str()[0] != '"' + || val.c_str()[val.size() - 1] != '"') { + ALOGE("Expected quoted string for NAME, got '%s' instead.", + val.c_str()); + + return ERROR_MALFORMED; + } + + groupName.setTo(val, 1, val.size() - 2); + haveGroupName = true; + } else if (!strcasecmp("autoselect", key.c_str())) { + groupAutoselect = false; + if (!strcasecmp("YES", val.c_str())) { + groupAutoselect = true; + } else if (!strcasecmp("NO", val.c_str())) { + groupAutoselect = false; + } else { + ALOGE("Expected YES or NO for AUTOSELECT attribute, " + "got '%s' instead.", + val.c_str()); + + return ERROR_MALFORMED; + } + + haveGroupAutoselect = true; + } else if (!strcasecmp("default", key.c_str())) { + groupDefault = false; + if (!strcasecmp("YES", val.c_str())) { + groupDefault = true; + } else if (!strcasecmp("NO", val.c_str())) { + groupDefault = false; + } else { + ALOGE("Expected YES or NO for DEFAULT attribute, " + "got '%s' instead.", + val.c_str()); + + return ERROR_MALFORMED; + } + + haveGroupDefault = true; + } else if (!strcasecmp("forced", key.c_str())) { + groupForced = false; + if (!strcasecmp("YES", val.c_str())) { + groupForced = true; + } else if (!strcasecmp("NO", val.c_str())) { + groupForced = false; + } else { + ALOGE("Expected YES or NO for FORCED attribute, " + "got '%s' instead.", + val.c_str()); + + return ERROR_MALFORMED; + } + + haveGroupForced = true; + } else if (!strcasecmp("uri", key.c_str())) { + if (val.size() < 2 + || val.c_str()[0] != '"' + || val.c_str()[val.size() - 1] != '"') { + ALOGE("Expected quoted string for URI, got '%s' instead.", + val.c_str()); + + return ERROR_MALFORMED; + } + + AString tmp(val, 1, val.size() - 2); + + if (!MakeURL(mBaseURI.c_str(), tmp.c_str(), &groupURI)) { + ALOGI("Failed to make absolute URI from '%s'.", tmp.c_str()); + } + + haveGroupURI = true; + } + } + + if (!haveGroupType || !haveGroupID || !haveGroupName) { + ALOGE("Incomplete EXT-X-MEDIA element."); + return ERROR_MALFORMED; + } + + uint32_t flags = 0; + if (haveGroupAutoselect && groupAutoselect) { + flags |= MediaGroup::FLAG_AUTOSELECT; + } + if (haveGroupDefault && groupDefault) { + flags |= MediaGroup::FLAG_DEFAULT; + } + if (haveGroupForced) { + if (groupType != MediaGroup::TYPE_SUBS) { + ALOGE("The FORCED attribute MUST not be present on anything " + "but SUBS media."); + + return ERROR_MALFORMED; + } + + if (groupForced) { + flags |= MediaGroup::FLAG_FORCED; + } + } + if (haveGroupLanguage) { + flags |= MediaGroup::FLAG_HAS_LANGUAGE; + } + if (haveGroupURI) { + flags |= MediaGroup::FLAG_HAS_URI; + } + + ssize_t groupIndex = mMediaGroups.indexOfKey(groupID); + sp<MediaGroup> group; + + if (groupIndex < 0) { + group = new MediaGroup(groupType); + mMediaGroups.add(groupID, group); + } else { + group = mMediaGroups.valueAt(groupIndex); + + if (group->type() != groupType) { + ALOGE("Attempt to put media item under group of different type " + "(groupType = %d, item type = %d", + group->type(), + groupType); + + return ERROR_MALFORMED; + } + } + + return group->addMedia( + groupName.c_str(), + haveGroupURI ? groupURI.c_str() : NULL, + haveGroupLanguage ? groupLanguage.c_str() : NULL, + flags); +} + // static status_t M3UParser::ParseInt32(const char *s, int32_t *x) { char *end; diff --git a/media/libstagefright/include/M3UParser.h b/media/libstagefright/httplive/M3UParser.h index 2d2f50f..5248004 100644 --- a/media/libstagefright/include/M3UParser.h +++ b/media/libstagefright/httplive/M3UParser.h @@ -40,10 +40,21 @@ struct M3UParser : public RefBase { size_t size(); bool itemAt(size_t index, AString *uri, sp<AMessage> *meta = NULL); + void pickRandomMediaItems(); + status_t selectTrack(size_t index, bool select); + status_t getTrackInfo(Parcel* reply) const; + ssize_t getSelectedIndex() const; + + bool getAudioURI(size_t index, AString *uri) const; + bool getVideoURI(size_t index, AString *uri) const; + bool getSubtitleURI(size_t index, AString *uri) const; + protected: virtual ~M3UParser(); private: + struct MediaGroup; + struct Item { AString mURI; sp<AMessage> mMeta; @@ -59,6 +70,10 @@ private: sp<AMessage> mMeta; Vector<Item> mItems; + ssize_t mSelectedIndex; + + // Media groups keyed by group ID. + KeyedVector<AString, sp<MediaGroup> > mMediaGroups; status_t parse(const void *data, size_t size); @@ -68,8 +83,8 @@ private: static status_t parseMetaDataDuration( const AString &line, sp<AMessage> *meta, const char *key); - static status_t parseStreamInf( - const AString &line, sp<AMessage> *meta); + status_t parseStreamInf( + const AString &line, sp<AMessage> *meta) const; static status_t parseCipherInfo( const AString &line, sp<AMessage> *meta, const AString &baseURI); @@ -78,6 +93,10 @@ private: const AString &line, uint64_t curOffset, uint64_t *length, uint64_t *offset); + status_t parseMedia(const AString &line); + + bool getTypeURI(size_t index, const char *key, AString *uri) const; + static status_t ParseInt32(const char *s, int32_t *x); static status_t ParseDouble(const char *s, double *x); diff --git a/media/libstagefright/httplive/PlaylistFetcher.cpp b/media/libstagefright/httplive/PlaylistFetcher.cpp new file mode 100644 index 0000000..973b779 --- /dev/null +++ b/media/libstagefright/httplive/PlaylistFetcher.cpp @@ -0,0 +1,976 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "PlaylistFetcher" +#include <utils/Log.h> + +#include "PlaylistFetcher.h" + +#include "LiveDataSource.h" +#include "LiveSession.h" +#include "M3UParser.h" + +#include "include/avc_utils.h" +#include "include/HTTPBase.h" +#include "include/ID3.h" +#include "mpeg2ts/AnotherPacketSource.h" + +#include <media/IStreamSource.h> +#include <media/stagefright/foundation/ABitReader.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/hexdump.h> +#include <media/stagefright/FileSource.h> +#include <media/stagefright/MediaDefs.h> +#include <media/stagefright/MetaData.h> +#include <media/stagefright/Utils.h> + +#include <ctype.h> +#include <openssl/aes.h> +#include <openssl/md5.h> + +namespace android { + +// static +const int64_t PlaylistFetcher::kMinBufferedDurationUs = 10000000ll; + +PlaylistFetcher::PlaylistFetcher( + const sp<AMessage> ¬ify, + const sp<LiveSession> &session, + const char *uri) + : mNotify(notify), + mSession(session), + mURI(uri), + mStreamTypeMask(0), + mStartTimeUs(-1ll), + mLastPlaylistFetchTimeUs(-1ll), + mSeqNumber(-1), + mNumRetries(0), + mStartup(true), + mNextPTSTimeUs(-1ll), + mMonitorQueueGeneration(0), + mRefreshState(INITIAL_MINIMUM_RELOAD_DELAY), + mFirstPTSValid(false), + mAbsoluteTimeAnchorUs(0ll) { + memset(mPlaylistHash, 0, sizeof(mPlaylistHash)); +} + +PlaylistFetcher::~PlaylistFetcher() { +} + +int64_t PlaylistFetcher::getSegmentStartTimeUs(int32_t seqNumber) const { + CHECK(mPlaylist != NULL); + + int32_t firstSeqNumberInPlaylist; + if (mPlaylist->meta() == NULL || !mPlaylist->meta()->findInt32( + "media-sequence", &firstSeqNumberInPlaylist)) { + firstSeqNumberInPlaylist = 0; + } + + int32_t lastSeqNumberInPlaylist = + firstSeqNumberInPlaylist + (int32_t)mPlaylist->size() - 1; + + CHECK_GE(seqNumber, firstSeqNumberInPlaylist); + CHECK_LE(seqNumber, lastSeqNumberInPlaylist); + + int64_t segmentStartUs = 0ll; + for (int32_t index = 0; + index < seqNumber - firstSeqNumberInPlaylist; ++index) { + sp<AMessage> itemMeta; + CHECK(mPlaylist->itemAt( + index, NULL /* uri */, &itemMeta)); + + int64_t itemDurationUs; + CHECK(itemMeta->findInt64("durationUs", &itemDurationUs)); + + segmentStartUs += itemDurationUs; + } + + return segmentStartUs; +} + +bool PlaylistFetcher::timeToRefreshPlaylist(int64_t nowUs) const { + if (mPlaylist == NULL) { + CHECK_EQ((int)mRefreshState, (int)INITIAL_MINIMUM_RELOAD_DELAY); + return true; + } + + int32_t targetDurationSecs; + CHECK(mPlaylist->meta()->findInt32("target-duration", &targetDurationSecs)); + + int64_t targetDurationUs = targetDurationSecs * 1000000ll; + + int64_t minPlaylistAgeUs; + + switch (mRefreshState) { + case INITIAL_MINIMUM_RELOAD_DELAY: + { + size_t n = mPlaylist->size(); + if (n > 0) { + sp<AMessage> itemMeta; + CHECK(mPlaylist->itemAt(n - 1, NULL /* uri */, &itemMeta)); + + int64_t itemDurationUs; + CHECK(itemMeta->findInt64("durationUs", &itemDurationUs)); + + minPlaylistAgeUs = itemDurationUs; + break; + } + + // fall through + } + + case FIRST_UNCHANGED_RELOAD_ATTEMPT: + { + minPlaylistAgeUs = targetDurationUs / 2; + break; + } + + case SECOND_UNCHANGED_RELOAD_ATTEMPT: + { + minPlaylistAgeUs = (targetDurationUs * 3) / 2; + break; + } + + case THIRD_UNCHANGED_RELOAD_ATTEMPT: + { + minPlaylistAgeUs = targetDurationUs * 3; + break; + } + + default: + TRESPASS(); + break; + } + + return mLastPlaylistFetchTimeUs + minPlaylistAgeUs <= nowUs; +} + +status_t PlaylistFetcher::decryptBuffer( + size_t playlistIndex, const sp<ABuffer> &buffer) { + sp<AMessage> itemMeta; + bool found = false; + AString method; + + for (ssize_t i = playlistIndex; i >= 0; --i) { + AString uri; + CHECK(mPlaylist->itemAt(i, &uri, &itemMeta)); + + if (itemMeta->findString("cipher-method", &method)) { + found = true; + break; + } + } + + if (!found) { + method = "NONE"; + } + + if (method == "NONE") { + return OK; + } else if (!(method == "AES-128")) { + ALOGE("Unsupported cipher method '%s'", method.c_str()); + return ERROR_UNSUPPORTED; + } + + AString keyURI; + if (!itemMeta->findString("cipher-uri", &keyURI)) { + ALOGE("Missing key uri"); + return ERROR_MALFORMED; + } + + ssize_t index = mAESKeyForURI.indexOfKey(keyURI); + + sp<ABuffer> key; + if (index >= 0) { + key = mAESKeyForURI.valueAt(index); + } else { + status_t err = mSession->fetchFile(keyURI.c_str(), &key); + + if (err != OK) { + ALOGE("failed to fetch cipher key from '%s'.", keyURI.c_str()); + return ERROR_IO; + } else if (key->size() != 16) { + ALOGE("key file '%s' wasn't 16 bytes in size.", keyURI.c_str()); + return ERROR_MALFORMED; + } + + mAESKeyForURI.add(keyURI, key); + } + + AES_KEY aes_key; + if (AES_set_decrypt_key(key->data(), 128, &aes_key) != 0) { + ALOGE("failed to set AES decryption key."); + return UNKNOWN_ERROR; + } + + unsigned char aes_ivec[16]; + + AString iv; + if (itemMeta->findString("cipher-iv", &iv)) { + if ((!iv.startsWith("0x") && !iv.startsWith("0X")) + || iv.size() != 16 * 2 + 2) { + ALOGE("malformed cipher IV '%s'.", iv.c_str()); + return ERROR_MALFORMED; + } + + memset(aes_ivec, 0, sizeof(aes_ivec)); + for (size_t i = 0; i < 16; ++i) { + char c1 = tolower(iv.c_str()[2 + 2 * i]); + char c2 = tolower(iv.c_str()[3 + 2 * i]); + if (!isxdigit(c1) || !isxdigit(c2)) { + ALOGE("malformed cipher IV '%s'.", iv.c_str()); + return ERROR_MALFORMED; + } + uint8_t nibble1 = isdigit(c1) ? c1 - '0' : c1 - 'a' + 10; + uint8_t nibble2 = isdigit(c2) ? c2 - '0' : c2 - 'a' + 10; + + aes_ivec[i] = nibble1 << 4 | nibble2; + } + } else { + memset(aes_ivec, 0, sizeof(aes_ivec)); + aes_ivec[15] = mSeqNumber & 0xff; + aes_ivec[14] = (mSeqNumber >> 8) & 0xff; + aes_ivec[13] = (mSeqNumber >> 16) & 0xff; + aes_ivec[12] = (mSeqNumber >> 24) & 0xff; + } + + AES_cbc_encrypt( + buffer->data(), buffer->data(), buffer->size(), + &aes_key, aes_ivec, AES_DECRYPT); + + // hexdump(buffer->data(), buffer->size()); + + size_t n = buffer->size(); + CHECK_GT(n, 0u); + + size_t pad = buffer->data()[n - 1]; + + CHECK_GT(pad, 0u); + CHECK_LE(pad, 16u); + CHECK_GE((size_t)n, pad); + for (size_t i = 0; i < pad; ++i) { + CHECK_EQ((unsigned)buffer->data()[n - 1 - i], pad); + } + + n -= pad; + + buffer->setRange(buffer->offset(), n); + + return OK; +} + +void PlaylistFetcher::postMonitorQueue(int64_t delayUs) { + sp<AMessage> msg = new AMessage(kWhatMonitorQueue, id()); + msg->setInt32("generation", mMonitorQueueGeneration); + msg->post(delayUs); +} + +void PlaylistFetcher::cancelMonitorQueue() { + ++mMonitorQueueGeneration; +} + +void PlaylistFetcher::startAsync( + const sp<AnotherPacketSource> &audioSource, + const sp<AnotherPacketSource> &videoSource, + const sp<AnotherPacketSource> &subtitleSource, + int64_t startTimeUs) { + sp<AMessage> msg = new AMessage(kWhatStart, id()); + + uint32_t streamTypeMask = 0ul; + + if (audioSource != NULL) { + msg->setPointer("audioSource", audioSource.get()); + streamTypeMask |= LiveSession::STREAMTYPE_AUDIO; + } + + if (videoSource != NULL) { + msg->setPointer("videoSource", videoSource.get()); + streamTypeMask |= LiveSession::STREAMTYPE_VIDEO; + } + + if (subtitleSource != NULL) { + msg->setPointer("subtitleSource", subtitleSource.get()); + streamTypeMask |= LiveSession::STREAMTYPE_SUBTITLES; + } + + msg->setInt32("streamTypeMask", streamTypeMask); + msg->setInt64("startTimeUs", startTimeUs); + msg->post(); +} + +void PlaylistFetcher::pauseAsync() { + (new AMessage(kWhatPause, id()))->post(); +} + +void PlaylistFetcher::stopAsync() { + (new AMessage(kWhatStop, id()))->post(); +} + +void PlaylistFetcher::onMessageReceived(const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatStart: + { + status_t err = onStart(msg); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatStarted); + notify->setInt32("err", err); + notify->post(); + break; + } + + case kWhatPause: + { + onPause(); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatPaused); + notify->post(); + break; + } + + case kWhatStop: + { + onStop(); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatStopped); + notify->post(); + break; + } + + case kWhatMonitorQueue: + { + int32_t generation; + CHECK(msg->findInt32("generation", &generation)); + + if (generation != mMonitorQueueGeneration) { + // Stale event + break; + } + + onMonitorQueue(); + break; + } + + default: + TRESPASS(); + } +} + +status_t PlaylistFetcher::onStart(const sp<AMessage> &msg) { + mPacketSources.clear(); + + uint32_t streamTypeMask; + CHECK(msg->findInt32("streamTypeMask", (int32_t *)&streamTypeMask)); + + int64_t startTimeUs; + CHECK(msg->findInt64("startTimeUs", &startTimeUs)); + + if (streamTypeMask & LiveSession::STREAMTYPE_AUDIO) { + void *ptr; + CHECK(msg->findPointer("audioSource", &ptr)); + + mPacketSources.add( + LiveSession::STREAMTYPE_AUDIO, + static_cast<AnotherPacketSource *>(ptr)); + } + + if (streamTypeMask & LiveSession::STREAMTYPE_VIDEO) { + void *ptr; + CHECK(msg->findPointer("videoSource", &ptr)); + + mPacketSources.add( + LiveSession::STREAMTYPE_VIDEO, + static_cast<AnotherPacketSource *>(ptr)); + } + + if (streamTypeMask & LiveSession::STREAMTYPE_SUBTITLES) { + void *ptr; + CHECK(msg->findPointer("subtitleSource", &ptr)); + + mPacketSources.add( + LiveSession::STREAMTYPE_SUBTITLES, + static_cast<AnotherPacketSource *>(ptr)); + } + + mStreamTypeMask = streamTypeMask; + mStartTimeUs = startTimeUs; + + if (mStartTimeUs >= 0ll) { + mSeqNumber = -1; + mStartup = true; + } + + postMonitorQueue(); + + return OK; +} + +void PlaylistFetcher::onPause() { + cancelMonitorQueue(); + + mPacketSources.clear(); + mStreamTypeMask = 0; +} + +void PlaylistFetcher::onStop() { + cancelMonitorQueue(); + + for (size_t i = 0; i < mPacketSources.size(); ++i) { + mPacketSources.valueAt(i)->clear(); + } + + mPacketSources.clear(); + mStreamTypeMask = 0; +} + +void PlaylistFetcher::notifyError(status_t err) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatError); + notify->setInt32("err", err); + notify->post(); +} + +void PlaylistFetcher::queueDiscontinuity( + ATSParser::DiscontinuityType type, const sp<AMessage> &extra) { + for (size_t i = 0; i < mPacketSources.size(); ++i) { + mPacketSources.valueAt(i)->queueDiscontinuity(type, extra); + } +} + +void PlaylistFetcher::onMonitorQueue() { + bool downloadMore = false; + + status_t finalResult; + if (mStreamTypeMask == LiveSession::STREAMTYPE_SUBTITLES) { + sp<AnotherPacketSource> packetSource = + mPacketSources.valueFor(LiveSession::STREAMTYPE_SUBTITLES); + + int64_t bufferedDurationUs = + packetSource->getBufferedDurationUs(&finalResult); + + downloadMore = (bufferedDurationUs < kMinBufferedDurationUs); + finalResult = OK; + } else { + bool first = true; + int64_t minBufferedDurationUs = 0ll; + + for (size_t i = 0; i < mPacketSources.size(); ++i) { + if ((mStreamTypeMask & mPacketSources.keyAt(i)) == 0) { + continue; + } + + int64_t bufferedDurationUs = + mPacketSources.valueAt(i)->getBufferedDurationUs(&finalResult); + + if (first || bufferedDurationUs < minBufferedDurationUs) { + minBufferedDurationUs = bufferedDurationUs; + first = false; + } + } + + downloadMore = + !first && (minBufferedDurationUs < kMinBufferedDurationUs); + } + + if (finalResult == OK && downloadMore) { + onDownloadNext(); + } else { + // Nothing to do yet, try again in a second. + + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("what", kWhatTemporarilyDoneFetching); + msg->post(); + + postMonitorQueue(1000000ll); + } +} + +void PlaylistFetcher::onDownloadNext() { + int64_t nowUs = ALooper::GetNowUs(); + + if (mLastPlaylistFetchTimeUs < 0ll + || (!mPlaylist->isComplete() && timeToRefreshPlaylist(nowUs))) { + bool unchanged; + sp<M3UParser> playlist = mSession->fetchPlaylist( + mURI.c_str(), mPlaylistHash, &unchanged); + + if (playlist == NULL) { + if (unchanged) { + // We succeeded in fetching the playlist, but it was + // unchanged from the last time we tried. + + if (mRefreshState != THIRD_UNCHANGED_RELOAD_ATTEMPT) { + mRefreshState = (RefreshState)(mRefreshState + 1); + } + } else { + ALOGE("failed to load playlist at url '%s'", mURI.c_str()); + notifyError(ERROR_IO); + return; + } + } else { + mRefreshState = INITIAL_MINIMUM_RELOAD_DELAY; + mPlaylist = playlist; + + if (mPlaylist->isComplete() || mPlaylist->isEvent()) { + updateDuration(); + } + } + + mLastPlaylistFetchTimeUs = ALooper::GetNowUs(); + } + + int32_t firstSeqNumberInPlaylist; + if (mPlaylist->meta() == NULL || !mPlaylist->meta()->findInt32( + "media-sequence", &firstSeqNumberInPlaylist)) { + firstSeqNumberInPlaylist = 0; + } + + bool seekDiscontinuity = false; + bool explicitDiscontinuity = false; + + const int32_t lastSeqNumberInPlaylist = + firstSeqNumberInPlaylist + (int32_t)mPlaylist->size() - 1; + + if (mSeqNumber < 0) { + CHECK_GE(mStartTimeUs, 0ll); + + if (mPlaylist->isComplete() || mPlaylist->isEvent()) { + mSeqNumber = getSeqNumberForTime(mStartTimeUs); + } else { + // If this is a live session, start 3 segments from the end. + mSeqNumber = lastSeqNumberInPlaylist - 3; + if (mSeqNumber < firstSeqNumberInPlaylist) { + mSeqNumber = firstSeqNumberInPlaylist; + } + } + + mStartTimeUs = -1ll; + } + + if (mSeqNumber < firstSeqNumberInPlaylist + || mSeqNumber > lastSeqNumberInPlaylist) { + if (!mPlaylist->isComplete() && mNumRetries < kMaxNumRetries) { + ++mNumRetries; + + if (mSeqNumber > lastSeqNumberInPlaylist) { + mLastPlaylistFetchTimeUs = -1; + postMonitorQueue(3000000ll); + return; + } + + // we've missed the boat, let's start from the lowest sequence + // number available and signal a discontinuity. + + ALOGI("We've missed the boat, restarting playback."); + mSeqNumber = lastSeqNumberInPlaylist; + explicitDiscontinuity = true; + + // fall through + } else { + ALOGE("Cannot find sequence number %d in playlist " + "(contains %d - %d)", + mSeqNumber, firstSeqNumberInPlaylist, + firstSeqNumberInPlaylist + mPlaylist->size() - 1); + + notifyError(ERROR_END_OF_STREAM); + return; + } + } + + mNumRetries = 0; + + AString uri; + sp<AMessage> itemMeta; + CHECK(mPlaylist->itemAt( + mSeqNumber - firstSeqNumberInPlaylist, + &uri, + &itemMeta)); + + int32_t val; + if (itemMeta->findInt32("discontinuity", &val) && val != 0) { + explicitDiscontinuity = true; + } + + int64_t range_offset, range_length; + if (!itemMeta->findInt64("range-offset", &range_offset) + || !itemMeta->findInt64("range-length", &range_length)) { + range_offset = 0; + range_length = -1; + } + + ALOGV("fetching segment %d from (%d .. %d)", + mSeqNumber, firstSeqNumberInPlaylist, lastSeqNumberInPlaylist); + + ALOGV("fetching '%s'", uri.c_str()); + + sp<ABuffer> buffer; + status_t err = mSession->fetchFile( + uri.c_str(), &buffer, range_offset, range_length); + + if (err != OK) { + ALOGE("failed to fetch .ts segment at url '%s'", uri.c_str()); + notifyError(err); + return; + } + + CHECK(buffer != NULL); + + err = decryptBuffer(mSeqNumber - firstSeqNumberInPlaylist, buffer); + + if (err != OK) { + ALOGE("decryptBuffer failed w/ error %d", err); + + notifyError(err); + return; + } + + if (mStartup || seekDiscontinuity || explicitDiscontinuity) { + // Signal discontinuity. + + if (mPlaylist->isComplete() || mPlaylist->isEvent()) { + // If this was a live event this made no sense since + // we don't have access to all the segment before the current + // one. + mNextPTSTimeUs = getSegmentStartTimeUs(mSeqNumber); + } + + if (seekDiscontinuity || explicitDiscontinuity) { + ALOGI("queueing discontinuity (seek=%d, explicit=%d)", + seekDiscontinuity, explicitDiscontinuity); + + queueDiscontinuity( + explicitDiscontinuity + ? ATSParser::DISCONTINUITY_FORMATCHANGE + : ATSParser::DISCONTINUITY_SEEK, + NULL /* extra */); + } + } + + err = extractAndQueueAccessUnits(buffer, itemMeta); + + if (err != OK) { + notifyError(err); + return; + } + + ++mSeqNumber; + + postMonitorQueue(); + + mStartup = false; +} + +int32_t PlaylistFetcher::getSeqNumberForTime(int64_t timeUs) const { + int32_t firstSeqNumberInPlaylist; + if (mPlaylist->meta() == NULL || !mPlaylist->meta()->findInt32( + "media-sequence", &firstSeqNumberInPlaylist)) { + firstSeqNumberInPlaylist = 0; + } + + size_t index = 0; + int64_t segmentStartUs = 0; + while (index < mPlaylist->size()) { + sp<AMessage> itemMeta; + CHECK(mPlaylist->itemAt( + index, NULL /* uri */, &itemMeta)); + + int64_t itemDurationUs; + CHECK(itemMeta->findInt64("durationUs", &itemDurationUs)); + + if (timeUs < segmentStartUs + itemDurationUs) { + break; + } + + segmentStartUs += itemDurationUs; + ++index; + } + + if (index >= mPlaylist->size()) { + index = mPlaylist->size() - 1; + } + + return firstSeqNumberInPlaylist + index; +} + +status_t PlaylistFetcher::extractAndQueueAccessUnits( + const sp<ABuffer> &buffer, const sp<AMessage> &itemMeta) { + if (buffer->size() > 0 && buffer->data()[0] == 0x47) { + // Let's assume this is an MPEG2 transport stream. + + if ((buffer->size() % 188) != 0) { + ALOGE("MPEG2 transport stream is not an even multiple of 188 " + "bytes in length."); + return ERROR_MALFORMED; + } + + if (mTSParser == NULL) { + mTSParser = new ATSParser; + } + + if (mNextPTSTimeUs >= 0ll) { + sp<AMessage> extra = new AMessage; + extra->setInt64(IStreamListener::kKeyMediaTimeUs, mNextPTSTimeUs); + + mTSParser->signalDiscontinuity( + ATSParser::DISCONTINUITY_SEEK, extra); + + mNextPTSTimeUs = -1ll; + } + + size_t offset = 0; + while (offset < buffer->size()) { + status_t err = mTSParser->feedTSPacket(buffer->data() + offset, 188); + + if (err != OK) { + return err; + } + + offset += 188; + } + + for (size_t i = mPacketSources.size(); i-- > 0;) { + sp<AnotherPacketSource> packetSource = mPacketSources.valueAt(i); + + ATSParser::SourceType type; + switch (mPacketSources.keyAt(i)) { + case LiveSession::STREAMTYPE_VIDEO: + type = ATSParser::VIDEO; + break; + + case LiveSession::STREAMTYPE_AUDIO: + type = ATSParser::AUDIO; + break; + + case LiveSession::STREAMTYPE_SUBTITLES: + { + ALOGE("MPEG2 Transport streams do not contain subtitles."); + return ERROR_MALFORMED; + break; + } + + default: + TRESPASS(); + } + + sp<AnotherPacketSource> source = + static_cast<AnotherPacketSource *>( + mTSParser->getSource(type).get()); + + if (source == NULL) { + ALOGW("MPEG2 Transport stream does not contain %s data.", + type == ATSParser::VIDEO ? "video" : "audio"); + + mStreamTypeMask &= ~mPacketSources.keyAt(i); + mPacketSources.removeItemsAt(i); + continue; + } + + sp<ABuffer> accessUnit; + status_t finalResult; + while (source->hasBufferAvailable(&finalResult) + && source->dequeueAccessUnit(&accessUnit) == OK) { + // Note that we do NOT dequeue any discontinuities. + + packetSource->queueAccessUnit(accessUnit); + } + + if (packetSource->getFormat() == NULL) { + packetSource->setFormat(source->getFormat()); + } + } + + return OK; + } else if (buffer->size() >= 7 && !memcmp("WEBVTT\n", buffer->data(), 7)) { + if (mStreamTypeMask != LiveSession::STREAMTYPE_SUBTITLES) { + ALOGE("This stream only contains subtitles."); + return ERROR_MALFORMED; + } + + const sp<AnotherPacketSource> packetSource = + mPacketSources.valueFor(LiveSession::STREAMTYPE_SUBTITLES); + + int64_t durationUs; + CHECK(itemMeta->findInt64("durationUs", &durationUs)); + buffer->meta()->setInt64("timeUs", getSegmentStartTimeUs(mSeqNumber)); + buffer->meta()->setInt64("durationUs", durationUs); + + packetSource->queueAccessUnit(buffer); + return OK; + } + + if (mNextPTSTimeUs >= 0ll) { + mFirstPTSValid = false; + mAbsoluteTimeAnchorUs = mNextPTSTimeUs; + mNextPTSTimeUs = -1ll; + } + + // This better be an ISO 13818-7 (AAC) or ISO 13818-1 (MPEG) audio + // stream prefixed by an ID3 tag. + + bool firstID3Tag = true; + uint64_t PTS = 0; + + for (;;) { + // Make sure to skip all ID3 tags preceding the audio data. + // At least one must be present to provide the PTS timestamp. + + ID3 id3(buffer->data(), buffer->size(), true /* ignoreV1 */); + if (!id3.isValid()) { + if (firstID3Tag) { + ALOGE("Unable to parse ID3 tag."); + return ERROR_MALFORMED; + } else { + break; + } + } + + if (firstID3Tag) { + bool found = false; + + ID3::Iterator it(id3, "PRIV"); + while (!it.done()) { + size_t length; + const uint8_t *data = it.getData(&length); + + static const char *kMatchName = + "com.apple.streaming.transportStreamTimestamp"; + static const size_t kMatchNameLen = strlen(kMatchName); + + if (length == kMatchNameLen + 1 + 8 + && !strncmp((const char *)data, kMatchName, kMatchNameLen)) { + found = true; + PTS = U64_AT(&data[kMatchNameLen + 1]); + } + + it.next(); + } + + if (!found) { + ALOGE("Unable to extract transportStreamTimestamp from ID3 tag."); + return ERROR_MALFORMED; + } + } + + // skip the ID3 tag + buffer->setRange( + buffer->offset() + id3.rawSize(), buffer->size() - id3.rawSize()); + + firstID3Tag = false; + } + + if (!mFirstPTSValid) { + mFirstPTSValid = true; + mFirstPTS = PTS; + } + PTS -= mFirstPTS; + + int64_t timeUs = (PTS * 100ll) / 9ll + mAbsoluteTimeAnchorUs; + + if (mStreamTypeMask != LiveSession::STREAMTYPE_AUDIO) { + ALOGW("This stream only contains audio data!"); + + mStreamTypeMask &= LiveSession::STREAMTYPE_AUDIO; + + if (mStreamTypeMask == 0) { + return OK; + } + } + + sp<AnotherPacketSource> packetSource = + mPacketSources.valueFor(LiveSession::STREAMTYPE_AUDIO); + + if (packetSource->getFormat() == NULL && buffer->size() >= 7) { + ABitReader bits(buffer->data(), buffer->size()); + + // adts_fixed_header + + CHECK_EQ(bits.getBits(12), 0xfffu); + bits.skipBits(3); // ID, layer + bool protection_absent = bits.getBits(1) != 0; + + unsigned profile = bits.getBits(2); + CHECK_NE(profile, 3u); + unsigned sampling_freq_index = bits.getBits(4); + bits.getBits(1); // private_bit + unsigned channel_configuration = bits.getBits(3); + CHECK_NE(channel_configuration, 0u); + bits.skipBits(2); // original_copy, home + + sp<MetaData> meta = MakeAACCodecSpecificData( + profile, sampling_freq_index, channel_configuration); + + meta->setInt32(kKeyIsADTS, true); + + packetSource->setFormat(meta); + } + + int64_t numSamples = 0ll; + int32_t sampleRate; + CHECK(packetSource->getFormat()->findInt32(kKeySampleRate, &sampleRate)); + + size_t offset = 0; + while (offset < buffer->size()) { + const uint8_t *adtsHeader = buffer->data() + offset; + CHECK_LT(offset + 5, buffer->size()); + + unsigned aac_frame_length = + ((adtsHeader[3] & 3) << 11) + | (adtsHeader[4] << 3) + | (adtsHeader[5] >> 5); + + CHECK_LE(offset + aac_frame_length, buffer->size()); + + sp<ABuffer> unit = new ABuffer(aac_frame_length); + memcpy(unit->data(), adtsHeader, aac_frame_length); + + int64_t unitTimeUs = timeUs + numSamples * 1000000ll / sampleRate; + unit->meta()->setInt64("timeUs", unitTimeUs); + + // Each AAC frame encodes 1024 samples. + numSamples += 1024; + + packetSource->queueAccessUnit(unit); + + offset += aac_frame_length; + } + + return OK; +} + +void PlaylistFetcher::updateDuration() { + int64_t durationUs = 0ll; + for (size_t index = 0; index < mPlaylist->size(); ++index) { + sp<AMessage> itemMeta; + CHECK(mPlaylist->itemAt( + index, NULL /* uri */, &itemMeta)); + + int64_t itemDurationUs; + CHECK(itemMeta->findInt64("durationUs", &itemDurationUs)); + + durationUs += itemDurationUs; + } + + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("what", kWhatDurationUpdate); + msg->setInt64("durationUs", durationUs); + msg->post(); +} + +} // namespace android diff --git a/media/libstagefright/httplive/PlaylistFetcher.h b/media/libstagefright/httplive/PlaylistFetcher.h new file mode 100644 index 0000000..1648e02 --- /dev/null +++ b/media/libstagefright/httplive/PlaylistFetcher.h @@ -0,0 +1,156 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PLAYLIST_FETCHER_H_ + +#define PLAYLIST_FETCHER_H_ + +#include <media/stagefright/foundation/AHandler.h> + +#include "mpeg2ts/ATSParser.h" +#include "LiveSession.h" + +namespace android { + +struct ABuffer; +struct AnotherPacketSource; +struct DataSource; +struct HTTPBase; +struct LiveDataSource; +struct M3UParser; +struct String8; + +struct PlaylistFetcher : public AHandler { + enum { + kWhatStarted, + kWhatPaused, + kWhatStopped, + kWhatError, + kWhatDurationUpdate, + kWhatTemporarilyDoneFetching, + kWhatPrepared, + kWhatPreparationFailed, + }; + + PlaylistFetcher( + const sp<AMessage> ¬ify, + const sp<LiveSession> &session, + const char *uri); + + sp<DataSource> getDataSource(); + + void startAsync( + const sp<AnotherPacketSource> &audioSource, + const sp<AnotherPacketSource> &videoSource, + const sp<AnotherPacketSource> &subtitleSource, + int64_t startTimeUs = -1ll); + + void pauseAsync(); + + void stopAsync(); + +protected: + virtual ~PlaylistFetcher(); + virtual void onMessageReceived(const sp<AMessage> &msg); + +private: + enum { + kMaxNumRetries = 5, + }; + + enum { + kWhatStart = 'strt', + kWhatPause = 'paus', + kWhatStop = 'stop', + kWhatMonitorQueue = 'moni', + }; + + static const int64_t kMinBufferedDurationUs; + + sp<AMessage> mNotify; + sp<LiveSession> mSession; + AString mURI; + + uint32_t mStreamTypeMask; + int64_t mStartTimeUs; + + KeyedVector<LiveSession::StreamType, sp<AnotherPacketSource> > + mPacketSources; + + KeyedVector<AString, sp<ABuffer> > mAESKeyForURI; + + int64_t mLastPlaylistFetchTimeUs; + sp<M3UParser> mPlaylist; + int32_t mSeqNumber; + int32_t mNumRetries; + bool mStartup; + int64_t mNextPTSTimeUs; + + int32_t mMonitorQueueGeneration; + + enum RefreshState { + INITIAL_MINIMUM_RELOAD_DELAY, + FIRST_UNCHANGED_RELOAD_ATTEMPT, + SECOND_UNCHANGED_RELOAD_ATTEMPT, + THIRD_UNCHANGED_RELOAD_ATTEMPT + }; + RefreshState mRefreshState; + + uint8_t mPlaylistHash[16]; + + sp<ATSParser> mTSParser; + + bool mFirstPTSValid; + uint64_t mFirstPTS; + int64_t mAbsoluteTimeAnchorUs; + + status_t decryptBuffer( + size_t playlistIndex, const sp<ABuffer> &buffer); + + void postMonitorQueue(int64_t delayUs = 0); + void cancelMonitorQueue(); + + bool timeToRefreshPlaylist(int64_t nowUs) const; + + // Returns the media time in us of the segment specified by seqNumber. + // This is computed by summing the durations of all segments before it. + int64_t getSegmentStartTimeUs(int32_t seqNumber) const; + + status_t onStart(const sp<AMessage> &msg); + void onPause(); + void onStop(); + void onMonitorQueue(); + void onDownloadNext(); + + status_t extractAndQueueAccessUnits( + const sp<ABuffer> &buffer, const sp<AMessage> &itemMeta); + + void notifyError(status_t err); + + void queueDiscontinuity( + ATSParser::DiscontinuityType type, const sp<AMessage> &extra); + + int32_t getSeqNumberForTime(int64_t timeUs) const; + + void updateDuration(); + + DISALLOW_EVIL_CONSTRUCTORS(PlaylistFetcher); +}; + +} // namespace android + +#endif // PLAYLIST_FETCHER_H_ + diff --git a/media/libstagefright/id3/Android.mk b/media/libstagefright/id3/Android.mk index 80a1a3a..bf6f7bb 100644 --- a/media/libstagefright/id3/Android.mk +++ b/media/libstagefright/id3/Android.mk @@ -21,7 +21,7 @@ LOCAL_SHARED_LIBRARIES := \ LOCAL_STATIC_LIBRARIES := \ libstagefright_id3 -LOCAL_MODULE_TAGS := debug +LOCAL_MODULE_TAGS := optional LOCAL_MODULE := testid3 diff --git a/media/libstagefright/id3/ID3.cpp b/media/libstagefright/id3/ID3.cpp index 22c2f5a..34d671a 100644 --- a/media/libstagefright/id3/ID3.cpp +++ b/media/libstagefright/id3/ID3.cpp @@ -30,12 +30,55 @@ namespace android { static const size_t kMaxMetadataSize = 3 * 1024 * 1024; +struct MemorySource : public DataSource { + MemorySource(const uint8_t *data, size_t size) + : mData(data), + mSize(size) { + } + + virtual status_t initCheck() const { + return OK; + } + + virtual ssize_t readAt(off64_t offset, void *data, size_t size) { + off64_t available = (offset >= mSize) ? 0ll : mSize - offset; + + size_t copy = (available > size) ? size : available; + memcpy(data, mData + offset, copy); + + return copy; + } + +private: + const uint8_t *mData; + size_t mSize; + + DISALLOW_EVIL_CONSTRUCTORS(MemorySource); +}; + ID3::ID3(const sp<DataSource> &source, bool ignoreV1) : mIsValid(false), mData(NULL), mSize(0), mFirstFrameOffset(0), - mVersion(ID3_UNKNOWN) { + mVersion(ID3_UNKNOWN), + mRawSize(0) { + mIsValid = parseV2(source); + + if (!mIsValid && !ignoreV1) { + mIsValid = parseV1(source); + } +} + +ID3::ID3(const uint8_t *data, size_t size, bool ignoreV1) + : mIsValid(false), + mData(NULL), + mSize(0), + mFirstFrameOffset(0), + mVersion(ID3_UNKNOWN), + mRawSize(0) { + sp<MemorySource> source = new MemorySource(data, size); + mIsValid = parseV2(source); if (!mIsValid && !ignoreV1) { @@ -140,6 +183,7 @@ struct id3_header { } mSize = size; + mRawSize = mSize + sizeof(header); if (source->readAt(sizeof(header), mData, mSize) != (ssize_t)mSize) { free(mData); @@ -313,17 +357,22 @@ bool ID3::removeUnsynchronizationV2_4(bool iTunesHack) { } if (flags & 2) { - // Unsynchronization added. + // This file has "unsynchronization", so we have to replace occurrences + // of 0xff 0x00 with just 0xff in order to get the real data. + size_t readOffset = offset + 11; + size_t writeOffset = offset + 11; for (size_t i = 0; i + 1 < dataSize; ++i) { - if (mData[offset + 10 + i] == 0xff - && mData[offset + 11 + i] == 0x00) { - memmove(&mData[offset + 11 + i], &mData[offset + 12 + i], - mSize - offset - 12 - i); + if (mData[readOffset - 1] == 0xff + && mData[readOffset] == 0x00) { + ++readOffset; --mSize; --dataSize; } + mData[writeOffset++] = mData[readOffset++]; } + // move the remaining data following this frame + memmove(&mData[writeOffset], &mData[readOffset], oldSize - readOffset); flags &= ~2; } @@ -505,7 +554,7 @@ void ID3::Iterator::getstring(String8 *id, bool otherdata) const { int32_t i = n - 4; while(--i >= 0 && *++frameData != 0) ; int skipped = (frameData - mFrameData); - if (skipped >= n) { + if (skipped >= (int)n) { return; } n -= skipped; diff --git a/media/libstagefright/include/AwesomePlayer.h b/media/libstagefright/include/AwesomePlayer.h index 2306f31..271df8e 100644 --- a/media/libstagefright/include/AwesomePlayer.h +++ b/media/libstagefright/include/AwesomePlayer.h @@ -25,6 +25,7 @@ #include <media/stagefright/DataSource.h> #include <media/stagefright/OMXClient.h> #include <media/stagefright/TimeSource.h> +#include <media/stagefright/MetaData.h> #include <utils/threads.h> #include <drm/DrmManagerClient.h> @@ -100,7 +101,7 @@ struct AwesomePlayer { void postAudioEOS(int64_t delayUs = 0ll); void postAudioSeekComplete(); - + void postAudioTearDown(); status_t dump(int fd, const Vector<String16> &args) const; private: @@ -168,9 +169,12 @@ private: sp<AwesomeRenderer> mVideoRenderer; bool mVideoRenderingStarted; bool mVideoRendererIsPreview; + int32_t mMediaRenderingStartGeneration; + int32_t mStartGeneration; ssize_t mActiveAudioTrackIndex; sp<MediaSource> mAudioTrack; + sp<MediaSource> mOmxSource; sp<MediaSource> mAudioSource; AudioPlayer *mAudioPlayer; int64_t mDurationUs; @@ -211,7 +215,8 @@ private: bool mAudioStatusEventPending; sp<TimedEventQueue::Event> mVideoLagEvent; bool mVideoLagEventPending; - + sp<TimedEventQueue::Event> mAudioTearDownEvent; + bool mAudioTearDownEventPending; sp<TimedEventQueue::Event> mAsyncPrepareEvent; Condition mPreparedCondition; bool mIsAsyncPrepare; @@ -223,6 +228,8 @@ private: void postStreamDoneEvent_l(status_t status); void postCheckAudioStatusEvent(int64_t delayUs); void postVideoLagEvent_l(); + void postAudioTearDownEvent(int64_t delayUs); + status_t play_l(); MediaBuffer *mVideoBuffer; @@ -257,6 +264,7 @@ private: void setAudioSource(sp<MediaSource> source); status_t initAudioDecoder(); + void setVideoSource(sp<MediaSource> source); status_t initVideoDecoder(uint32_t flags = 0); @@ -273,6 +281,9 @@ private: void abortPrepare(status_t err); void finishAsyncPrepare_l(); void onVideoLagUpdate(); + void onAudioTearDownEvent(); + + void beginPrepareAsync_l(); bool getCachedDuration_l(int64_t *durationUs, bool *eos); @@ -285,6 +296,8 @@ private: void finishSeekIfNecessary(int64_t videoTimeUs); void ensureCacheIsFetching_l(); + void notifyIfMediaStarted_l(); + void createAudioPlayer_l(); status_t startAudioPlayer_l(bool sendErrorNotification = true); void shutdownVideoDecoder_l(); @@ -327,6 +340,11 @@ private: Vector<TrackStat> mTracks; } mStats; + bool mOffloadAudio; + bool mAudioTearDown; + bool mAudioTearDownWasPlaying; + int64_t mAudioTearDownPosition; + status_t setVideoScalingMode(int32_t mode); status_t setVideoScalingMode_l(int32_t mode); status_t getTrackInfo(Parcel* reply) const; diff --git a/media/libstagefright/include/ESDS.h b/media/libstagefright/include/ESDS.h index 3a79951..2f40dae 100644 --- a/media/libstagefright/include/ESDS.h +++ b/media/libstagefright/include/ESDS.h @@ -33,6 +33,9 @@ public: status_t getObjectTypeIndication(uint8_t *objectTypeIndication) const; status_t getCodecSpecificInfo(const void **data, size_t *size) const; + status_t getCodecSpecificOffset(size_t *offset, size_t *size) const; + status_t getBitRate(uint32_t *brateMax, uint32_t *brateAvg) const; + status_t getStreamType(uint8_t *streamType) const; private: enum { @@ -49,6 +52,9 @@ private: size_t mDecoderSpecificOffset; size_t mDecoderSpecificLength; uint8_t mObjectTypeIndication; + uint8_t mStreamType; + uint32_t mBitRateMax; + uint32_t mBitRateAvg; status_t skipDescriptorHeader( size_t offset, size_t size, diff --git a/media/libstagefright/include/HTTPBase.h b/media/libstagefright/include/HTTPBase.h index c2dc351..d4b7f9f 100644 --- a/media/libstagefright/include/HTTPBase.h +++ b/media/libstagefright/include/HTTPBase.h @@ -59,6 +59,9 @@ struct HTTPBase : public DataSource { static void RegisterSocketUserTag(int sockfd, uid_t uid, uint32_t kTag); static void UnRegisterSocketUserTag(int sockfd); + static void RegisterSocketUserMark(int sockfd, uid_t uid); + static void UnRegisterSocketUserMark(int sockfd); + protected: void addBandwidthMeasurement(size_t numBytes, int64_t delayUs); diff --git a/media/libstagefright/include/ID3.h b/media/libstagefright/include/ID3.h index 3028f56..cca83ab 100644 --- a/media/libstagefright/include/ID3.h +++ b/media/libstagefright/include/ID3.h @@ -36,6 +36,7 @@ struct ID3 { }; ID3(const sp<DataSource> &source, bool ignoreV1 = false); + ID3(const uint8_t *data, size_t size, bool ignoreV1 = false); ~ID3(); bool isValid() const; @@ -71,6 +72,8 @@ struct ID3 { Iterator &operator=(const Iterator &); }; + size_t rawSize() const { return mRawSize; } + private: bool mIsValid; uint8_t *mData; @@ -78,6 +81,10 @@ private: size_t mFirstFrameOffset; Version mVersion; + // size of the ID3 tag including header before any unsynchronization. + // only valid for IDV2+ + size_t mRawSize; + bool parseV1(const sp<DataSource> &source); bool parseV2(const sp<DataSource> &source); void removeUnsynchronization(); diff --git a/media/libstagefright/include/MPEG2TSExtractor.h b/media/libstagefright/include/MPEG2TSExtractor.h index fe74a42..c5e86a6 100644 --- a/media/libstagefright/include/MPEG2TSExtractor.h +++ b/media/libstagefright/include/MPEG2TSExtractor.h @@ -31,7 +31,6 @@ struct ATSParser; struct DataSource; struct MPEG2TSSource; struct String8; -struct LiveSession; struct MPEG2TSExtractor : public MediaExtractor { MPEG2TSExtractor(const sp<DataSource> &source); @@ -44,16 +43,12 @@ struct MPEG2TSExtractor : public MediaExtractor { virtual uint32_t flags() const; - void setLiveSession(const sp<LiveSession> &liveSession); - void seekTo(int64_t seekTimeUs); - private: friend struct MPEG2TSSource; mutable Mutex mLock; sp<DataSource> mDataSource; - sp<LiveSession> mLiveSession; sp<ATSParser> mParser; diff --git a/media/libstagefright/include/MPEG4Extractor.h b/media/libstagefright/include/MPEG4Extractor.h index 15253c3..bd5e4b9 100644 --- a/media/libstagefright/include/MPEG4Extractor.h +++ b/media/libstagefright/include/MPEG4Extractor.h @@ -82,6 +82,7 @@ private: sp<DataSource> mDataSource; status_t mInitCheck; bool mHasVideo; + uint32_t mHeaderTimescale; Track *mFirstTrack, *mLastTrack; diff --git a/media/libstagefright/include/OMX.h b/media/libstagefright/include/OMX.h index 24b8d98..31a5077 100644 --- a/media/libstagefright/include/OMX.h +++ b/media/libstagefright/include/OMX.h @@ -71,6 +71,10 @@ public: virtual status_t storeMetaDataInBuffers( node_id node, OMX_U32 port_index, OMX_BOOL enable); + virtual status_t prepareForAdaptivePlayback( + node_id node, OMX_U32 portIndex, OMX_BOOL enable, + OMX_U32 max_frame_width, OMX_U32 max_frame_height); + virtual status_t useBuffer( node_id node, OMX_U32 port_index, const sp<IMemory> ¶ms, buffer_id *buffer); @@ -79,6 +83,10 @@ public: node_id node, OMX_U32 port_index, const sp<GraphicBuffer> &graphicBuffer, buffer_id *buffer); + virtual status_t updateGraphicBufferInMeta( + node_id node, OMX_U32 port_index, + const sp<GraphicBuffer> &graphicBuffer, buffer_id buffer); + virtual status_t createInputSurface( node_id node, OMX_U32 port_index, sp<IGraphicBufferProducer> *bufferProducer); @@ -109,6 +117,13 @@ public: const char *parameter_name, OMX_INDEXTYPE *index); + virtual status_t setInternalOption( + node_id node, + OMX_U32 port_index, + InternalOptionType type, + const void *data, + size_t size); + virtual void binderDied(const wp<IBinder> &the_late_who); OMX_ERRORTYPE OnEvent( diff --git a/media/libstagefright/include/OMXNodeInstance.h b/media/libstagefright/include/OMXNodeInstance.h index 67aba6b..339179e 100644 --- a/media/libstagefright/include/OMXNodeInstance.h +++ b/media/libstagefright/include/OMXNodeInstance.h @@ -58,6 +58,10 @@ struct OMXNodeInstance { status_t storeMetaDataInBuffers(OMX_U32 portIndex, OMX_BOOL enable); + status_t prepareForAdaptivePlayback( + OMX_U32 portIndex, OMX_BOOL enable, + OMX_U32 maxFrameWidth, OMX_U32 maxFrameHeight); + status_t useBuffer( OMX_U32 portIndex, const sp<IMemory> ¶ms, OMX::buffer_id *buffer); @@ -66,6 +70,10 @@ struct OMXNodeInstance { OMX_U32 portIndex, const sp<GraphicBuffer> &graphicBuffer, OMX::buffer_id *buffer); + status_t updateGraphicBufferInMeta( + OMX_U32 portIndex, const sp<GraphicBuffer> &graphicBuffer, + OMX::buffer_id buffer); + status_t createInputSurface( OMX_U32 portIndex, sp<IGraphicBufferProducer> *bufferProducer); @@ -96,6 +104,12 @@ struct OMXNodeInstance { status_t getExtensionIndex( const char *parameterName, OMX_INDEXTYPE *index); + status_t setInternalOption( + OMX_U32 portIndex, + IOMX::InternalOptionType type, + const void *data, + size_t size); + void onMessage(const omx_message &msg); void onObserverDied(OMXMaster *master); void onGetHandleFailed(); diff --git a/media/libstagefright/include/SoftVideoDecoderOMXComponent.h b/media/libstagefright/include/SoftVideoDecoderOMXComponent.h new file mode 100644 index 0000000..d050fa6 --- /dev/null +++ b/media/libstagefright/include/SoftVideoDecoderOMXComponent.h @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SOFT_VIDEO_DECODER_OMX_COMPONENT_H_ + +#define SOFT_VIDEO_DECODER_OMX_COMPONENT_H_ + +#include "SimpleSoftOMXComponent.h" + +#include <media/stagefright/foundation/AHandlerReflector.h> +#include <media/IOMX.h> + +#include <utils/RefBase.h> +#include <utils/threads.h> +#include <utils/Vector.h> + +#define ARRAY_SIZE(a) (sizeof(a) / sizeof(*(a))) + +namespace android { + +struct SoftVideoDecoderOMXComponent : public SimpleSoftOMXComponent { + SoftVideoDecoderOMXComponent( + const char *name, + const char *componentRole, + OMX_VIDEO_CODINGTYPE codingType, + const CodecProfileLevel *profileLevels, + size_t numProfileLevels, + int32_t width, + int32_t height, + const OMX_CALLBACKTYPE *callbacks, + OMX_PTR appData, + OMX_COMPONENTTYPE **component); + +protected: + virtual void onPortEnableCompleted(OMX_U32 portIndex, bool enabled); + virtual void onReset(); + + virtual OMX_ERRORTYPE internalGetParameter( + OMX_INDEXTYPE index, OMX_PTR params); + + virtual OMX_ERRORTYPE internalSetParameter( + OMX_INDEXTYPE index, const OMX_PTR params); + + virtual OMX_ERRORTYPE getConfig( + OMX_INDEXTYPE index, OMX_PTR params); + + void initPorts(OMX_U32 numInputBuffers, + OMX_U32 inputBufferSize, + OMX_U32 numOutputBuffers, + const char *mimeType); + + virtual void updatePortDefinitions(); + + enum { + kInputPortIndex = 0, + kOutputPortIndex = 1, + kMaxPortIndex = 1, + }; + + uint32_t mWidth, mHeight; + uint32_t mCropLeft, mCropTop, mCropWidth, mCropHeight; + + enum { + NONE, + AWAITING_DISABLED, + AWAITING_ENABLED + } mOutputPortSettingsChange; + +private: + const char *mComponentRole; + OMX_VIDEO_CODINGTYPE mCodingType; + const CodecProfileLevel *mProfileLevels; + size_t mNumProfileLevels; + + DISALLOW_EVIL_CONSTRUCTORS(SoftVideoDecoderOMXComponent); +}; + +} // namespace android + +#endif // SOFT_VIDEO_DECODER_OMX_COMPONENT_H_ diff --git a/media/libstagefright/include/TimedEventQueue.h b/media/libstagefright/include/TimedEventQueue.h index 11f844c..4e49c83 100644 --- a/media/libstagefright/include/TimedEventQueue.h +++ b/media/libstagefright/include/TimedEventQueue.h @@ -23,6 +23,7 @@ #include <utils/List.h> #include <utils/RefBase.h> #include <utils/threads.h> +#include <powermanager/IPowerManager.h> namespace android { @@ -57,6 +58,21 @@ struct TimedEventQueue { Event &operator=(const Event &); }; + class PMDeathRecipient : public IBinder::DeathRecipient { + public: + PMDeathRecipient(TimedEventQueue *queue) : mQueue(queue) {} + virtual ~PMDeathRecipient() {} + + // IBinder::DeathRecipient + virtual void binderDied(const wp<IBinder>& who); + + private: + PMDeathRecipient(const PMDeathRecipient&); + PMDeathRecipient& operator = (const PMDeathRecipient&); + + TimedEventQueue *mQueue; + }; + TimedEventQueue(); ~TimedEventQueue(); @@ -96,6 +112,8 @@ struct TimedEventQueue { static int64_t getRealTimeUs(); + void clearPowerManager(); + private: struct QueueItem { sp<Event> event; @@ -118,11 +136,18 @@ private: bool mRunning; bool mStopped; + sp<IPowerManager> mPowerManager; + sp<IBinder> mWakeLockToken; + const sp<PMDeathRecipient> mDeathRecipient; + static void *ThreadWrapper(void *me); void threadEntry(); sp<Event> removeEventFromQueue_l(event_id id); + void acquireWakeLock_l(); + void releaseWakeLock_l(); + TimedEventQueue(const TimedEventQueue &); TimedEventQueue &operator=(const TimedEventQueue &); }; diff --git a/media/libstagefright/matroska/MatroskaExtractor.cpp b/media/libstagefright/matroska/MatroskaExtractor.cpp index b304749..d260d0f 100644 --- a/media/libstagefright/matroska/MatroskaExtractor.cpp +++ b/media/libstagefright/matroska/MatroskaExtractor.cpp @@ -870,7 +870,9 @@ void MatroskaExtractor::addTracks() { continue; } } else if (!strcmp("V_VP8", codecID)) { - meta->setCString(kKeyMIMEType, MEDIA_MIMETYPE_VIDEO_VPX); + meta->setCString(kKeyMIMEType, MEDIA_MIMETYPE_VIDEO_VP8); + } else if (!strcmp("V_VP9", codecID)) { + meta->setCString(kKeyMIMEType, MEDIA_MIMETYPE_VIDEO_VP9); } else { ALOGW("%s is not supported.", codecID); continue; diff --git a/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp b/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp index 3de3a61..3153c8b 100644 --- a/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp +++ b/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp @@ -32,9 +32,22 @@ const int64_t kNearEOSMarkUs = 2000000ll; // 2 secs AnotherPacketSource::AnotherPacketSource(const sp<MetaData> &meta) : mIsAudio(false), - mFormat(meta), + mFormat(NULL), mLastQueuedTimeUs(0), mEOSResult(OK) { + setFormat(meta); +} + +void AnotherPacketSource::setFormat(const sp<MetaData> &meta) { + CHECK(mFormat == NULL); + + mIsAudio = false; + + if (meta == NULL) { + return; + } + + mFormat = meta; const char *mime; CHECK(meta->findCString(kKeyMIMEType, &mime)); @@ -45,11 +58,6 @@ AnotherPacketSource::AnotherPacketSource(const sp<MetaData> &meta) } } -void AnotherPacketSource::setFormat(const sp<MetaData> &meta) { - CHECK(mFormat == NULL); - mFormat = meta; -} - AnotherPacketSource::~AnotherPacketSource() { } @@ -152,6 +160,15 @@ void AnotherPacketSource::queueAccessUnit(const sp<ABuffer> &buffer) { mCondition.signal(); } +void AnotherPacketSource::clear() { + Mutex::Autolock autoLock(mLock); + + mBuffers.clear(); + mEOSResult = OK; + + mFormat = NULL; +} + void AnotherPacketSource::queueDiscontinuity( ATSParser::DiscontinuityType type, const sp<AMessage> &extra) { diff --git a/media/libstagefright/mpeg2ts/AnotherPacketSource.h b/media/libstagefright/mpeg2ts/AnotherPacketSource.h index 1db4068..e16cf78 100644 --- a/media/libstagefright/mpeg2ts/AnotherPacketSource.h +++ b/media/libstagefright/mpeg2ts/AnotherPacketSource.h @@ -41,6 +41,8 @@ struct AnotherPacketSource : public MediaSource { virtual status_t read( MediaBuffer **buffer, const ReadOptions *options = NULL); + void clear(); + bool hasBufferAvailable(status_t *finalResult); // Returns the difference between the last and the first queued diff --git a/media/libstagefright/mpeg2ts/ESQueue.cpp b/media/libstagefright/mpeg2ts/ESQueue.cpp index 9f3b19c..8f9c9c8 100644 --- a/media/libstagefright/mpeg2ts/ESQueue.cpp +++ b/media/libstagefright/mpeg2ts/ESQueue.cpp @@ -504,15 +504,11 @@ int64_t ElementaryStreamQueue::fetchTimestamp(size_t size) { if (first) { timeUs = info->mTimestampUs; + first = false; } if (info->mLength > size) { info->mLength -= size; - - if (first) { - info->mTimestampUs = -1; - } - size = 0; } else { size -= info->mLength; @@ -521,7 +517,6 @@ int64_t ElementaryStreamQueue::fetchTimestamp(size_t size) { info = NULL; } - first = false; } if (timeUs == 0ll) { diff --git a/media/libstagefright/mpeg2ts/MPEG2TSExtractor.cpp b/media/libstagefright/mpeg2ts/MPEG2TSExtractor.cpp index e1589b4..d449c34 100644 --- a/media/libstagefright/mpeg2ts/MPEG2TSExtractor.cpp +++ b/media/libstagefright/mpeg2ts/MPEG2TSExtractor.cpp @@ -19,7 +19,6 @@ #include <utils/Log.h> #include "include/MPEG2TSExtractor.h" -#include "include/LiveSession.h" #include "include/NuCachedSource2.h" #include <media/stagefright/foundation/ADebug.h> @@ -79,15 +78,7 @@ status_t MPEG2TSSource::stop() { } sp<MetaData> MPEG2TSSource::getFormat() { - sp<MetaData> meta = mImpl->getFormat(); - - int64_t durationUs; - if (mExtractor->mLiveSession != NULL - && mExtractor->mLiveSession->getDuration(&durationUs) == OK) { - meta->setInt64(kKeyDuration, durationUs); - } - - return meta; + return mImpl->getFormat(); } status_t MPEG2TSSource::read( @@ -97,7 +88,7 @@ status_t MPEG2TSSource::read( int64_t seekTimeUs; ReadOptions::SeekMode seekMode; if (mSeekable && options && options->getSeekTo(&seekTimeUs, &seekMode)) { - mExtractor->seekTo(seekTimeUs); + return ERROR_UNSUPPORTED; } status_t finalResult; @@ -216,32 +207,8 @@ status_t MPEG2TSExtractor::feedMore() { return mParser->feedTSPacket(packet, kTSPacketSize); } -void MPEG2TSExtractor::setLiveSession(const sp<LiveSession> &liveSession) { - Mutex::Autolock autoLock(mLock); - - mLiveSession = liveSession; -} - -void MPEG2TSExtractor::seekTo(int64_t seekTimeUs) { - Mutex::Autolock autoLock(mLock); - - if (mLiveSession == NULL) { - return; - } - - mLiveSession->seekTo(seekTimeUs); -} - uint32_t MPEG2TSExtractor::flags() const { - Mutex::Autolock autoLock(mLock); - - uint32_t flags = CAN_PAUSE; - - if (mLiveSession != NULL && mLiveSession->isSeekable()) { - flags |= CAN_SEEK_FORWARD | CAN_SEEK_BACKWARD | CAN_SEEK; - } - - return flags; + return CAN_PAUSE; } //////////////////////////////////////////////////////////////////////////////// diff --git a/media/libstagefright/omx/Android.mk b/media/libstagefright/omx/Android.mk index a8b4939..cd912e7 100644 --- a/media/libstagefright/omx/Android.mk +++ b/media/libstagefright/omx/Android.mk @@ -9,6 +9,7 @@ LOCAL_SRC_FILES:= \ SimpleSoftOMXComponent.cpp \ SoftOMXComponent.cpp \ SoftOMXPlugin.cpp \ + SoftVideoDecoderOMXComponent.cpp \ LOCAL_C_INCLUDES += \ $(TOP)/frameworks/av/media/libstagefright \ diff --git a/media/libstagefright/omx/GraphicBufferSource.cpp b/media/libstagefright/omx/GraphicBufferSource.cpp index ef27879..cf43e94 100644 --- a/media/libstagefright/omx/GraphicBufferSource.cpp +++ b/media/libstagefright/omx/GraphicBufferSource.cpp @@ -18,12 +18,13 @@ //#define LOG_NDEBUG 0 #include <utils/Log.h> -#include <GraphicBufferSource.h> +#include "GraphicBufferSource.h" #include <OMX_Core.h> #include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> -#include <MetadataBufferType.h> +#include <media/hardware/MetadataBufferType.h> #include <ui/GraphicBuffer.h> namespace android { @@ -36,9 +37,16 @@ GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance, mInitCheck(UNKNOWN_ERROR), mNodeInstance(nodeInstance), mExecuting(false), + mSuspended(false), mNumFramesAvailable(0), mEndOfStream(false), - mEndOfStreamSent(false) { + mEndOfStreamSent(false), + mRepeatAfterUs(-1ll), + mRepeatLastFrameGeneration(0), + mLatestSubmittedBufferId(-1), + mLatestSubmittedBufferFrameNum(0), + mLatestSubmittedBufferUseCount(0), + mRepeatBufferDeferred(false) { ALOGV("GraphicBufferSource w=%u h=%u c=%u", bufferWidth, bufferHeight, bufferCount); @@ -51,10 +59,9 @@ GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance, String8 name("GraphicBufferSource"); - mBufferQueue = new BufferQueue(true); + mBufferQueue = new BufferQueue(); mBufferQueue->setConsumerName(name); mBufferQueue->setDefaultBufferSize(bufferWidth, bufferHeight); - mBufferQueue->setSynchronousMode(true); mBufferQueue->setConsumerUsageBits(GRALLOC_USAGE_HW_VIDEO_ENCODER | GRALLOC_USAGE_HW_TEXTURE); @@ -69,13 +76,10 @@ GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance, // reference once the ctor ends, as that would cause the refcount of 'this' // dropping to 0 at the end of the ctor. Since all we need is a wp<...> // that's what we create. - wp<BufferQueue::ConsumerListener> listener; - listener = static_cast<BufferQueue::ConsumerListener*>(this); + wp<BufferQueue::ConsumerListener> listener = static_cast<BufferQueue::ConsumerListener*>(this); + sp<BufferQueue::ProxyConsumerListener> proxy = new BufferQueue::ProxyConsumerListener(listener); - sp<BufferQueue::ConsumerListener> proxy; - proxy = new BufferQueue::ProxyConsumerListener(listener); - - mInitCheck = mBufferQueue->consumerConnect(proxy); + mInitCheck = mBufferQueue->consumerConnect(proxy, false); if (mInitCheck != NO_ERROR) { ALOGE("Error connecting to BufferQueue: %s (%d)", strerror(-mInitCheck), mInitCheck); @@ -126,14 +130,40 @@ void GraphicBufferSource::omxExecuting() { if (mEndOfStream && mNumFramesAvailable == 0) { submitEndOfInputStream_l(); } + + if (mRepeatAfterUs > 0ll && mLooper == NULL) { + mReflector = new AHandlerReflector<GraphicBufferSource>(this); + + mLooper = new ALooper; + mLooper->registerHandler(mReflector); + mLooper->start(); + + if (mLatestSubmittedBufferId >= 0) { + sp<AMessage> msg = + new AMessage(kWhatRepeatLastFrame, mReflector->id()); + + msg->setInt32("generation", ++mRepeatLastFrameGeneration); + msg->post(mRepeatAfterUs); + } + } } void GraphicBufferSource::omxLoaded(){ Mutex::Autolock autoLock(mMutex); - ALOGV("--> loaded"); - CHECK(mExecuting); + if (!mExecuting) { + // This can happen if something failed very early. + ALOGW("Dropped back down to Loaded without Executing"); + } + + if (mLooper != NULL) { + mLooper->unregisterHandler(mReflector->id()); + mReflector.clear(); - ALOGV("Dropped down to loaded, avail=%d eos=%d eosSent=%d", + mLooper->stop(); + mLooper.clear(); + } + + ALOGV("--> loaded; avail=%d eos=%d eosSent=%d", mNumFramesAvailable, mEndOfStream, mEndOfStreamSent); // Codec is no longer executing. Discard all codec-related state. @@ -206,24 +236,19 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) { // Find matching entry in our cached copy of the BufferQueue slots. // If we find a match, release that slot. If we don't, the BufferQueue // has dropped that GraphicBuffer, and there's nothing for us to release. - // - // (We could store "id" in CodecBuffer and avoid the slot search.) - int id; - for (id = 0; id < BufferQueue::NUM_BUFFER_SLOTS; id++) { - if (mBufferSlot[id] == NULL) { - continue; + int id = codecBuffer.mBuf; + if (mBufferSlot[id] != NULL && + mBufferSlot[id]->handle == codecBuffer.mGraphicBuffer->handle) { + ALOGV("cbi %d matches bq slot %d, handle=%p", + cbi, id, mBufferSlot[id]->handle); + + if (id == mLatestSubmittedBufferId) { + CHECK_GT(mLatestSubmittedBufferUseCount--, 0); + } else { + mBufferQueue->releaseBuffer(id, codecBuffer.mFrameNumber, + EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE); } - - if (mBufferSlot[id]->handle == codecBuffer.mGraphicBuffer->handle) { - ALOGV("cbi %d matches bq slot %d, handle=%p", - cbi, id, mBufferSlot[id]->handle); - - mBufferQueue->releaseBuffer(id, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, - Fence::NO_FENCE); - break; - } - } - if (id == BufferQueue::NUM_BUFFER_SLOTS) { + } else { ALOGV("codecBufferEmptied: no match for emptied buffer in cbi %d", cbi); } @@ -242,13 +267,66 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) { // send that. ALOGV("buffer freed, EOS pending"); submitEndOfInputStream_l(); + } else if (mRepeatBufferDeferred) { + bool success = repeatLatestSubmittedBuffer_l(); + if (success) { + ALOGV("deferred repeatLatestSubmittedBuffer_l SUCCESS"); + } else { + ALOGV("deferred repeatLatestSubmittedBuffer_l FAILURE"); + } + mRepeatBufferDeferred = false; } + return; } +void GraphicBufferSource::suspend(bool suspend) { + Mutex::Autolock autoLock(mMutex); + + if (suspend) { + mSuspended = true; + + while (mNumFramesAvailable > 0) { + BufferQueue::BufferItem item; + status_t err = mBufferQueue->acquireBuffer(&item, 0); + + if (err == BufferQueue::NO_BUFFER_AVAILABLE) { + // shouldn't happen. + ALOGW("suspend: frame was not available"); + break; + } else if (err != OK) { + ALOGW("suspend: acquireBuffer returned err=%d", err); + break; + } + + --mNumFramesAvailable; + + mBufferQueue->releaseBuffer(item.mBuf, item.mFrameNumber, + EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, item.mFence); + } + return; + } + + mSuspended = false; + + if (mExecuting && mNumFramesAvailable == 0 && mRepeatBufferDeferred) { + if (repeatLatestSubmittedBuffer_l()) { + ALOGV("suspend/deferred repeatLatestSubmittedBuffer_l SUCCESS"); + + mRepeatBufferDeferred = false; + } else { + ALOGV("suspend/deferred repeatLatestSubmittedBuffer_l FAILURE"); + } + } +} + bool GraphicBufferSource::fillCodecBuffer_l() { CHECK(mExecuting && mNumFramesAvailable > 0); + if (mSuspended) { + return false; + } + int cbi = findAvailableCodecBuffer_l(); if (cbi < 0) { // No buffers available, bail. @@ -260,7 +338,7 @@ bool GraphicBufferSource::fillCodecBuffer_l() { ALOGV("fillCodecBuffer_l: acquiring buffer, avail=%d", mNumFramesAvailable); BufferQueue::BufferItem item; - status_t err = mBufferQueue->acquireBuffer(&item); + status_t err = mBufferQueue->acquireBuffer(&item, 0); if (err == BufferQueue::NO_BUFFER_AVAILABLE) { // shouldn't happen ALOGW("fillCodecBuffer_l: frame was not available"); @@ -287,18 +365,75 @@ bool GraphicBufferSource::fillCodecBuffer_l() { mBufferSlot[item.mBuf] = item.mGraphicBuffer; } - err = submitBuffer_l(mBufferSlot[item.mBuf], item.mTimestamp / 1000, cbi); + err = submitBuffer_l(item, cbi); if (err != OK) { ALOGV("submitBuffer_l failed, releasing bq buf %d", item.mBuf); - mBufferQueue->releaseBuffer(item.mBuf, EGL_NO_DISPLAY, - EGL_NO_SYNC_KHR, Fence::NO_FENCE); + mBufferQueue->releaseBuffer(item.mBuf, item.mFrameNumber, + EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE); } else { ALOGV("buffer submitted (bq %d, cbi %d)", item.mBuf, cbi); + setLatestSubmittedBuffer_l(item); } return true; } +bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() { + CHECK(mExecuting && mNumFramesAvailable == 0); + + if (mLatestSubmittedBufferId < 0 || mSuspended) { + return false; + } + + int cbi = findAvailableCodecBuffer_l(); + if (cbi < 0) { + // No buffers available, bail. + ALOGV("repeatLatestSubmittedBuffer_l: no codec buffers."); + return false; + } + + BufferQueue::BufferItem item; + item.mBuf = mLatestSubmittedBufferId; + item.mFrameNumber = mLatestSubmittedBufferFrameNum; + + status_t err = submitBuffer_l(item, cbi); + + if (err != OK) { + return false; + } + + ++mLatestSubmittedBufferUseCount; + + return true; +} + +void GraphicBufferSource::setLatestSubmittedBuffer_l( + const BufferQueue::BufferItem &item) { + ALOGV("setLatestSubmittedBuffer_l"); + + if (mLatestSubmittedBufferId >= 0) { + if (mLatestSubmittedBufferUseCount == 0) { + mBufferQueue->releaseBuffer( + mLatestSubmittedBufferId, + mLatestSubmittedBufferFrameNum, + EGL_NO_DISPLAY, + EGL_NO_SYNC_KHR, + Fence::NO_FENCE); + } + } + + mLatestSubmittedBufferId = item.mBuf; + mLatestSubmittedBufferFrameNum = item.mFrameNumber; + mLatestSubmittedBufferUseCount = 1; + mRepeatBufferDeferred = false; + + if (mReflector != NULL) { + sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id()); + msg->setInt32("generation", ++mRepeatLastFrameGeneration); + msg->post(mRepeatAfterUs); + } +} + status_t GraphicBufferSource::signalEndOfInputStream() { Mutex::Autolock autoLock(mMutex); ALOGV("signalEndOfInputStream: exec=%d avail=%d eos=%d", @@ -326,11 +461,13 @@ status_t GraphicBufferSource::signalEndOfInputStream() { return OK; } -status_t GraphicBufferSource::submitBuffer_l(sp<GraphicBuffer>& graphicBuffer, - int64_t timestampUsec, int cbi) { +status_t GraphicBufferSource::submitBuffer_l( + const BufferQueue::BufferItem &item, int cbi) { ALOGV("submitBuffer_l cbi=%d", cbi); CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi)); - codecBuffer.mGraphicBuffer = graphicBuffer; + codecBuffer.mGraphicBuffer = mBufferSlot[item.mBuf]; + codecBuffer.mBuf = item.mBuf; + codecBuffer.mFrameNumber = item.mFrameNumber; OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader; CHECK(header->nAllocLen >= 4 + sizeof(buffer_handle_t)); @@ -342,7 +479,7 @@ status_t GraphicBufferSource::submitBuffer_l(sp<GraphicBuffer>& graphicBuffer, status_t err = mNodeInstance->emptyDirectBuffer(header, 0, 4 + sizeof(buffer_handle_t), OMX_BUFFERFLAG_ENDOFFRAME, - timestampUsec); + item.mTimestamp / 1000); if (err != OK) { ALOGW("WARNING: emptyDirectBuffer failed: 0x%x", err); codecBuffer.mGraphicBuffer = NULL; @@ -423,22 +560,30 @@ void GraphicBufferSource::onFrameAvailable() { ALOGV("onFrameAvailable exec=%d avail=%d", mExecuting, mNumFramesAvailable); - if (mEndOfStream) { - // This should only be possible if a new buffer was queued after - // EOS was signaled, i.e. the app is misbehaving. - ALOGW("onFrameAvailable: EOS is set, ignoring frame"); + if (mEndOfStream || mSuspended) { + if (mEndOfStream) { + // This should only be possible if a new buffer was queued after + // EOS was signaled, i.e. the app is misbehaving. + + ALOGW("onFrameAvailable: EOS is set, ignoring frame"); + } else { + ALOGV("onFrameAvailable: suspended, ignoring frame"); + } BufferQueue::BufferItem item; - status_t err = mBufferQueue->acquireBuffer(&item); + status_t err = mBufferQueue->acquireBuffer(&item, 0); if (err == OK) { - mBufferQueue->releaseBuffer(item.mBuf, EGL_NO_DISPLAY, - EGL_NO_SYNC_KHR, item.mFence); + mBufferQueue->releaseBuffer(item.mBuf, item.mFrameNumber, + EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, item.mFence); } return; } mNumFramesAvailable++; + mRepeatBufferDeferred = false; + ++mRepeatLastFrameGeneration; + if (mExecuting) { fillCodecBuffer_l(); } @@ -464,4 +609,51 @@ void GraphicBufferSource::onBuffersReleased() { } } +status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs( + int64_t repeatAfterUs) { + Mutex::Autolock autoLock(mMutex); + + if (mExecuting || repeatAfterUs <= 0ll) { + return INVALID_OPERATION; + } + + mRepeatAfterUs = repeatAfterUs; + + return OK; +} + +void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatRepeatLastFrame: + { + Mutex::Autolock autoLock(mMutex); + + int32_t generation; + CHECK(msg->findInt32("generation", &generation)); + + if (generation != mRepeatLastFrameGeneration) { + // stale + break; + } + + if (!mExecuting || mNumFramesAvailable > 0) { + break; + } + + bool success = repeatLatestSubmittedBuffer_l(); + + if (success) { + ALOGV("repeatLatestSubmittedBuffer_l SUCCESS"); + } else { + ALOGV("repeatLatestSubmittedBuffer_l FAILURE"); + mRepeatBufferDeferred = true; + } + break; + } + + default: + TRESPASS(); + } +} + } // namespace android diff --git a/media/libstagefright/omx/GraphicBufferSource.h b/media/libstagefright/omx/GraphicBufferSource.h index 562d342..244a843 100644 --- a/media/libstagefright/omx/GraphicBufferSource.h +++ b/media/libstagefright/omx/GraphicBufferSource.h @@ -25,6 +25,8 @@ #include <OMX_Core.h> #include "../include/OMXNodeInstance.h" #include <media/stagefright/foundation/ABase.h> +#include <media/stagefright/foundation/AHandlerReflector.h> +#include <media/stagefright/foundation/ALooper.h> namespace android { @@ -85,6 +87,19 @@ public: // have a codec buffer ready, we just set the mEndOfStream flag. status_t signalEndOfInputStream(); + // If suspend is true, all incoming buffers (including those currently + // in the BufferQueue) will be discarded until the suspension is lifted. + void suspend(bool suspend); + + // Specifies the interval after which we requeue the buffer previously + // queued to the encoder. This is useful in the case of surface flinger + // providing the input surface if the resulting encoded stream is to + // be displayed "live". If we were not to push through the extra frame + // the decoder on the remote end would be unable to decode the latest frame. + // This API must be called before transitioning the encoder to "executing" + // state and once this behaviour is specified it cannot be reset. + status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs); + protected: // BufferQueue::ConsumerListener interface, called when a new frame of // data is available. If we're executing and a codec buffer is @@ -104,6 +119,13 @@ private: // (mGraphicBuffer == NULL) or in use by the codec. struct CodecBuffer { OMX_BUFFERHEADERTYPE* mHeader; + + // buffer producer's frame-number for buffer + uint64_t mFrameNumber; + + // buffer producer's buffer slot for buffer + int mBuf; + sp<GraphicBuffer> mGraphicBuffer; }; @@ -130,13 +152,15 @@ private: // Marks the mCodecBuffers entry as in-use, copies the GraphicBuffer // reference into the codec buffer, and submits the data to the codec. - status_t submitBuffer_l(sp<GraphicBuffer>& graphicBuffer, - int64_t timestampUsec, int cbi); + status_t submitBuffer_l(const BufferQueue::BufferItem &item, int cbi); // Submits an empty buffer, with the EOS flag set. Returns without // doing anything if we don't have a codec buffer available. void submitEndOfInputStream_l(); + void setLatestSubmittedBuffer_l(const BufferQueue::BufferItem &item); + bool repeatLatestSubmittedBuffer_l(); + // Lock, covers all member variables. mutable Mutex mMutex; @@ -149,6 +173,8 @@ private: // Set by omxExecuting() / omxIdling(). bool mExecuting; + bool mSuspended; + // We consume graphic buffers from this. sp<BufferQueue> mBufferQueue; @@ -169,6 +195,30 @@ private: // Tracks codec buffers. Vector<CodecBuffer> mCodecBuffers; + //// + friend class AHandlerReflector<GraphicBufferSource>; + + enum { + kWhatRepeatLastFrame, + }; + + int64_t mRepeatAfterUs; + + sp<ALooper> mLooper; + sp<AHandlerReflector<GraphicBufferSource> > mReflector; + + int32_t mRepeatLastFrameGeneration; + + int mLatestSubmittedBufferId; + uint64_t mLatestSubmittedBufferFrameNum; + int32_t mLatestSubmittedBufferUseCount; + + // The previously submitted buffer should've been repeated but + // no codec buffer was available at the time. + bool mRepeatBufferDeferred; + + void onMessageReceived(const sp<AMessage> &msg); + DISALLOW_EVIL_CONSTRUCTORS(GraphicBufferSource); }; diff --git a/media/libstagefright/omx/OMX.cpp b/media/libstagefright/omx/OMX.cpp index 3987ead..84a0e10 100644 --- a/media/libstagefright/omx/OMX.cpp +++ b/media/libstagefright/omx/OMX.cpp @@ -331,6 +331,13 @@ status_t OMX::storeMetaDataInBuffers( return findInstance(node)->storeMetaDataInBuffers(port_index, enable); } +status_t OMX::prepareForAdaptivePlayback( + node_id node, OMX_U32 portIndex, OMX_BOOL enable, + OMX_U32 maxFrameWidth, OMX_U32 maxFrameHeight) { + return findInstance(node)->prepareForAdaptivePlayback( + portIndex, enable, maxFrameWidth, maxFrameHeight); +} + status_t OMX::useBuffer( node_id node, OMX_U32 port_index, const sp<IMemory> ¶ms, buffer_id *buffer) { @@ -345,6 +352,13 @@ status_t OMX::useGraphicBuffer( port_index, graphicBuffer, buffer); } +status_t OMX::updateGraphicBufferInMeta( + node_id node, OMX_U32 port_index, + const sp<GraphicBuffer> &graphicBuffer, buffer_id buffer) { + return findInstance(node)->updateGraphicBufferInMeta( + port_index, graphicBuffer, buffer); +} + status_t OMX::createInputSurface( node_id node, OMX_U32 port_index, sp<IGraphicBufferProducer> *bufferProducer) { @@ -396,6 +410,15 @@ status_t OMX::getExtensionIndex( parameter_name, index); } +status_t OMX::setInternalOption( + node_id node, + OMX_U32 port_index, + InternalOptionType type, + const void *data, + size_t size) { + return findInstance(node)->setInternalOption(port_index, type, data, size); +} + OMX_ERRORTYPE OMX::OnEvent( node_id node, OMX_IN OMX_EVENTTYPE eEvent, diff --git a/media/libstagefright/omx/OMXNodeInstance.cpp b/media/libstagefright/omx/OMXNodeInstance.cpp index a9eb94f..46e5d71 100644 --- a/media/libstagefright/omx/OMXNodeInstance.cpp +++ b/media/libstagefright/omx/OMXNodeInstance.cpp @@ -70,6 +70,10 @@ struct BufferMeta { header->nFilledLen); } + void setGraphicBuffer(const sp<GraphicBuffer> &graphicBuffer) { + mGraphicBuffer = graphicBuffer; + } + private: sp<GraphicBuffer> mGraphicBuffer; sp<IMemory> mMem; @@ -238,6 +242,18 @@ status_t OMXNodeInstance::freeNode(OMXMaster *master) { status_t OMXNodeInstance::sendCommand( OMX_COMMANDTYPE cmd, OMX_S32 param) { + const sp<GraphicBufferSource>& bufferSource(getGraphicBufferSource()); + if (bufferSource != NULL + && cmd == OMX_CommandStateSet + && param == OMX_StateLoaded) { + // Initiating transition from Executing -> Loaded + // Buffers are about to be freed. + bufferSource->omxLoaded(); + setGraphicBufferSource(NULL); + + // fall through + } + Mutex::Autolock autoLock(mLock); OMX_ERRORTYPE err = OMX_SendCommand(mHandle, cmd, param, NULL); @@ -401,6 +417,40 @@ status_t OMXNodeInstance::storeMetaDataInBuffers_l( return err; } +status_t OMXNodeInstance::prepareForAdaptivePlayback( + OMX_U32 portIndex, OMX_BOOL enable, OMX_U32 maxFrameWidth, + OMX_U32 maxFrameHeight) { + Mutex::Autolock autolock(mLock); + + OMX_INDEXTYPE index; + OMX_STRING name = const_cast<OMX_STRING>( + "OMX.google.android.index.prepareForAdaptivePlayback"); + + OMX_ERRORTYPE err = OMX_GetExtensionIndex(mHandle, name, &index); + if (err != OMX_ErrorNone) { + ALOGW_IF(enable, "OMX_GetExtensionIndex %s failed", name); + return StatusFromOMXError(err); + } + + PrepareForAdaptivePlaybackParams params; + params.nSize = sizeof(params); + params.nVersion.s.nVersionMajor = 1; + params.nVersion.s.nVersionMinor = 0; + params.nVersion.s.nRevision = 0; + params.nVersion.s.nStep = 0; + + params.nPortIndex = portIndex; + params.bEnable = enable; + params.nMaxFrameWidth = maxFrameWidth; + params.nMaxFrameHeight = maxFrameHeight; + if ((err = OMX_SetParameter(mHandle, index, ¶ms)) != OMX_ErrorNone) { + ALOGW("OMX_SetParameter failed for PrepareForAdaptivePlayback " + "with error %d (0x%08x)", err, err); + return UNKNOWN_ERROR; + } + return err; +} + status_t OMXNodeInstance::useBuffer( OMX_U32 portIndex, const sp<IMemory> ¶ms, OMX::buffer_id *buffer) { @@ -554,6 +604,22 @@ status_t OMXNodeInstance::useGraphicBuffer( return OK; } +status_t OMXNodeInstance::updateGraphicBufferInMeta( + OMX_U32 portIndex, const sp<GraphicBuffer>& graphicBuffer, + OMX::buffer_id buffer) { + Mutex::Autolock autoLock(mLock); + + OMX_BUFFERHEADERTYPE *header = (OMX_BUFFERHEADERTYPE *)(buffer); + VideoDecoderOutputMetaData *metadata = + (VideoDecoderOutputMetaData *)(header->pBuffer); + BufferMeta *bufferMeta = (BufferMeta *)(header->pAppPrivate); + bufferMeta->setGraphicBuffer(graphicBuffer); + metadata->eType = kMetadataBufferTypeGrallocSource; + metadata->pHandle = graphicBuffer->handle; + + return OK; +} + status_t OMXNodeInstance::createInputSurface( OMX_U32 portIndex, sp<IGraphicBufferProducer> *bufferProducer) { Mutex::Autolock autolock(mLock); @@ -584,7 +650,8 @@ status_t OMXNodeInstance::createInputSurface( CHECK(oerr == OMX_ErrorNone); if (def.format.video.eColorFormat != OMX_COLOR_FormatAndroidOpaque) { - ALOGE("createInputSurface requires AndroidOpaque color format"); + ALOGE("createInputSurface requires COLOR_FormatSurface " + "(AndroidOpaque) color format"); return INVALID_OPERATION; } @@ -769,6 +836,47 @@ status_t OMXNodeInstance::getExtensionIndex( return StatusFromOMXError(err); } +status_t OMXNodeInstance::setInternalOption( + OMX_U32 portIndex, + IOMX::InternalOptionType type, + const void *data, + size_t size) { + switch (type) { + case IOMX::INTERNAL_OPTION_SUSPEND: + case IOMX::INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY: + { + const sp<GraphicBufferSource> &bufferSource = + getGraphicBufferSource(); + + if (bufferSource == NULL || portIndex != kPortIndexInput) { + return ERROR_UNSUPPORTED; + } + + if (type == IOMX::INTERNAL_OPTION_SUSPEND) { + if (size != sizeof(bool)) { + return INVALID_OPERATION; + } + + bool suspend = *(bool *)data; + bufferSource->suspend(suspend); + } else { + if (size != sizeof(int64_t)) { + return INVALID_OPERATION; + } + + int64_t delayUs = *(int64_t *)data; + + return bufferSource->setRepeatPreviousFrameDelayUs(delayUs); + } + + return OK; + } + + default: + return ERROR_UNSUPPORTED; + } +} + void OMXNodeInstance::onMessage(const omx_message &msg) { if (msg.type == omx_message::FILL_BUFFER_DONE) { OMX_BUFFERHEADERTYPE *buffer = @@ -818,16 +926,11 @@ void OMXNodeInstance::onEvent( OMX_EVENTTYPE event, OMX_U32 arg1, OMX_U32 arg2) { const sp<GraphicBufferSource>& bufferSource(getGraphicBufferSource()); - if (bufferSource != NULL && event == OMX_EventCmdComplete && - arg1 == OMX_CommandStateSet) { - if (arg2 == OMX_StateExecuting) { - bufferSource->omxExecuting(); - } else if (arg2 == OMX_StateLoaded) { - // Must be shutting down -- won't have a GraphicBufferSource - // on the way up. - bufferSource->omxLoaded(); - setGraphicBufferSource(NULL); - } + if (bufferSource != NULL + && event == OMX_EventCmdComplete + && arg1 == OMX_CommandStateSet + && arg2 == OMX_StateExecuting) { + bufferSource->omxExecuting(); } } diff --git a/media/libstagefright/omx/SoftOMXPlugin.cpp b/media/libstagefright/omx/SoftOMXPlugin.cpp index b3fe98e..d6cde73 100644 --- a/media/libstagefright/omx/SoftOMXPlugin.cpp +++ b/media/libstagefright/omx/SoftOMXPlugin.cpp @@ -50,8 +50,9 @@ static const struct { { "OMX.google.mpeg4.encoder", "mpeg4enc", "video_encoder.mpeg4" }, { "OMX.google.mp3.decoder", "mp3dec", "audio_decoder.mp3" }, { "OMX.google.vorbis.decoder", "vorbisdec", "audio_decoder.vorbis" }, - { "OMX.google.vpx.decoder", "vpxdec", "video_decoder.vpx" }, - { "OMX.google.vpx.encoder", "vpxenc", "video_encoder.vpx" }, + { "OMX.google.vp8.decoder", "vpxdec", "video_decoder.vp8" }, + { "OMX.google.vp9.decoder", "vpxdec", "video_decoder.vp9" }, + { "OMX.google.vp8.encoder", "vpxenc", "video_encoder.vp8" }, { "OMX.google.raw.decoder", "rawdec", "audio_decoder.raw" }, { "OMX.google.flac.encoder", "flacenc", "audio_encoder.flac" }, { "OMX.google.gsm.decoder", "gsmdec", "audio_decoder.gsm" }, diff --git a/media/libstagefright/omx/SoftVideoDecoderOMXComponent.cpp b/media/libstagefright/omx/SoftVideoDecoderOMXComponent.cpp new file mode 100644 index 0000000..08a3d42 --- /dev/null +++ b/media/libstagefright/omx/SoftVideoDecoderOMXComponent.cpp @@ -0,0 +1,290 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "SoftVideoDecoderOMXComponent" +#include <utils/Log.h> + +#include "include/SoftVideoDecoderOMXComponent.h" + +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/ALooper.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/MediaDefs.h> + +namespace android { + +template<class T> +static void InitOMXParams(T *params) { + params->nSize = sizeof(T); + params->nVersion.s.nVersionMajor = 1; + params->nVersion.s.nVersionMinor = 0; + params->nVersion.s.nRevision = 0; + params->nVersion.s.nStep = 0; +} + +SoftVideoDecoderOMXComponent::SoftVideoDecoderOMXComponent( + const char *name, + const char *componentRole, + OMX_VIDEO_CODINGTYPE codingType, + const CodecProfileLevel *profileLevels, + size_t numProfileLevels, + int32_t width, + int32_t height, + const OMX_CALLBACKTYPE *callbacks, + OMX_PTR appData, + OMX_COMPONENTTYPE **component) + : SimpleSoftOMXComponent(name, callbacks, appData, component), + mWidth(width), + mHeight(height), + mCropLeft(0), + mCropTop(0), + mCropWidth(width), + mCropHeight(height), + mOutputPortSettingsChange(NONE), + mComponentRole(componentRole), + mCodingType(codingType), + mProfileLevels(profileLevels), + mNumProfileLevels(numProfileLevels) { +} + +void SoftVideoDecoderOMXComponent::initPorts( + OMX_U32 numInputBuffers, + OMX_U32 inputBufferSize, + OMX_U32 numOutputBuffers, + const char *mimeType) { + OMX_PARAM_PORTDEFINITIONTYPE def; + InitOMXParams(&def); + + def.nPortIndex = kInputPortIndex; + def.eDir = OMX_DirInput; + def.nBufferCountMin = numInputBuffers; + def.nBufferCountActual = def.nBufferCountMin; + def.nBufferSize = inputBufferSize; + def.bEnabled = OMX_TRUE; + def.bPopulated = OMX_FALSE; + def.eDomain = OMX_PortDomainVideo; + def.bBuffersContiguous = OMX_FALSE; + def.nBufferAlignment = 1; + + def.format.video.cMIMEType = const_cast<char *>(mimeType); + def.format.video.pNativeRender = NULL; + /* size is initialized in updatePortDefinitions() */ + def.format.video.nBitrate = 0; + def.format.video.xFramerate = 0; + def.format.video.bFlagErrorConcealment = OMX_FALSE; + def.format.video.eCompressionFormat = mCodingType; + def.format.video.eColorFormat = OMX_COLOR_FormatUnused; + def.format.video.pNativeWindow = NULL; + + addPort(def); + + def.nPortIndex = kOutputPortIndex; + def.eDir = OMX_DirOutput; + def.nBufferCountMin = numOutputBuffers; + def.nBufferCountActual = def.nBufferCountMin; + def.bEnabled = OMX_TRUE; + def.bPopulated = OMX_FALSE; + def.eDomain = OMX_PortDomainVideo; + def.bBuffersContiguous = OMX_FALSE; + def.nBufferAlignment = 2; + + def.format.video.cMIMEType = const_cast<char *>("video/raw"); + def.format.video.pNativeRender = NULL; + /* size is initialized in updatePortDefinitions() */ + def.format.video.nBitrate = 0; + def.format.video.xFramerate = 0; + def.format.video.bFlagErrorConcealment = OMX_FALSE; + def.format.video.eCompressionFormat = OMX_VIDEO_CodingUnused; + def.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar; + def.format.video.pNativeWindow = NULL; + + addPort(def); + + updatePortDefinitions(); +} + +void SoftVideoDecoderOMXComponent::updatePortDefinitions() { + OMX_PARAM_PORTDEFINITIONTYPE *def = &editPortInfo(kInputPortIndex)->mDef; + def->format.video.nFrameWidth = mWidth; + def->format.video.nFrameHeight = mHeight; + def->format.video.nStride = def->format.video.nFrameWidth; + def->format.video.nSliceHeight = def->format.video.nFrameHeight; + + def = &editPortInfo(kOutputPortIndex)->mDef; + def->format.video.nFrameWidth = mWidth; + def->format.video.nFrameHeight = mHeight; + def->format.video.nStride = def->format.video.nFrameWidth; + def->format.video.nSliceHeight = def->format.video.nFrameHeight; + + def->nBufferSize = + (def->format.video.nFrameWidth * + def->format.video.nFrameHeight * 3) / 2; + + mCropLeft = 0; + mCropTop = 0; + mCropWidth = mWidth; + mCropHeight = mHeight; +} + +OMX_ERRORTYPE SoftVideoDecoderOMXComponent::internalGetParameter( + OMX_INDEXTYPE index, OMX_PTR params) { + switch (index) { + case OMX_IndexParamVideoPortFormat: + { + OMX_VIDEO_PARAM_PORTFORMATTYPE *formatParams = + (OMX_VIDEO_PARAM_PORTFORMATTYPE *)params; + + if (formatParams->nPortIndex > kMaxPortIndex) { + return OMX_ErrorUndefined; + } + + if (formatParams->nIndex != 0) { + return OMX_ErrorNoMore; + } + + if (formatParams->nPortIndex == kInputPortIndex) { + formatParams->eCompressionFormat = mCodingType; + formatParams->eColorFormat = OMX_COLOR_FormatUnused; + formatParams->xFramerate = 0; + } else { + CHECK_EQ(formatParams->nPortIndex, 1u); + + formatParams->eCompressionFormat = OMX_VIDEO_CodingUnused; + formatParams->eColorFormat = OMX_COLOR_FormatYUV420Planar; + formatParams->xFramerate = 0; + } + + return OMX_ErrorNone; + } + + case OMX_IndexParamVideoProfileLevelQuerySupported: + { + OMX_VIDEO_PARAM_PROFILELEVELTYPE *profileLevel = + (OMX_VIDEO_PARAM_PROFILELEVELTYPE *) params; + + if (profileLevel->nPortIndex != kInputPortIndex) { + ALOGE("Invalid port index: %ld", profileLevel->nPortIndex); + return OMX_ErrorUnsupportedIndex; + } + + if (index >= mNumProfileLevels) { + return OMX_ErrorNoMore; + } + + profileLevel->eProfile = mProfileLevels[index].mProfile; + profileLevel->eLevel = mProfileLevels[index].mLevel; + return OMX_ErrorNone; + } + + default: + return SimpleSoftOMXComponent::internalGetParameter(index, params); + } +} + +OMX_ERRORTYPE SoftVideoDecoderOMXComponent::internalSetParameter( + OMX_INDEXTYPE index, const OMX_PTR params) { + switch (index) { + case OMX_IndexParamStandardComponentRole: + { + const OMX_PARAM_COMPONENTROLETYPE *roleParams = + (const OMX_PARAM_COMPONENTROLETYPE *)params; + + if (strncmp((const char *)roleParams->cRole, + mComponentRole, + OMX_MAX_STRINGNAME_SIZE - 1)) { + return OMX_ErrorUndefined; + } + + return OMX_ErrorNone; + } + + case OMX_IndexParamVideoPortFormat: + { + OMX_VIDEO_PARAM_PORTFORMATTYPE *formatParams = + (OMX_VIDEO_PARAM_PORTFORMATTYPE *)params; + + if (formatParams->nPortIndex > kMaxPortIndex) { + return OMX_ErrorUndefined; + } + + if (formatParams->nIndex != 0) { + return OMX_ErrorNoMore; + } + + return OMX_ErrorNone; + } + + default: + return SimpleSoftOMXComponent::internalSetParameter(index, params); + } +} + +OMX_ERRORTYPE SoftVideoDecoderOMXComponent::getConfig( + OMX_INDEXTYPE index, OMX_PTR params) { + switch (index) { + case OMX_IndexConfigCommonOutputCrop: + { + OMX_CONFIG_RECTTYPE *rectParams = (OMX_CONFIG_RECTTYPE *)params; + + if (rectParams->nPortIndex != kOutputPortIndex) { + return OMX_ErrorUndefined; + } + + rectParams->nLeft = mCropLeft; + rectParams->nTop = mCropTop; + rectParams->nWidth = mCropWidth; + rectParams->nHeight = mCropHeight; + + return OMX_ErrorNone; + } + + default: + return OMX_ErrorUnsupportedIndex; + } +} + +void SoftVideoDecoderOMXComponent::onReset() { + mOutputPortSettingsChange = NONE; +} + +void SoftVideoDecoderOMXComponent::onPortEnableCompleted(OMX_U32 portIndex, bool enabled) { + if (portIndex != kOutputPortIndex) { + return; + } + + switch (mOutputPortSettingsChange) { + case NONE: + break; + + case AWAITING_DISABLED: + { + CHECK(!enabled); + mOutputPortSettingsChange = AWAITING_ENABLED; + break; + } + + default: + { + CHECK_EQ((int)mOutputPortSettingsChange, (int)AWAITING_ENABLED); + CHECK(enabled); + mOutputPortSettingsChange = NONE; + break; + } + } +} + +} // namespace android diff --git a/media/libstagefright/omx/tests/OMXHarness.cpp b/media/libstagefright/omx/tests/OMXHarness.cpp index 6cca8da..4bee808 100644 --- a/media/libstagefright/omx/tests/OMXHarness.cpp +++ b/media/libstagefright/omx/tests/OMXHarness.cpp @@ -449,7 +449,8 @@ static const char *GetMimeFromComponentRole(const char *componentRole) { { "video_decoder.avc", "video/avc" }, { "video_decoder.mpeg4", "video/mp4v-es" }, { "video_decoder.h263", "video/3gpp" }, - { "video_decoder.vpx", "video/x-vnd.on2.vp8" }, + { "video_decoder.vp8", "video/x-vnd.on2.vp8" }, + { "video_decoder.vp9", "video/x-vnd.on2.vp9" }, // we appear to use this as a synonym to amrnb. { "audio_decoder.amr", "audio/3gpp" }, diff --git a/media/libstagefright/rtsp/ARTSPConnection.cpp b/media/libstagefright/rtsp/ARTSPConnection.cpp index faac3fd..efde7a9 100644 --- a/media/libstagefright/rtsp/ARTSPConnection.cpp +++ b/media/libstagefright/rtsp/ARTSPConnection.cpp @@ -60,6 +60,7 @@ ARTSPConnection::~ARTSPConnection() { ALOGE("Connection is still open, closing the socket."); if (mUIDValid) { HTTPBase::UnRegisterSocketUserTag(mSocket); + HTTPBase::UnRegisterSocketUserMark(mSocket); } close(mSocket); mSocket = -1; @@ -214,6 +215,7 @@ void ARTSPConnection::onConnect(const sp<AMessage> &msg) { if (mState != DISCONNECTED) { if (mUIDValid) { HTTPBase::UnRegisterSocketUserTag(mSocket); + HTTPBase::UnRegisterSocketUserMark(mSocket); } close(mSocket); mSocket = -1; @@ -266,6 +268,7 @@ void ARTSPConnection::onConnect(const sp<AMessage> &msg) { if (mUIDValid) { HTTPBase::RegisterSocketUserTag(mSocket, mUID, (uint32_t)*(uint32_t*) "RTSP"); + HTTPBase::RegisterSocketUserMark(mSocket, mUID); } MakeSocketBlocking(mSocket, false); @@ -295,6 +298,7 @@ void ARTSPConnection::onConnect(const sp<AMessage> &msg) { if (mUIDValid) { HTTPBase::UnRegisterSocketUserTag(mSocket); + HTTPBase::UnRegisterSocketUserMark(mSocket); } close(mSocket); mSocket = -1; @@ -312,6 +316,7 @@ void ARTSPConnection::onConnect(const sp<AMessage> &msg) { void ARTSPConnection::performDisconnect() { if (mUIDValid) { HTTPBase::UnRegisterSocketUserTag(mSocket); + HTTPBase::UnRegisterSocketUserMark(mSocket); } close(mSocket); mSocket = -1; @@ -385,6 +390,7 @@ void ARTSPConnection::onCompleteConnection(const sp<AMessage> &msg) { mState = DISCONNECTED; if (mUIDValid) { HTTPBase::UnRegisterSocketUserTag(mSocket); + HTTPBase::UnRegisterSocketUserMark(mSocket); } close(mSocket); mSocket = -1; diff --git a/media/libstagefright/rtsp/Android.mk b/media/libstagefright/rtsp/Android.mk index 9e2724d..e77c69c 100644 --- a/media/libstagefright/rtsp/Android.mk +++ b/media/libstagefright/rtsp/Android.mk @@ -51,7 +51,7 @@ LOCAL_C_INCLUDES:= \ LOCAL_CFLAGS += -Wno-multichar -LOCAL_MODULE_TAGS := debug +LOCAL_MODULE_TAGS := optional LOCAL_MODULE:= rtp_test diff --git a/media/libstagefright/rtsp/MyHandler.h b/media/libstagefright/rtsp/MyHandler.h index 8681682..cd77aa0 100644 --- a/media/libstagefright/rtsp/MyHandler.h +++ b/media/libstagefright/rtsp/MyHandler.h @@ -716,7 +716,9 @@ struct MyHandler : public AHandler { // Clear the tag if (mUIDValid) { HTTPBase::UnRegisterSocketUserTag(track->mRTPSocket); + HTTPBase::UnRegisterSocketUserMark(track->mRTPSocket); HTTPBase::UnRegisterSocketUserTag(track->mRTCPSocket); + HTTPBase::UnRegisterSocketUserMark(track->mRTCPSocket); } close(track->mRTPSocket); @@ -847,7 +849,9 @@ struct MyHandler : public AHandler { // Clear the tag if (mUIDValid) { HTTPBase::UnRegisterSocketUserTag(info->mRTPSocket); + HTTPBase::UnRegisterSocketUserMark(info->mRTPSocket); HTTPBase::UnRegisterSocketUserTag(info->mRTCPSocket); + HTTPBase::UnRegisterSocketUserMark(info->mRTCPSocket); } close(info->mRTPSocket); @@ -1605,6 +1609,8 @@ private: (uint32_t)*(uint32_t*) "RTP_"); HTTPBase::RegisterSocketUserTag(info->mRTCPSocket, mUID, (uint32_t)*(uint32_t*) "RTP_"); + HTTPBase::RegisterSocketUserMark(info->mRTPSocket, mUID); + HTTPBase::RegisterSocketUserMark(info->mRTCPSocket, mUID); } request.append("Transport: RTP/AVP/UDP;unicast;client_port="); @@ -1681,6 +1687,26 @@ private: return true; } + void handleFirstAccessUnit() { + if (mFirstAccessUnit) { + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("what", kWhatConnected); + msg->post(); + + if (mSeekable) { + for (size_t i = 0; i < mTracks.size(); ++i) { + TrackInfo *info = &mTracks.editItemAt(i); + + postNormalPlayTimeMapping( + i, + info->mNormalPlayTimeRTP, info->mNormalPlayTimeUs); + } + } + + mFirstAccessUnit = false; + } + } + void onTimeUpdate(int32_t trackIndex, uint32_t rtpTime, uint64_t ntpTime) { ALOGV("onTimeUpdate track %d, rtpTime = 0x%08x, ntpTime = 0x%016llx", trackIndex, rtpTime, ntpTime); @@ -1712,6 +1738,8 @@ private: } } if (mAllTracksHaveTime && dataReceivedOnAllChannels()) { + handleFirstAccessUnit(); + // Time is now established, lets start timestamping immediately for (size_t i = 0; i < mTracks.size(); ++i) { TrackInfo *trackInfo = &mTracks.editItemAt(i); @@ -1745,23 +1773,7 @@ private: return; } - if (mFirstAccessUnit) { - sp<AMessage> msg = mNotify->dup(); - msg->setInt32("what", kWhatConnected); - msg->post(); - - if (mSeekable) { - for (size_t i = 0; i < mTracks.size(); ++i) { - TrackInfo *info = &mTracks.editItemAt(i); - - postNormalPlayTimeMapping( - i, - info->mNormalPlayTimeRTP, info->mNormalPlayTimeUs); - } - } - - mFirstAccessUnit = false; - } + handleFirstAccessUnit(); TrackInfo *track = &mTracks.editItemAt(trackIndex); diff --git a/media/libstagefright/tests/SurfaceMediaSource_test.cpp b/media/libstagefright/tests/SurfaceMediaSource_test.cpp index a5459fe..49ffcd6 100644 --- a/media/libstagefright/tests/SurfaceMediaSource_test.cpp +++ b/media/libstagefright/tests/SurfaceMediaSource_test.cpp @@ -23,6 +23,8 @@ #include <fcntl.h> #include <unistd.h> +#include <GLES2/gl2.h> + #include <media/stagefright/SurfaceMediaSource.h> #include <media/mediarecorder.h> diff --git a/media/libstagefright/wifi-display/Android.mk b/media/libstagefright/wifi-display/Android.mk index 061ae89..f70454a 100644 --- a/media/libstagefright/wifi-display/Android.mk +++ b/media/libstagefright/wifi-display/Android.mk @@ -3,10 +3,8 @@ LOCAL_PATH:= $(call my-dir) include $(CLEAR_VARS) LOCAL_SRC_FILES:= \ - ANetworkSession.cpp \ MediaSender.cpp \ Parameters.cpp \ - ParsedMessage.cpp \ rtp/RTPSender.cpp \ source/Converter.cpp \ source/MediaPuller.cpp \ @@ -37,26 +35,3 @@ LOCAL_MODULE:= libstagefright_wfd LOCAL_MODULE_TAGS:= optional include $(BUILD_SHARED_LIBRARY) - -################################################################################ - -include $(CLEAR_VARS) - -LOCAL_SRC_FILES:= \ - wfd.cpp \ - -LOCAL_SHARED_LIBRARIES:= \ - libbinder \ - libgui \ - libmedia \ - libstagefright \ - libstagefright_foundation \ - libstagefright_wfd \ - libutils \ - liblog \ - -LOCAL_MODULE:= wfd - -LOCAL_MODULE_TAGS := debug - -include $(BUILD_EXECUTABLE) diff --git a/media/libstagefright/wifi-display/MediaSender.cpp b/media/libstagefright/wifi-display/MediaSender.cpp index 8a3566f..b1cdec0 100644 --- a/media/libstagefright/wifi-display/MediaSender.cpp +++ b/media/libstagefright/wifi-display/MediaSender.cpp @@ -20,16 +20,18 @@ #include "MediaSender.h" -#include "ANetworkSession.h" #include "rtp/RTPSender.h" #include "source/TSPacketizer.h" #include "include/avc_utils.h" #include <media/IHDCP.h> +#include <media/stagefright/MediaBuffer.h> #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/foundation/ANetworkSession.h> +#include <ui/GraphicBuffer.h> namespace android { @@ -341,6 +343,22 @@ void MediaSender::onSenderNotify(const sp<AMessage> &msg) { break; } + case kWhatInformSender: + { + int64_t avgLatencyUs; + CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs)); + + int64_t maxLatencyUs; + CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs)); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatInformSender); + notify->setInt64("avgLatencyUs", avgLatencyUs); + notify->setInt64("maxLatencyUs", maxLatencyUs); + notify->post(); + break; + } + default: TRESPASS(); } @@ -392,11 +410,36 @@ status_t MediaSender::packetizeAccessUnit( info.mPacketizerTrackIndex, accessUnit); } - status_t err = mHDCP->encrypt( - accessUnit->data(), accessUnit->size(), - trackIndex /* streamCTR */, - &inputCTR, - accessUnit->data()); + status_t err; + native_handle_t* handle; + if (accessUnit->meta()->findPointer("handle", (void**)&handle) + && handle != NULL) { + int32_t rangeLength, rangeOffset; + sp<AMessage> notify; + CHECK(accessUnit->meta()->findInt32("rangeOffset", &rangeOffset)); + CHECK(accessUnit->meta()->findInt32("rangeLength", &rangeLength)); + CHECK(accessUnit->meta()->findMessage("notify", ¬ify) + && notify != NULL); + CHECK_GE(accessUnit->size(), rangeLength); + + sp<GraphicBuffer> grbuf(new GraphicBuffer( + rangeOffset + rangeLength, 1, HAL_PIXEL_FORMAT_Y8, + GRALLOC_USAGE_HW_VIDEO_ENCODER, rangeOffset + rangeLength, + handle, false)); + + err = mHDCP->encryptNative( + grbuf, rangeOffset, rangeLength, + trackIndex /* streamCTR */, + &inputCTR, + accessUnit->data()); + notify->post(); + } else { + err = mHDCP->encrypt( + accessUnit->data(), accessUnit->size(), + trackIndex /* streamCTR */, + &inputCTR, + accessUnit->data()); + } if (err != OK) { ALOGE("Failed to HDCP-encrypt media data (err %d)", diff --git a/media/libstagefright/wifi-display/MediaSender.h b/media/libstagefright/wifi-display/MediaSender.h index 64722c5..04538ea 100644 --- a/media/libstagefright/wifi-display/MediaSender.h +++ b/media/libstagefright/wifi-display/MediaSender.h @@ -43,6 +43,7 @@ struct MediaSender : public AHandler { kWhatInitDone, kWhatError, kWhatNetworkStall, + kWhatInformSender, }; MediaSender( diff --git a/media/libstagefright/wifi-display/VideoFormats.cpp b/media/libstagefright/wifi-display/VideoFormats.cpp index 458b163..04e02c1 100644 --- a/media/libstagefright/wifi-display/VideoFormats.cpp +++ b/media/libstagefright/wifi-display/VideoFormats.cpp @@ -24,7 +24,8 @@ namespace android { -VideoFormats::config_t VideoFormats::mConfigs[][32] = { +// static +const VideoFormats::config_t VideoFormats::mResolutionTable[][32] = { { // CEA Resolutions { 640, 480, 60, false, 0, 0}, @@ -133,6 +134,8 @@ VideoFormats::config_t VideoFormats::mConfigs[][32] = { }; VideoFormats::VideoFormats() { + memcpy(mConfigs, mResolutionTable, sizeof(mConfigs)); + for (size_t i = 0; i < kNumResolutionTypes; ++i) { mResolutionEnabled[i] = 0; } @@ -175,6 +178,29 @@ void VideoFormats::enableAll() { } } +void VideoFormats::enableResolutionUpto( + ResolutionType type, size_t index, + ProfileType profile, LevelType level) { + size_t width, height, fps, score; + bool interlaced; + if (!GetConfiguration(type, index, &width, &height, + &fps, &interlaced)) { + ALOGE("Maximum resolution not found!"); + return; + } + score = width * height * fps * (!interlaced + 1); + for (size_t i = 0; i < kNumResolutionTypes; ++i) { + for (size_t j = 0; j < 32; j++) { + if (GetConfiguration((ResolutionType)i, j, + &width, &height, &fps, &interlaced) + && score >= width * height * fps * (!interlaced + 1)) { + setResolutionEnabled((ResolutionType)i, j); + setProfileLevel((ResolutionType)i, j, profile, level); + } + } + } +} + void VideoFormats::setResolutionEnabled( ResolutionType type, size_t index, bool enabled) { CHECK_LT(type, kNumResolutionTypes); @@ -182,11 +208,56 @@ void VideoFormats::setResolutionEnabled( if (enabled) { mResolutionEnabled[type] |= (1ul << index); + mConfigs[type][index].profile = (1ul << PROFILE_CBP); + mConfigs[type][index].level = (1ul << LEVEL_31); } else { mResolutionEnabled[type] &= ~(1ul << index); + mConfigs[type][index].profile = 0; + mConfigs[type][index].level = 0; } } +void VideoFormats::setProfileLevel( + ResolutionType type, size_t index, + ProfileType profile, LevelType level) { + CHECK_LT(type, kNumResolutionTypes); + CHECK(GetConfiguration(type, index, NULL, NULL, NULL, NULL)); + + mConfigs[type][index].profile = (1ul << profile); + mConfigs[type][index].level = (1ul << level); +} + +void VideoFormats::getProfileLevel( + ResolutionType type, size_t index, + ProfileType *profile, LevelType *level) const{ + CHECK_LT(type, kNumResolutionTypes); + CHECK(GetConfiguration(type, index, NULL, NULL, NULL, NULL)); + + int i, bestProfile = -1, bestLevel = -1; + + for (i = 0; i < kNumProfileTypes; ++i) { + if (mConfigs[type][index].profile & (1ul << i)) { + bestProfile = i; + } + } + + for (i = 0; i < kNumLevelTypes; ++i) { + if (mConfigs[type][index].level & (1ul << i)) { + bestLevel = i; + } + } + + if (bestProfile == -1 || bestLevel == -1) { + ALOGE("Profile or level not set for resolution type %d, index %d", + type, index); + bestProfile = PROFILE_CBP; + bestLevel = LEVEL_31; + } + + *profile = (ProfileType) bestProfile; + *level = (LevelType) bestLevel; +} + bool VideoFormats::isResolutionEnabled( ResolutionType type, size_t index) const { CHECK_LT(type, kNumResolutionTypes); @@ -207,7 +278,7 @@ bool VideoFormats::GetConfiguration( return false; } - const config_t *config = &mConfigs[type][index]; + const config_t *config = &mResolutionTable[type][index]; if (config->width == 0) { return false; @@ -251,9 +322,12 @@ bool VideoFormats::parseH264Codec(const char *spec) { if (res[i] & (1ul << j)){ mResolutionEnabled[i] |= (1ul << j); if (profile > mConfigs[i][j].profile) { + // prefer higher profile (even if level is lower) mConfigs[i][j].profile = profile; - if (level > mConfigs[i][j].level) - mConfigs[i][j].level = level; + mConfigs[i][j].level = level; + } else if (profile == mConfigs[i][j].profile && + level > mConfigs[i][j].level) { + mConfigs[i][j].level = level; } } } @@ -262,9 +336,51 @@ bool VideoFormats::parseH264Codec(const char *spec) { return true; } +// static +bool VideoFormats::GetProfileLevel( + ProfileType profile, LevelType level, unsigned *profileIdc, + unsigned *levelIdc, unsigned *constraintSet) { + CHECK_LT(profile, kNumProfileTypes); + CHECK_LT(level, kNumLevelTypes); + + static const unsigned kProfileIDC[kNumProfileTypes] = { + 66, // PROFILE_CBP + 100, // PROFILE_CHP + }; + + static const unsigned kLevelIDC[kNumLevelTypes] = { + 31, // LEVEL_31 + 32, // LEVEL_32 + 40, // LEVEL_40 + 41, // LEVEL_41 + 42, // LEVEL_42 + }; + + static const unsigned kConstraintSet[kNumProfileTypes] = { + 0xc0, // PROFILE_CBP + 0x0c, // PROFILE_CHP + }; + + if (profileIdc) { + *profileIdc = kProfileIDC[profile]; + } + + if (levelIdc) { + *levelIdc = kLevelIDC[level]; + } + + if (constraintSet) { + *constraintSet = kConstraintSet[profile]; + } + + return true; +} + bool VideoFormats::parseFormatSpec(const char *spec) { CHECK_EQ(kNumResolutionTypes, 3); + disableAll(); + unsigned native, dummy; unsigned res[3]; size_t size = strlen(spec); @@ -320,8 +436,10 @@ AString VideoFormats::getFormatSpec(bool forM4Message) const { // max-vres (none or 2 byte) return StringPrintf( - "%02x 00 02 02 %08x %08x %08x 00 0000 0000 00 none none", + "%02x 00 %02x %02x %08x %08x %08x 00 0000 0000 00 none none", forM4Message ? 0x00 : ((mNativeIndex << 3) | mNativeType), + mConfigs[mNativeType][mNativeIndex].profile, + mConfigs[mNativeType][mNativeIndex].level, mResolutionEnabled[0], mResolutionEnabled[1], mResolutionEnabled[2]); @@ -332,7 +450,9 @@ bool VideoFormats::PickBestFormat( const VideoFormats &sinkSupported, const VideoFormats &sourceSupported, ResolutionType *chosenType, - size_t *chosenIndex) { + size_t *chosenIndex, + ProfileType *chosenProfile, + LevelType *chosenLevel) { #if 0 // Support for the native format is a great idea, the spec includes // these features, but nobody supports it and the tests don't validate it. @@ -412,6 +532,18 @@ bool VideoFormats::PickBestFormat( *chosenType = (ResolutionType)bestType; *chosenIndex = bestIndex; + // Pick the best profile/level supported by both sink and source. + ProfileType srcProfile, sinkProfile; + LevelType srcLevel, sinkLevel; + sourceSupported.getProfileLevel( + (ResolutionType)bestType, bestIndex, + &srcProfile, &srcLevel); + sinkSupported.getProfileLevel( + (ResolutionType)bestType, bestIndex, + &sinkProfile, &sinkLevel); + *chosenProfile = srcProfile < sinkProfile ? srcProfile : sinkProfile; + *chosenLevel = srcLevel < sinkLevel ? srcLevel : sinkLevel; + return true; } diff --git a/media/libstagefright/wifi-display/VideoFormats.h b/media/libstagefright/wifi-display/VideoFormats.h index 01de246..fd38fd1 100644 --- a/media/libstagefright/wifi-display/VideoFormats.h +++ b/media/libstagefright/wifi-display/VideoFormats.h @@ -69,17 +69,33 @@ struct VideoFormats { void disableAll(); void enableAll(); + void enableResolutionUpto( + ResolutionType type, size_t index, + ProfileType profile, LevelType level); void setResolutionEnabled( ResolutionType type, size_t index, bool enabled = true); bool isResolutionEnabled(ResolutionType type, size_t index) const; + void setProfileLevel( + ResolutionType type, size_t index, + ProfileType profile, LevelType level); + + void getProfileLevel( + ResolutionType type, size_t index, + ProfileType *profile, LevelType *level) const; + static bool GetConfiguration( ResolutionType type, size_t index, size_t *width, size_t *height, size_t *framesPerSecond, bool *interlaced); + static bool GetProfileLevel( + ProfileType profile, LevelType level, + unsigned *profileIdc, unsigned *levelIdc, + unsigned *constraintSet); + bool parseFormatSpec(const char *spec); AString getFormatSpec(bool forM4Message = false) const; @@ -87,7 +103,9 @@ struct VideoFormats { const VideoFormats &sinkSupported, const VideoFormats &sourceSupported, ResolutionType *chosenType, - size_t *chosenIndex); + size_t *chosenIndex, + ProfileType *chosenProfile, + LevelType *chosenLevel); private: bool parseH264Codec(const char *spec); @@ -95,7 +113,8 @@ private: size_t mNativeIndex; uint32_t mResolutionEnabled[kNumResolutionTypes]; - static config_t mConfigs[kNumResolutionTypes][32]; + static const config_t mResolutionTable[kNumResolutionTypes][32]; + config_t mConfigs[kNumResolutionTypes][32]; DISALLOW_EVIL_CONSTRUCTORS(VideoFormats); }; diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.cpp b/media/libstagefright/wifi-display/rtp/RTPSender.cpp index 095fd97..1887b8b 100644 --- a/media/libstagefright/wifi-display/rtp/RTPSender.cpp +++ b/media/libstagefright/wifi-display/rtp/RTPSender.cpp @@ -20,11 +20,10 @@ #include "RTPSender.h" -#include "ANetworkSession.h" - #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/foundation/ANetworkSession.h> #include <media/stagefright/foundation/hexdump.h> #include <media/stagefright/MediaErrors.h> #include <media/stagefright/Utils.h> @@ -767,6 +766,17 @@ status_t RTPSender::parseTSFB(const uint8_t *data, size_t size) { } status_t RTPSender::parseAPP(const uint8_t *data, size_t size) { + if (!memcmp("late", &data[8], 4)) { + int64_t avgLatencyUs = (int64_t)U64_AT(&data[12]); + int64_t maxLatencyUs = (int64_t)U64_AT(&data[20]); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatInformSender); + notify->setInt64("avgLatencyUs", avgLatencyUs); + notify->setInt64("maxLatencyUs", maxLatencyUs); + notify->post(); + } + return OK; } diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.h b/media/libstagefright/wifi-display/rtp/RTPSender.h index 7dc138a..fefcab7 100644 --- a/media/libstagefright/wifi-display/rtp/RTPSender.h +++ b/media/libstagefright/wifi-display/rtp/RTPSender.h @@ -37,6 +37,7 @@ struct RTPSender : public RTPBase, public AHandler { kWhatInitDone, kWhatError, kWhatNetworkStall, + kWhatInformSender, }; RTPSender( const sp<ANetworkSession> &netSession, diff --git a/media/libstagefright/wifi-display/source/Converter.cpp b/media/libstagefright/wifi-display/source/Converter.cpp index 5344623..753b3ec 100644 --- a/media/libstagefright/wifi-display/source/Converter.cpp +++ b/media/libstagefright/wifi-display/source/Converter.cpp @@ -21,6 +21,7 @@ #include "Converter.h" #include "MediaPuller.h" +#include "include/avc_utils.h" #include <cutils/properties.h> #include <gui/Surface.h> @@ -33,6 +34,8 @@ #include <media/stagefright/MediaDefs.h> #include <media/stagefright/MediaErrors.h> +#include <arpa/inet.h> + #include <OMX_Video.h> namespace android { @@ -40,14 +43,15 @@ namespace android { Converter::Converter( const sp<AMessage> ¬ify, const sp<ALooper> &codecLooper, - const sp<AMessage> &format, - bool usePCMAudio) - : mInitCheck(NO_INIT), - mNotify(notify), + const sp<AMessage> &outputFormat, + uint32_t flags) + : mNotify(notify), mCodecLooper(codecLooper), - mInputFormat(format), + mOutputFormat(outputFormat), + mFlags(flags), mIsVideo(false), - mIsPCMAudio(usePCMAudio), + mIsH264(false), + mIsPCMAudio(false), mNeedToManuallyPrependSPSPPS(false), mDoMoreWorkPending(false) #if ENABLE_SILENCE_DETECTION @@ -56,20 +60,17 @@ Converter::Converter( #endif ,mPrevVideoBitrate(-1) ,mNumFramesToDrop(0) + ,mEncodingSuspended(false) { AString mime; - CHECK(mInputFormat->findString("mime", &mime)); + CHECK(mOutputFormat->findString("mime", &mime)); if (!strncasecmp("video/", mime.c_str(), 6)) { mIsVideo = true; - } - - CHECK(!usePCMAudio || !mIsVideo); - mInitCheck = initEncoder(); - - if (mInitCheck != OK) { - releaseEncoder(); + mIsH264 = !strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC); + } else if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_RAW, mime.c_str())) { + mIsPCMAudio = true; } } @@ -119,8 +120,19 @@ void Converter::shutdownAsync() { (new AMessage(kWhatShutdown, id()))->post(); } -status_t Converter::initCheck() const { - return mInitCheck; +status_t Converter::init() { + status_t err = initEncoder(); + + if (err != OK) { + releaseEncoder(); + } + + return err; +} + +sp<IGraphicBufferProducer> Converter::getGraphicBufferProducer() { + CHECK(mFlags & FLAG_USE_SURFACE_INPUT); + return mGraphicBufferProducer; } size_t Converter::getInputBufferCount() const { @@ -152,23 +164,10 @@ int32_t Converter::GetInt32Property( } status_t Converter::initEncoder() { - AString inputMIME; - CHECK(mInputFormat->findString("mime", &inputMIME)); - AString outputMIME; - bool isAudio = false; - if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_AUDIO_RAW)) { - if (mIsPCMAudio) { - outputMIME = MEDIA_MIMETYPE_AUDIO_RAW; - } else { - outputMIME = MEDIA_MIMETYPE_AUDIO_AAC; - } - isAudio = true; - } else if (!strcasecmp(inputMIME.c_str(), MEDIA_MIMETYPE_VIDEO_RAW)) { - outputMIME = MEDIA_MIMETYPE_VIDEO_AVC; - } else { - TRESPASS(); - } + CHECK(mOutputFormat->findString("mime", &outputMIME)); + + bool isAudio = !strncasecmp(outputMIME.c_str(), "audio/", 6); if (!mIsPCMAudio) { mEncoder = MediaCodec::CreateByType( @@ -179,14 +178,10 @@ status_t Converter::initEncoder() { } } - mOutputFormat = mInputFormat->dup(); - if (mIsPCMAudio) { return OK; } - mOutputFormat->setString("mime", outputMIME.c_str()); - int32_t audioBitrate = GetInt32Property("media.wfd.audio-bitrate", 128000); int32_t videoBitrate = GetInt32Property("media.wfd.video-bitrate", 5000000); mPrevVideoBitrate = videoBitrate; @@ -262,6 +257,16 @@ status_t Converter::initEncoder() { return err; } + if (mFlags & FLAG_USE_SURFACE_INPUT) { + CHECK(mIsVideo); + + err = mEncoder->createInputSurface(&mGraphicBufferProducer); + + if (err != OK) { + return err; + } + } + err = mEncoder->start(); if (err != OK) { @@ -274,7 +279,17 @@ status_t Converter::initEncoder() { return err; } - return mEncoder->getOutputBuffers(&mEncoderOutputBuffers); + err = mEncoder->getOutputBuffers(&mEncoderOutputBuffers); + + if (err != OK) { + return err; + } + + if (mFlags & FLAG_USE_SURFACE_INPUT) { + scheduleDoMoreWork(); + } + + return OK; } void Converter::notifyError(status_t err) { @@ -330,9 +345,12 @@ void Converter::onMessageReceived(const sp<AMessage> &msg) { sp<ABuffer> accessUnit; CHECK(msg->findBuffer("accessUnit", &accessUnit)); - if (mIsVideo && mNumFramesToDrop) { - --mNumFramesToDrop; - ALOGI("dropping frame."); + if (mNumFramesToDrop > 0 || mEncodingSuspended) { + if (mNumFramesToDrop > 0) { + --mNumFramesToDrop; + ALOGI("dropping frame."); + } + ReleaseMediaBufferReference(accessUnit); break; } @@ -414,7 +432,7 @@ void Converter::onMessageReceived(const sp<AMessage> &msg) { } if (mIsVideo) { - ALOGI("requesting IDR frame"); + ALOGV("requesting IDR frame"); mEncoder->requestIDRFrame(); } break; @@ -427,8 +445,12 @@ void Converter::onMessageReceived(const sp<AMessage> &msg) { releaseEncoder(); AString mime; - CHECK(mInputFormat->findString("mime", &mime)); + CHECK(mOutputFormat->findString("mime", &mime)); ALOGI("encoder (%s) shut down.", mime.c_str()); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatShutdownCompleted); + notify->post(); break; } @@ -438,6 +460,32 @@ void Converter::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatReleaseOutputBuffer: + { + if (mEncoder != NULL) { + size_t bufferIndex; + CHECK(msg->findInt32("bufferIndex", (int32_t*)&bufferIndex)); + CHECK(bufferIndex < mEncoderOutputBuffers.size()); + mEncoder->releaseOutputBuffer(bufferIndex); + } + break; + } + + case kWhatSuspendEncoding: + { + int32_t suspend; + CHECK(msg->findInt32("suspend", &suspend)); + + mEncodingSuspended = suspend; + + if (mFlags & FLAG_USE_SURFACE_INPUT) { + sp<AMessage> params = new AMessage; + params->setInt32("drop-input-frames",suspend); + mEncoder->setParameters(params); + } + break; + } + default: TRESPASS(); } @@ -623,28 +671,46 @@ status_t Converter::feedEncoderInputBuffers() { return OK; } +sp<ABuffer> Converter::prependCSD(const sp<ABuffer> &accessUnit) const { + CHECK(mCSD0 != NULL); + + sp<ABuffer> dup = new ABuffer(accessUnit->size() + mCSD0->size()); + memcpy(dup->data(), mCSD0->data(), mCSD0->size()); + memcpy(dup->data() + mCSD0->size(), accessUnit->data(), accessUnit->size()); + + int64_t timeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + + dup->meta()->setInt64("timeUs", timeUs); + + return dup; +} + status_t Converter::doMoreWork() { status_t err; - for (;;) { - size_t bufferIndex; - err = mEncoder->dequeueInputBuffer(&bufferIndex); + if (!(mFlags & FLAG_USE_SURFACE_INPUT)) { + for (;;) { + size_t bufferIndex; + err = mEncoder->dequeueInputBuffer(&bufferIndex); - if (err != OK) { - break; + if (err != OK) { + break; + } + + mAvailEncoderInputIndices.push_back(bufferIndex); } - mAvailEncoderInputIndices.push_back(bufferIndex); + feedEncoderInputBuffers(); } - feedEncoderInputBuffers(); - for (;;) { size_t bufferIndex; size_t offset; size_t size; int64_t timeUs; uint32_t flags; + native_handle_t* handle = NULL; err = mEncoder->dequeueOutputBuffer( &bufferIndex, &offset, &size, &timeUs, &flags); @@ -667,19 +733,63 @@ status_t Converter::doMoreWork() { notify->setInt32("what", kWhatEOS); notify->post(); } else { - sp<ABuffer> buffer = new ABuffer(size); +#if 0 + if (mIsVideo) { + int32_t videoBitrate = GetInt32Property( + "media.wfd.video-bitrate", 5000000); + + setVideoBitrate(videoBitrate); + } +#endif + + sp<ABuffer> buffer; + sp<ABuffer> outbuf = mEncoderOutputBuffers.itemAt(bufferIndex); + + if (outbuf->meta()->findPointer("handle", (void**)&handle) && + handle != NULL) { + int32_t rangeLength, rangeOffset; + CHECK(outbuf->meta()->findInt32("rangeOffset", &rangeOffset)); + CHECK(outbuf->meta()->findInt32("rangeLength", &rangeLength)); + outbuf->meta()->setPointer("handle", NULL); + + // MediaSender will post the following message when HDCP + // is done, to release the output buffer back to encoder. + sp<AMessage> notify(new AMessage( + kWhatReleaseOutputBuffer, id())); + notify->setInt32("bufferIndex", bufferIndex); + + buffer = new ABuffer( + rangeLength > (int32_t)size ? rangeLength : size); + buffer->meta()->setPointer("handle", handle); + buffer->meta()->setInt32("rangeOffset", rangeOffset); + buffer->meta()->setInt32("rangeLength", rangeLength); + buffer->meta()->setMessage("notify", notify); + } else { + buffer = new ABuffer(size); + } + buffer->meta()->setInt64("timeUs", timeUs); ALOGV("[%s] time %lld us (%.2f secs)", mIsVideo ? "video" : "audio", timeUs, timeUs / 1E6); - memcpy(buffer->data(), - mEncoderOutputBuffers.itemAt(bufferIndex)->base() + offset, - size); + memcpy(buffer->data(), outbuf->base() + offset, size); if (flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) { - mOutputFormat->setBuffer("csd-0", buffer); + if (!handle) { + if (mIsH264) { + mCSD0 = buffer; + } + mOutputFormat->setBuffer("csd-0", buffer); + } } else { + if (mNeedToManuallyPrependSPSPPS + && mIsH264 + && (mFlags & FLAG_PREPEND_CSD_IF_NECESSARY) + && IsIDR(buffer)) { + buffer = prependCSD(buffer); + } + sp<AMessage> notify = mNotify->dup(); notify->setInt32("what", kWhatAccessUnit); notify->setBuffer("accessUnit", buffer); @@ -687,7 +797,9 @@ status_t Converter::doMoreWork() { } } - mEncoder->releaseOutputBuffer(bufferIndex); + if (!handle) { + mEncoder->releaseOutputBuffer(bufferIndex); + } if (flags & MediaCodec::BUFFER_FLAG_EOS) { break; @@ -702,9 +814,18 @@ void Converter::requestIDRFrame() { } void Converter::dropAFrame() { + // Unsupported in surface input mode. + CHECK(!(mFlags & FLAG_USE_SURFACE_INPUT)); + (new AMessage(kWhatDropAFrame, id()))->post(); } +void Converter::suspendEncoding(bool suspend) { + sp<AMessage> msg = new AMessage(kWhatSuspendEncoding, id()); + msg->setInt32("suspend", suspend); + msg->post(); +} + int32_t Converter::getVideoBitrate() const { return mPrevVideoBitrate; } @@ -712,7 +833,7 @@ int32_t Converter::getVideoBitrate() const { void Converter::setVideoBitrate(int32_t bitRate) { if (mIsVideo && mEncoder != NULL && bitRate != mPrevVideoBitrate) { sp<AMessage> params = new AMessage; - params->setInt32("videoBitrate", bitRate); + params->setInt32("video-bitrate", bitRate); mEncoder->setParameters(params); diff --git a/media/libstagefright/wifi-display/source/Converter.h b/media/libstagefright/wifi-display/source/Converter.h index ba297c4..5876e07 100644 --- a/media/libstagefright/wifi-display/source/Converter.h +++ b/media/libstagefright/wifi-display/source/Converter.h @@ -18,13 +18,12 @@ #define CONVERTER_H_ -#include "WifiDisplaySource.h" - #include <media/stagefright/foundation/AHandler.h> namespace android { struct ABuffer; +struct IGraphicBufferProducer; struct MediaCodec; #define ENABLE_SILENCE_DETECTION 0 @@ -33,13 +32,25 @@ struct MediaCodec; // media access unit of a different format. // Right now this'll convert raw video into H.264 and raw audio into AAC. struct Converter : public AHandler { - Converter( - const sp<AMessage> ¬ify, - const sp<ALooper> &codecLooper, - const sp<AMessage> &format, - bool usePCMAudio); + enum { + kWhatAccessUnit, + kWhatEOS, + kWhatError, + kWhatShutdownCompleted, + }; + + enum FlagBits { + FLAG_USE_SURFACE_INPUT = 1, + FLAG_PREPEND_CSD_IF_NECESSARY = 2, + }; + Converter(const sp<AMessage> ¬ify, + const sp<ALooper> &codecLooper, + const sp<AMessage> &outputFormat, + uint32_t flags = 0); - status_t initCheck() const; + status_t init(); + + sp<IGraphicBufferProducer> getGraphicBufferProducer(); size_t getInputBufferCount() const; @@ -52,21 +63,7 @@ struct Converter : public AHandler { void requestIDRFrame(); void dropAFrame(); - - enum { - kWhatAccessUnit, - kWhatEOS, - kWhatError, - }; - - enum { - kWhatDoMoreWork, - kWhatRequestIDRFrame, - kWhatShutdown, - kWhatMediaPullerNotify, - kWhatEncoderActivity, - kWhatDropAFrame, - }; + void suspendEncoding(bool suspend); void shutdownAsync(); @@ -75,23 +72,40 @@ struct Converter : public AHandler { static int32_t GetInt32Property(const char *propName, int32_t defaultValue); + enum { + // MUST not conflict with private enums below. + kWhatMediaPullerNotify = 'pulN', + }; + protected: virtual ~Converter(); virtual void onMessageReceived(const sp<AMessage> &msg); private: - status_t mInitCheck; + enum { + kWhatDoMoreWork, + kWhatRequestIDRFrame, + kWhatSuspendEncoding, + kWhatShutdown, + kWhatEncoderActivity, + kWhatDropAFrame, + kWhatReleaseOutputBuffer, + }; + sp<AMessage> mNotify; sp<ALooper> mCodecLooper; - sp<AMessage> mInputFormat; + sp<AMessage> mOutputFormat; + uint32_t mFlags; bool mIsVideo; + bool mIsH264; bool mIsPCMAudio; - sp<AMessage> mOutputFormat; bool mNeedToManuallyPrependSPSPPS; sp<MediaCodec> mEncoder; sp<AMessage> mEncoderActivityNotify; + sp<IGraphicBufferProducer> mGraphicBufferProducer; + Vector<sp<ABuffer> > mEncoderInputBuffers; Vector<sp<ABuffer> > mEncoderOutputBuffers; @@ -99,6 +113,8 @@ private: List<sp<ABuffer> > mInputBufferQueue; + sp<ABuffer> mCSD0; + bool mDoMoreWorkPending; #if ENABLE_SILENCE_DETECTION @@ -111,6 +127,7 @@ private: int32_t mPrevVideoBitrate; int32_t mNumFramesToDrop; + bool mEncodingSuspended; status_t initEncoder(); void releaseEncoder(); @@ -129,6 +146,8 @@ private: static bool IsSilence(const sp<ABuffer> &accessUnit); + sp<ABuffer> prependCSD(const sp<ABuffer> &accessUnit) const; + DISALLOW_EVIL_CONSTRUCTORS(Converter); }; diff --git a/media/libstagefright/wifi-display/source/MediaPuller.cpp b/media/libstagefright/wifi-display/source/MediaPuller.cpp index 189bea3..7e8891d 100644 --- a/media/libstagefright/wifi-display/source/MediaPuller.cpp +++ b/media/libstagefright/wifi-display/source/MediaPuller.cpp @@ -93,6 +93,9 @@ void MediaPuller::onMessageReceived(const sp<AMessage> &msg) { err = mSource->start(params.get()); } else { err = mSource->start(); + if (err != OK) { + ALOGE("source failed to start w/ err %d", err); + } } if (err == OK) { diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.cpp b/media/libstagefright/wifi-display/source/PlaybackSession.cpp index 3d7b865..286ea13 100644 --- a/media/libstagefright/wifi-display/source/PlaybackSession.cpp +++ b/media/libstagefright/wifi-display/source/PlaybackSession.cpp @@ -378,7 +378,9 @@ status_t WifiDisplaySource::PlaybackSession::init( bool usePCMAudio, bool enableVideo, VideoFormats::ResolutionType videoResolutionType, - size_t videoResolutionIndex) { + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType) { sp<AMessage> notify = new AMessage(kWhatMediaSenderNotify, id()); mMediaSender = new MediaSender(mNetSession, notify); looper()->registerHandler(mMediaSender); @@ -390,7 +392,9 @@ status_t WifiDisplaySource::PlaybackSession::init( usePCMAudio, enableVideo, videoResolutionType, - videoResolutionIndex); + videoResolutionIndex, + videoProfileType, + videoLevelType); if (err == OK) { err = mMediaSender->initAsync( @@ -517,7 +521,7 @@ void WifiDisplaySource::PlaybackSession::onMessageReceived( if (mTracks.isEmpty()) { ALOGI("Reached EOS"); } - } else { + } else if (what != Converter::kWhatShutdownCompleted) { CHECK_EQ(what, Converter::kWhatError); status_t err; @@ -559,6 +563,8 @@ void WifiDisplaySource::PlaybackSession::onMessageReceived( converter->dropAFrame(); } } + } else if (what == MediaSender::kWhatInformSender) { + onSinkFeedback(msg); } else { TRESPASS(); } @@ -654,6 +660,89 @@ void WifiDisplaySource::PlaybackSession::onMessageReceived( } } +void WifiDisplaySource::PlaybackSession::onSinkFeedback(const sp<AMessage> &msg) { + int64_t avgLatencyUs; + CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs)); + + int64_t maxLatencyUs; + CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs)); + + ALOGI("sink reports avg. latency of %lld ms (max %lld ms)", + avgLatencyUs / 1000ll, + maxLatencyUs / 1000ll); + + if (mVideoTrackIndex >= 0) { + const sp<Track> &videoTrack = mTracks.valueFor(mVideoTrackIndex); + sp<Converter> converter = videoTrack->converter(); + + if (converter != NULL) { + int32_t videoBitrate = + Converter::GetInt32Property("media.wfd.video-bitrate", -1); + + char val[PROPERTY_VALUE_MAX]; + if (videoBitrate < 0 + && property_get("media.wfd.video-bitrate", val, NULL) + && !strcasecmp("adaptive", val)) { + videoBitrate = converter->getVideoBitrate(); + + if (avgLatencyUs > 300000ll) { + videoBitrate *= 0.6; + } else if (avgLatencyUs < 100000ll) { + videoBitrate *= 1.1; + } + } + + if (videoBitrate > 0) { + if (videoBitrate < 500000) { + videoBitrate = 500000; + } else if (videoBitrate > 10000000) { + videoBitrate = 10000000; + } + + if (videoBitrate != converter->getVideoBitrate()) { + ALOGI("setting video bitrate to %d bps", videoBitrate); + + converter->setVideoBitrate(videoBitrate); + } + } + } + + sp<RepeaterSource> repeaterSource = videoTrack->repeaterSource(); + if (repeaterSource != NULL) { + double rateHz = + Converter::GetInt32Property( + "media.wfd.video-framerate", -1); + + char val[PROPERTY_VALUE_MAX]; + if (rateHz < 0.0 + && property_get("media.wfd.video-framerate", val, NULL) + && !strcasecmp("adaptive", val)) { + rateHz = repeaterSource->getFrameRate(); + + if (avgLatencyUs > 300000ll) { + rateHz *= 0.9; + } else if (avgLatencyUs < 200000ll) { + rateHz *= 1.1; + } + } + + if (rateHz > 0) { + if (rateHz < 5.0) { + rateHz = 5.0; + } else if (rateHz > 30.0) { + rateHz = 30.0; + } + + if (rateHz != repeaterSource->getFrameRate()) { + ALOGI("setting frame rate to %.2f Hz", rateHz); + + repeaterSource->setFrameRate(rateHz); + } + } + } + } +} + status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer( bool enableAudio, bool enableVideo) { DataSource::RegisterDefaultSniffers(); @@ -785,7 +874,9 @@ status_t WifiDisplaySource::PlaybackSession::setupPacketizer( bool usePCMAudio, bool enableVideo, VideoFormats::ResolutionType videoResolutionType, - size_t videoResolutionIndex) { + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType) { CHECK(enableAudio || enableVideo); if (!mMediaPath.empty()) { @@ -794,7 +885,8 @@ status_t WifiDisplaySource::PlaybackSession::setupPacketizer( if (enableVideo) { status_t err = addVideoSource( - videoResolutionType, videoResolutionIndex); + videoResolutionType, videoResolutionIndex, videoProfileType, + videoLevelType); if (err != OK) { return err; @@ -810,9 +902,13 @@ status_t WifiDisplaySource::PlaybackSession::setupPacketizer( status_t WifiDisplaySource::PlaybackSession::addSource( bool isVideo, const sp<MediaSource> &source, bool isRepeaterSource, - bool usePCMAudio, size_t *numInputBuffers) { + bool usePCMAudio, unsigned profileIdc, unsigned levelIdc, + unsigned constraintSet, size_t *numInputBuffers) { CHECK(!usePCMAudio || !isVideo); CHECK(!isRepeaterSource || isVideo); + CHECK(!profileIdc || isVideo); + CHECK(!levelIdc || isVideo); + CHECK(!constraintSet || isVideo); sp<ALooper> pullLooper = new ALooper; pullLooper->setName("pull_looper"); @@ -841,26 +937,37 @@ status_t WifiDisplaySource::PlaybackSession::addSource( CHECK_EQ(err, (status_t)OK); if (isVideo) { + format->setString("mime", MEDIA_MIMETYPE_VIDEO_AVC); format->setInt32("store-metadata-in-buffers", true); - + format->setInt32("store-metadata-in-buffers-output", (mHDCP != NULL) + && (mHDCP->getCaps() & HDCPModule::HDCP_CAPS_ENCRYPT_NATIVE)); format->setInt32( "color-format", OMX_COLOR_FormatAndroidOpaque); + format->setInt32("profile-idc", profileIdc); + format->setInt32("level-idc", levelIdc); + format->setInt32("constraint-set", constraintSet); + } else { + format->setString( + "mime", + usePCMAudio + ? MEDIA_MIMETYPE_AUDIO_RAW : MEDIA_MIMETYPE_AUDIO_AAC); } notify = new AMessage(kWhatConverterNotify, id()); notify->setSize("trackIndex", trackIndex); - sp<Converter> converter = - new Converter(notify, codecLooper, format, usePCMAudio); + sp<Converter> converter = new Converter(notify, codecLooper, format); + + looper()->registerHandler(converter); - err = converter->initCheck(); + err = converter->init(); if (err != OK) { ALOGE("%s converter returned err %d", isVideo ? "video" : "audio", err); + + looper()->unregisterHandler(converter->id()); return err; } - looper()->registerHandler(converter); - notify = new AMessage(Converter::kWhatMediaPullerNotify, converter->id()); notify->setSize("trackIndex", trackIndex); @@ -905,7 +1012,9 @@ status_t WifiDisplaySource::PlaybackSession::addSource( status_t WifiDisplaySource::PlaybackSession::addVideoSource( VideoFormats::ResolutionType videoResolutionType, - size_t videoResolutionIndex) { + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType) { size_t width, height, framesPerSecond; bool interlaced; CHECK(VideoFormats::GetConfiguration( @@ -916,6 +1025,14 @@ status_t WifiDisplaySource::PlaybackSession::addVideoSource( &framesPerSecond, &interlaced)); + unsigned profileIdc, levelIdc, constraintSet; + CHECK(VideoFormats::GetProfileLevel( + videoProfileType, + videoLevelType, + &profileIdc, + &levelIdc, + &constraintSet)); + sp<SurfaceMediaSource> source = new SurfaceMediaSource(width, height); source->setUseAbsoluteTimestamps(); @@ -926,7 +1043,8 @@ status_t WifiDisplaySource::PlaybackSession::addVideoSource( size_t numInputBuffers; status_t err = addSource( true /* isVideo */, videoSource, true /* isRepeaterSource */, - false /* usePCMAudio */, &numInputBuffers); + false /* usePCMAudio */, profileIdc, levelIdc, constraintSet, + &numInputBuffers); if (err != OK) { return err; @@ -949,7 +1067,8 @@ status_t WifiDisplaySource::PlaybackSession::addAudioSource(bool usePCMAudio) { if (audioSource->initCheck() == OK) { return addSource( false /* isVideo */, audioSource, false /* isRepeaterSource */, - usePCMAudio, NULL /* numInputBuffers */); + usePCMAudio, 0 /* profileIdc */, 0 /* levelIdc */, + 0 /* constraintSet */, NULL /* numInputBuffers */); } ALOGW("Unable to instantiate audio source"); diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.h b/media/libstagefright/wifi-display/source/PlaybackSession.h index 39086a1..5c8ee94 100644 --- a/media/libstagefright/wifi-display/source/PlaybackSession.h +++ b/media/libstagefright/wifi-display/source/PlaybackSession.h @@ -53,7 +53,9 @@ struct WifiDisplaySource::PlaybackSession : public AHandler { bool usePCMAudio, bool enableVideo, VideoFormats::ResolutionType videoResolutionType, - size_t videoResolutionIndex); + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType); void destroyAsync(); @@ -130,18 +132,25 @@ private: bool usePCMAudio, bool enableVideo, VideoFormats::ResolutionType videoResolutionType, - size_t videoResolutionIndex); + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType); status_t addSource( bool isVideo, const sp<MediaSource> &source, bool isRepeaterSource, bool usePCMAudio, + unsigned profileIdc, + unsigned levelIdc, + unsigned contraintSet, size_t *numInputBuffers); status_t addVideoSource( VideoFormats::ResolutionType videoResolutionType, - size_t videoResolutionIndex); + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType); status_t addAudioSource(bool usePCMAudio); diff --git a/media/libstagefright/wifi-display/source/TSPacketizer.cpp b/media/libstagefright/wifi-display/source/TSPacketizer.cpp index 2c4a373..c674700 100644 --- a/media/libstagefright/wifi-display/source/TSPacketizer.cpp +++ b/media/libstagefright/wifi-display/source/TSPacketizer.cpp @@ -261,12 +261,24 @@ void TSPacketizer::Track::finalize() { data[0] = 40; // descriptor_tag data[1] = 4; // descriptor_length - CHECK_GE(mCSD.size(), 1u); - const sp<ABuffer> &sps = mCSD.itemAt(0); - CHECK(!memcmp("\x00\x00\x00\x01", sps->data(), 4)); - CHECK_GE(sps->size(), 7u); - // profile_idc, constraint_set*, level_idc - memcpy(&data[2], sps->data() + 4, 3); + if (mCSD.size() > 0) { + CHECK_GE(mCSD.size(), 1u); + const sp<ABuffer> &sps = mCSD.itemAt(0); + CHECK(!memcmp("\x00\x00\x00\x01", sps->data(), 4)); + CHECK_GE(sps->size(), 7u); + // profile_idc, constraint_set*, level_idc + memcpy(&data[2], sps->data() + 4, 3); + } else { + int32_t profileIdc, levelIdc, constraintSet; + CHECK(mFormat->findInt32("profile-idc", &profileIdc)); + CHECK(mFormat->findInt32("level-idc", &levelIdc)); + CHECK(mFormat->findInt32("constraint-set", &constraintSet)); + CHECK_GE(profileIdc, 0u); + CHECK_GE(levelIdc, 0u); + data[2] = profileIdc; // profile_idc + data[3] = constraintSet; // constraint_set* + data[4] = levelIdc; // level_idc + } // AVC_still_present=0, AVC_24_hour_picture_flag=0, reserved data[5] = 0x3f; diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp index 22dd0b1..05e4018 100644 --- a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp @@ -21,7 +21,6 @@ #include "WifiDisplaySource.h" #include "PlaybackSession.h" #include "Parameters.h" -#include "ParsedMessage.h" #include "rtp/RTPSender.h" #include <binder/IServiceManager.h> @@ -32,6 +31,7 @@ #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/foundation/ParsedMessage.h> #include <media/stagefright/MediaErrors.h> #include <media/stagefright/Utils.h> @@ -73,6 +73,12 @@ WifiDisplaySource::WifiDisplaySource( mSupportedSourceVideoFormats.setNativeResolution( VideoFormats::RESOLUTION_CEA, 5); // 1280x720 p30 + + // Enable all resolutions up to 1280x720p30 + mSupportedSourceVideoFormats.enableResolutionUpto( + VideoFormats::RESOLUTION_CEA, 5, + VideoFormats::PROFILE_CHP, // Constrained High Profile + VideoFormats::LEVEL_32); // Level 3.2 } WifiDisplaySource::~WifiDisplaySource() { @@ -164,10 +170,10 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { } else { err = -EINVAL; } - - mState = AWAITING_CLIENT_CONNECTION; } + mState = AWAITING_CLIENT_CONNECTION; + sp<AMessage> response = new AMessage; response->setInt32("err", err); response->postReply(replyID); @@ -401,7 +407,8 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { 0, // height, mUsingHDCP ? IRemoteDisplayClient::kDisplayFlagSecure - : 0); + : 0, + 0); } else { size_t width, height; @@ -420,7 +427,8 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { height, mUsingHDCP ? IRemoteDisplayClient::kDisplayFlagSecure - : 0); + : 0, + playbackSessionID); } } @@ -617,6 +625,9 @@ status_t WifiDisplaySource::sendM4(int32_t sessionID) { chosenVideoFormat.disableAll(); chosenVideoFormat.setNativeResolution( mChosenVideoResolutionType, mChosenVideoResolutionIndex); + chosenVideoFormat.setProfileLevel( + mChosenVideoResolutionType, mChosenVideoResolutionIndex, + mChosenVideoProfile, mChosenVideoLevel); body.append(chosenVideoFormat.getFormatSpec(true /* forM4Message */)); body.append("\r\n"); @@ -729,6 +740,8 @@ status_t WifiDisplaySource::sendM16(int32_t sessionID) { ++mNextCSeq; + scheduleKeepAlive(sessionID); + return OK; } @@ -845,7 +858,9 @@ status_t WifiDisplaySource::onReceiveM3Response( mSupportedSinkVideoFormats, mSupportedSourceVideoFormats, &mChosenVideoResolutionType, - &mChosenVideoResolutionIndex)) { + &mChosenVideoResolutionIndex, + &mChosenVideoProfile, + &mChosenVideoLevel)) { ALOGE("Sink and source share no commonly supported video " "formats."); @@ -864,6 +879,9 @@ status_t WifiDisplaySource::onReceiveM3Response( ALOGI("Picked video resolution %u x %u %c%u", width, height, interlaced ? 'i' : 'p', framesPerSecond); + + ALOGI("Picked AVC profile %d, level %d", + mChosenVideoProfile, mChosenVideoLevel); } else { ALOGI("Sink doesn't support video at all."); } @@ -994,8 +1012,6 @@ status_t WifiDisplaySource::onReceiveM16Response( if (mClientInfo.mPlaybackSession != NULL) { mClientInfo.mPlaybackSession->updateLiveness(); - - scheduleKeepAlive(sessionID); } return OK; @@ -1257,7 +1273,9 @@ status_t WifiDisplaySource::onSetupRequest( mUsingPCMAudio, mSinkSupportsVideo, mChosenVideoResolutionType, - mChosenVideoResolutionIndex); + mChosenVideoResolutionIndex, + mChosenVideoProfile, + mChosenVideoLevel); if (err != OK) { looper()->unregisterHandler(playbackSession->id()); @@ -1340,7 +1358,9 @@ status_t WifiDisplaySource::onPlayRequest( return ERROR_MALFORMED; } - if (mState != AWAITING_CLIENT_PLAY) { + if (mState != AWAITING_CLIENT_PLAY + && mState != PAUSED_TO_PLAYING + && mState != PAUSED) { ALOGW("Received PLAY request but we're in state %d", mState); sendErrorResponse( @@ -1367,7 +1387,7 @@ status_t WifiDisplaySource::onPlayRequest( return err; } - if (mState == PAUSED_TO_PLAYING) { + if (mState == PAUSED_TO_PLAYING || mPlaybackSessionEstablished) { mState = PLAYING; return OK; } @@ -1401,7 +1421,7 @@ status_t WifiDisplaySource::onPauseRequest( ALOGI("Received PAUSE request."); - if (mState != PLAYING_TO_PAUSED) { + if (mState != PLAYING_TO_PAUSED && mState != PLAYING) { return INVALID_OPERATION; } diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.h b/media/libstagefright/wifi-display/source/WifiDisplaySource.h index 44d3e4d..750265f 100644 --- a/media/libstagefright/wifi-display/source/WifiDisplaySource.h +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.h @@ -18,10 +18,10 @@ #define WIFI_DISPLAY_SOURCE_H_ -#include "ANetworkSession.h" #include "VideoFormats.h" #include <media/stagefright/foundation/AHandler.h> +#include <media/stagefright/foundation/ANetworkSession.h> #include <netinet/in.h> @@ -131,6 +131,8 @@ private: VideoFormats::ResolutionType mChosenVideoResolutionType; size_t mChosenVideoResolutionIndex; + VideoFormats::ProfileType mChosenVideoProfile; + VideoFormats::LevelType mChosenVideoLevel; bool mSinkSupportsAudio; diff --git a/media/libstagefright/wifi-display/wfd.cpp b/media/libstagefright/wifi-display/wfd.cpp deleted file mode 100644 index c947765..0000000 --- a/media/libstagefright/wifi-display/wfd.cpp +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Copyright 2012, The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -//#define LOG_NDEBUG 0 -#define LOG_TAG "wfd" -#include <utils/Log.h> - -#include "source/WifiDisplaySource.h" - -#include <binder/ProcessState.h> -#include <binder/IServiceManager.h> -#include <gui/ISurfaceComposer.h> -#include <gui/SurfaceComposerClient.h> -#include <media/AudioSystem.h> -#include <media/IMediaPlayerService.h> -#include <media/IRemoteDisplay.h> -#include <media/IRemoteDisplayClient.h> -#include <media/stagefright/DataSource.h> -#include <media/stagefright/foundation/ADebug.h> -#include <media/stagefright/foundation/AMessage.h> -#include <ui/DisplayInfo.h> - -namespace android { - -static void usage(const char *me) { - fprintf(stderr, - "usage:\n" - " %s -l iface[:port]\tcreate a wifi display source\n" - " -f(ilename) \tstream media\n", - me); -} - -struct RemoteDisplayClient : public BnRemoteDisplayClient { - RemoteDisplayClient(); - - virtual void onDisplayConnected( - const sp<IGraphicBufferProducer> &bufferProducer, - uint32_t width, - uint32_t height, - uint32_t flags); - - virtual void onDisplayDisconnected(); - virtual void onDisplayError(int32_t error); - - void waitUntilDone(); - -protected: - virtual ~RemoteDisplayClient(); - -private: - Mutex mLock; - Condition mCondition; - - bool mDone; - - sp<SurfaceComposerClient> mComposerClient; - sp<IGraphicBufferProducer> mSurfaceTexture; - sp<IBinder> mDisplayBinder; - - DISALLOW_EVIL_CONSTRUCTORS(RemoteDisplayClient); -}; - -RemoteDisplayClient::RemoteDisplayClient() - : mDone(false) { - mComposerClient = new SurfaceComposerClient; - CHECK_EQ(mComposerClient->initCheck(), (status_t)OK); -} - -RemoteDisplayClient::~RemoteDisplayClient() { -} - -void RemoteDisplayClient::onDisplayConnected( - const sp<IGraphicBufferProducer> &bufferProducer, - uint32_t width, - uint32_t height, - uint32_t flags) { - ALOGI("onDisplayConnected width=%u, height=%u, flags = 0x%08x", - width, height, flags); - - if (bufferProducer != NULL) { - mSurfaceTexture = bufferProducer; - mDisplayBinder = mComposerClient->createDisplay( - String8("foo"), false /* secure */); - - SurfaceComposerClient::openGlobalTransaction(); - mComposerClient->setDisplaySurface(mDisplayBinder, mSurfaceTexture); - - Rect layerStackRect(1280, 720); // XXX fix this. - Rect displayRect(1280, 720); - - mComposerClient->setDisplayProjection( - mDisplayBinder, 0 /* 0 degree rotation */, - layerStackRect, - displayRect); - - SurfaceComposerClient::closeGlobalTransaction(); - } -} - -void RemoteDisplayClient::onDisplayDisconnected() { - ALOGI("onDisplayDisconnected"); - - Mutex::Autolock autoLock(mLock); - mDone = true; - mCondition.broadcast(); -} - -void RemoteDisplayClient::onDisplayError(int32_t error) { - ALOGI("onDisplayError error=%d", error); - - Mutex::Autolock autoLock(mLock); - mDone = true; - mCondition.broadcast(); -} - -void RemoteDisplayClient::waitUntilDone() { - Mutex::Autolock autoLock(mLock); - while (!mDone) { - mCondition.wait(mLock); - } -} - -static status_t enableAudioSubmix(bool enable) { - status_t err = AudioSystem::setDeviceConnectionState( - AUDIO_DEVICE_IN_REMOTE_SUBMIX, - enable - ? AUDIO_POLICY_DEVICE_STATE_AVAILABLE - : AUDIO_POLICY_DEVICE_STATE_UNAVAILABLE, - NULL /* device_address */); - - if (err != OK) { - return err; - } - - err = AudioSystem::setDeviceConnectionState( - AUDIO_DEVICE_OUT_REMOTE_SUBMIX, - enable - ? AUDIO_POLICY_DEVICE_STATE_AVAILABLE - : AUDIO_POLICY_DEVICE_STATE_UNAVAILABLE, - NULL /* device_address */); - - return err; -} - -static void createSource(const AString &addr, int32_t port) { - sp<IServiceManager> sm = defaultServiceManager(); - sp<IBinder> binder = sm->getService(String16("media.player")); - sp<IMediaPlayerService> service = - interface_cast<IMediaPlayerService>(binder); - - CHECK(service.get() != NULL); - - enableAudioSubmix(true /* enable */); - - String8 iface; - iface.append(addr.c_str()); - iface.append(StringPrintf(":%d", port).c_str()); - - sp<RemoteDisplayClient> client = new RemoteDisplayClient; - sp<IRemoteDisplay> display = service->listenForRemoteDisplay(client, iface); - - client->waitUntilDone(); - - display->dispose(); - display.clear(); - - enableAudioSubmix(false /* enable */); -} - -static void createFileSource( - const AString &addr, int32_t port, const char *path) { - sp<ANetworkSession> session = new ANetworkSession; - session->start(); - - sp<ALooper> looper = new ALooper; - looper->start(); - - sp<RemoteDisplayClient> client = new RemoteDisplayClient; - sp<WifiDisplaySource> source = new WifiDisplaySource(session, client, path); - looper->registerHandler(source); - - AString iface = StringPrintf("%s:%d", addr.c_str(), port); - CHECK_EQ((status_t)OK, source->start(iface.c_str())); - - client->waitUntilDone(); - - source->stop(); -} - -} // namespace android - -int main(int argc, char **argv) { - using namespace android; - - ProcessState::self()->startThreadPool(); - - DataSource::RegisterDefaultSniffers(); - - AString listenOnAddr; - int32_t listenOnPort = -1; - - AString path; - - int res; - while ((res = getopt(argc, argv, "hl:f:")) >= 0) { - switch (res) { - case 'f': - { - path = optarg; - break; - } - - case 'l': - { - const char *colonPos = strrchr(optarg, ':'); - - if (colonPos == NULL) { - listenOnAddr = optarg; - listenOnPort = WifiDisplaySource::kWifiDisplayDefaultPort; - } else { - listenOnAddr.setTo(optarg, colonPos - optarg); - - char *end; - listenOnPort = strtol(colonPos + 1, &end, 10); - - if (*end != '\0' || end == colonPos + 1 - || listenOnPort < 1 || listenOnPort > 65535) { - fprintf(stderr, "Illegal port specified.\n"); - exit(1); - } - } - break; - } - - case '?': - case 'h': - default: - usage(argv[0]); - exit(1); - } - } - - if (listenOnPort >= 0) { - if (path.empty()) { - createSource(listenOnAddr, listenOnPort); - } else { - createFileSource(listenOnAddr, listenOnPort, path.c_str()); - } - - exit(0); - } - - usage(argv[0]); - - return 0; -} diff --git a/services/audioflinger/Android.mk b/services/audioflinger/Android.mk index 061a079..54377f1 100644 --- a/services/audioflinger/Android.mk +++ b/services/audioflinger/Android.mk @@ -27,9 +27,6 @@ LOCAL_SRC_FILES:= \ LOCAL_SRC_FILES += StateQueue.cpp -# uncomment for debugging timing problems related to StateQueue::push() -LOCAL_CFLAGS += -DSTATE_QUEUE_DUMP - LOCAL_C_INCLUDES := \ $(call include-path-for, audio-effects) \ $(call include-path-for, audio-utils) @@ -56,24 +53,10 @@ LOCAL_STATIC_LIBRARIES := \ LOCAL_MODULE:= libaudioflinger -LOCAL_SRC_FILES += FastMixer.cpp FastMixerState.cpp - -LOCAL_CFLAGS += -DFAST_MIXER_STATISTICS - -# uncomment to display CPU load adjusted for CPU frequency -# LOCAL_CFLAGS += -DCPU_FREQUENCY_STATISTICS +LOCAL_SRC_FILES += FastMixer.cpp FastMixerState.cpp AudioWatchdog.cpp LOCAL_CFLAGS += -DSTATE_QUEUE_INSTANTIATIONS='"StateQueueInstantiations.cpp"' -LOCAL_CFLAGS += -UFAST_TRACKS_AT_NON_NATIVE_SAMPLE_RATE - -# uncomment to allow tee sink debugging to be enabled by property -# LOCAL_CFLAGS += -DTEE_SINK - -# uncomment to enable the audio watchdog -# LOCAL_SRC_FILES += AudioWatchdog.cpp -# LOCAL_CFLAGS += -DAUDIO_WATCHDOG - # Define ANDROID_SMP appropriately. Used to get inline tracing fast-path. ifeq ($(TARGET_CPU_SMP),true) LOCAL_CFLAGS += -DANDROID_SMP=1 @@ -81,6 +64,8 @@ else LOCAL_CFLAGS += -DANDROID_SMP=0 endif +LOCAL_CFLAGS += -fvisibility=hidden + include $(BUILD_SHARED_LIBRARY) # diff --git a/services/audioflinger/AudioFlinger.cpp b/services/audioflinger/AudioFlinger.cpp index 87eb6aa..a9c9b56 100644 --- a/services/audioflinger/AudioFlinger.cpp +++ b/services/audioflinger/AudioFlinger.cpp @@ -19,6 +19,7 @@ #define LOG_TAG "AudioFlinger" //#define LOG_NDEBUG 0 +#include "Configuration.h" #include <dirent.h> #include <math.h> #include <signal.h> @@ -36,10 +37,6 @@ #include <cutils/bitops.h> #include <cutils/properties.h> -#include <cutils/compiler.h> - -//#include <private/media/AudioTrackShared.h> -//#include <private/media/AudioEffectShared.h> #include <system/audio.h> #include <hardware/audio.h> @@ -58,12 +55,13 @@ #include <powermanager/PowerManager.h> #include <common_time/cc_helper.h> -//#include <common_time/local_clock.h> #include <media/IMediaLogService.h> #include <media/nbaio/Pipe.h> #include <media/nbaio/PipeReader.h> +#include <media/AudioParameter.h> +#include <private/android_filesystem_config.h> // ---------------------------------------------------------------------------- @@ -100,6 +98,10 @@ size_t AudioFlinger::mTeeSinkOutputFrames = kTeeSinkOutputFramesDefault; size_t AudioFlinger::mTeeSinkTrackFrames = kTeeSinkTrackFramesDefault; #endif +// In order to avoid invalidating offloaded tracks each time a Visualizer is turned on and off +// we define a minimum time during which a global effect is considered enabled. +static const nsecs_t kMinGlobalEffectEnabletimeNs = seconds(7200); + // ---------------------------------------------------------------------------- static int load_audio_interface(const char *if_name, audio_hw_device_t **dev) @@ -141,7 +143,10 @@ AudioFlinger::AudioFlinger() mMasterMute(false), mNextUniqueId(1), mMode(AUDIO_MODE_INVALID), - mBtNrecIsOff(false) + mBtNrecIsOff(false), + mIsLowRamDevice(true), + mIsDeviceTypeKnown(false), + mGlobalEffectEnableTime(0) { getpid_cached = getpid(); char value[PROPERTY_VALUE_MAX]; @@ -259,6 +264,12 @@ void AudioFlinger::dumpClients(int fd, const Vector<String16>& args) } } + result.append("Notification Clients:\n"); + for (size_t i = 0; i < mNotificationClients.size(); ++i) { + snprintf(buffer, SIZE, " pid: %d\n", mNotificationClients.keyAt(i)); + result.append(buffer); + } + result.append("Global session refs:\n"); result.append(" session pid count\n"); for (size_t i = 0; i < mAudioSessionRefs.size(); i++) { @@ -436,6 +447,7 @@ sp<IAudioTrack> AudioFlinger::createTrack( audio_io_handle_t output, pid_t tid, int *sessionId, + String8& name, status_t *status) { sp<PlaybackThread::Track> track; @@ -524,6 +536,9 @@ sp<IAudioTrack> AudioFlinger::createTrack( } } if (lStatus == NO_ERROR) { + // s for server's pid, n for normal mixer name, f for fast index + name = String8::format("s:%d;n:%d;f:%d", getpid_cached, track->name() - AudioMixer::TRACK0, + track->fastIndex()); trackHandle = new TrackHandle(track); } else { // remove local strong reference to Client before deleting the Track so that the Client @@ -981,11 +996,12 @@ size_t AudioFlinger::getInputBufferSize(uint32_t sampleRate, audio_format_t form AutoMutex lock(mHardwareLock); mHardwareStatus = AUDIO_HW_GET_INPUT_BUFFER_SIZE; - struct audio_config config = { - sample_rate: sampleRate, - channel_mask: channelMask, - format: format, - }; + struct audio_config config; + memset(&config, 0, sizeof(config)); + config.sample_rate = sampleRate; + config.channel_mask = channelMask; + config.format = format; + audio_hw_device_t *dev = mPrimaryHardwareDev->hwDevice(); size_t size = dev->get_input_buffer_size(dev, &config); mHardwareStatus = AUDIO_HW_IDLE; @@ -1201,13 +1217,17 @@ void AudioFlinger::NotificationClient::binderDied(const wp<IBinder>& who) // ---------------------------------------------------------------------------- +static bool deviceRequiresCaptureAudioOutputPermission(audio_devices_t inDevice) { + return audio_is_remote_submix_device(inDevice); +} + sp<IAudioRecord> AudioFlinger::openRecord( audio_io_handle_t input, uint32_t sampleRate, audio_format_t format, audio_channel_mask_t channelMask, size_t frameCount, - IAudioFlinger::track_flags_t flags, + IAudioFlinger::track_flags_t *flags, pid_t tid, int *sessionId, status_t *status) @@ -1222,19 +1242,34 @@ sp<IAudioRecord> AudioFlinger::openRecord( // check calling permissions if (!recordingAllowed()) { + ALOGE("openRecord() permission denied: recording not allowed"); lStatus = PERMISSION_DENIED; goto Exit; } + if (format != AUDIO_FORMAT_PCM_16_BIT) { + ALOGE("openRecord() invalid format %d", format); + lStatus = BAD_VALUE; + goto Exit; + } + // add client to list { // scope for mLock Mutex::Autolock _l(mLock); thread = checkRecordThread_l(input); if (thread == NULL) { + ALOGE("openRecord() checkRecordThread_l failed"); lStatus = BAD_VALUE; goto Exit; } + if (deviceRequiresCaptureAudioOutputPermission(thread->inDevice()) + && !captureAudioOutputAllowed()) { + ALOGE("openRecord() permission denied: capture not allowed"); + lStatus = PERMISSION_DENIED; + goto Exit; + } + pid_t pid = IPCThreadState::self()->getCallingPid(); client = registerPid_l(pid); @@ -1251,6 +1286,7 @@ sp<IAudioRecord> AudioFlinger::openRecord( // The record track uses one track in mHardwareMixerThread by convention. recordTrack = thread->createRecordTrack_l(client, sampleRate, format, channelMask, frameCount, lSessionId, flags, tid, &lStatus); + LOG_ALWAYS_FATAL_IF((recordTrack != 0) != (lStatus == NO_ERROR)); } if (lStatus != NO_ERROR) { // remove local strong reference to Client before deleting the RecordTrack so that the @@ -1382,31 +1418,53 @@ size_t AudioFlinger::getPrimaryOutputFrameCount() // ---------------------------------------------------------------------------- +status_t AudioFlinger::setLowRamDevice(bool isLowRamDevice) +{ + uid_t uid = IPCThreadState::self()->getCallingUid(); + if (uid != AID_SYSTEM) { + return PERMISSION_DENIED; + } + Mutex::Autolock _l(mLock); + if (mIsDeviceTypeKnown) { + return INVALID_OPERATION; + } + mIsLowRamDevice = isLowRamDevice; + mIsDeviceTypeKnown = true; + return NO_ERROR; +} + +// ---------------------------------------------------------------------------- + audio_io_handle_t AudioFlinger::openOutput(audio_module_handle_t module, audio_devices_t *pDevices, uint32_t *pSamplingRate, audio_format_t *pFormat, audio_channel_mask_t *pChannelMask, uint32_t *pLatencyMs, - audio_output_flags_t flags) + audio_output_flags_t flags, + const audio_offload_info_t *offloadInfo) { - status_t status; PlaybackThread *thread = NULL; - struct audio_config config = { - sample_rate: pSamplingRate ? *pSamplingRate : 0, - channel_mask: pChannelMask ? *pChannelMask : 0, - format: pFormat ? *pFormat : AUDIO_FORMAT_DEFAULT, - }; + struct audio_config config; + config.sample_rate = (pSamplingRate != NULL) ? *pSamplingRate : 0; + config.channel_mask = (pChannelMask != NULL) ? *pChannelMask : 0; + config.format = (pFormat != NULL) ? *pFormat : AUDIO_FORMAT_DEFAULT; + if (offloadInfo) { + config.offload_info = *offloadInfo; + } + audio_stream_out_t *outStream = NULL; AudioHwDevice *outHwDev; - ALOGV("openOutput(), module %d Device %x, SamplingRate %d, Format %d, Channels %x, flags %x", + ALOGV("openOutput(), module %d Device %x, SamplingRate %d, Format %#08x, Channels %x, flags %x", module, (pDevices != NULL) ? *pDevices : 0, config.sample_rate, config.format, config.channel_mask, flags); + ALOGV("openOutput(), offloadInfo %p version 0x%04x", + offloadInfo, offloadInfo == NULL ? -1 : offloadInfo->version ); if (pDevices == NULL || *pDevices == 0) { return 0; @@ -1423,7 +1481,7 @@ audio_io_handle_t AudioFlinger::openOutput(audio_module_handle_t module, mHardwareStatus = AUDIO_HW_OUTPUT_OPEN; - status = hwDevHal->open_output_stream(hwDevHal, + status_t status = hwDevHal->open_output_stream(hwDevHal, id, *pDevices, (audio_output_flags_t)flags, @@ -1431,7 +1489,7 @@ audio_io_handle_t AudioFlinger::openOutput(audio_module_handle_t module, &outStream); mHardwareStatus = AUDIO_HW_IDLE; - ALOGV("openOutput() openOutputStream returned output %p, SamplingRate %d, Format %d, " + ALOGV("openOutput() openOutputStream returned output %p, SamplingRate %d, Format %#08x, " "Channels %x, status %d", outStream, config.sample_rate, @@ -1440,9 +1498,12 @@ audio_io_handle_t AudioFlinger::openOutput(audio_module_handle_t module, status); if (status == NO_ERROR && outStream != NULL) { - AudioStreamOut *output = new AudioStreamOut(outHwDev, outStream); + AudioStreamOut *output = new AudioStreamOut(outHwDev, outStream, flags); - if ((flags & AUDIO_OUTPUT_FLAG_DIRECT) || + if (flags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) { + thread = new OffloadThread(this, output, id, *pDevices); + ALOGV("openOutput() created offload output: ID %d thread %p", id, thread); + } else if ((flags & AUDIO_OUTPUT_FLAG_DIRECT) || (config.format != AUDIO_FORMAT_PCM_16_BIT) || (config.channel_mask != AUDIO_CHANNEL_OUT_STEREO)) { thread = new DirectOutputThread(this, output, id, *pDevices); @@ -1453,10 +1514,18 @@ audio_io_handle_t AudioFlinger::openOutput(audio_module_handle_t module, } mPlaybackThreads.add(id, thread); - if (pSamplingRate != NULL) *pSamplingRate = config.sample_rate; - if (pFormat != NULL) *pFormat = config.format; - if (pChannelMask != NULL) *pChannelMask = config.channel_mask; - if (pLatencyMs != NULL) *pLatencyMs = thread->latency(); + if (pSamplingRate != NULL) { + *pSamplingRate = config.sample_rate; + } + if (pFormat != NULL) { + *pFormat = config.format; + } + if (pChannelMask != NULL) { + *pChannelMask = config.channel_mask; + } + if (pLatencyMs != NULL) { + *pLatencyMs = thread->latency(); + } // notify client processes of the new output creation thread->audioConfigChanged_l(AudioSystem::OUTPUT_OPENED); @@ -1524,11 +1593,28 @@ status_t AudioFlinger::closeOutput_nonvirtual(audio_io_handle_t output) DuplicatingThread *dupThread = (DuplicatingThread *)mPlaybackThreads.valueAt(i).get(); dupThread->removeOutputTrack((MixerThread *)thread.get()); + } } } - audioConfigChanged_l(AudioSystem::OUTPUT_CLOSED, output, NULL); + + mPlaybackThreads.removeItem(output); + // save all effects to the default thread + if (mPlaybackThreads.size()) { + PlaybackThread *dstThread = checkPlaybackThread_l(mPlaybackThreads.keyAt(0)); + if (dstThread != NULL) { + // audioflinger lock is held here so the acquisition order of thread locks does not + // matter + Mutex::Autolock _dl(dstThread->mLock); + Mutex::Autolock _sl(thread->mLock); + Vector< sp<EffectChain> > effectChains = thread->getEffectChains_l(); + for (size_t i = 0; i < effectChains.size(); i ++) { + moveEffectChain_l(effectChains[i]->sessionId(), thread.get(), dstThread, true); + } + } + } + audioConfigChanged_l(AudioSystem::OUTPUT_CLOSED, output, NULL); } thread->exit(); // The thread entity (active unit of execution) is no longer running here, @@ -1583,11 +1669,11 @@ audio_io_handle_t AudioFlinger::openInput(audio_module_handle_t module, { status_t status; RecordThread *thread = NULL; - struct audio_config config = { - sample_rate: pSamplingRate ? *pSamplingRate : 0, - channel_mask: pChannelMask ? *pChannelMask : 0, - format: pFormat ? *pFormat : AUDIO_FORMAT_DEFAULT, - }; + struct audio_config config; + config.sample_rate = (pSamplingRate != NULL) ? *pSamplingRate : 0; + config.channel_mask = (pChannelMask != NULL) ? *pChannelMask : 0; + config.format = (pFormat != NULL) ? *pFormat : AUDIO_FORMAT_DEFAULT; + uint32_t reqSamplingRate = config.sample_rate; audio_format_t reqFormat = config.format; audio_channel_mask_t reqChannels = config.channel_mask; @@ -1683,7 +1769,7 @@ audio_io_handle_t AudioFlinger::openInput(audio_module_handle_t module, AudioStreamIn *input = new AudioStreamIn(inHwDev, inStream); // Start record thread - // RecorThread require both input and output device indication to forward to audio + // RecordThread requires both input and output device indication to forward to audio // pre processing modules thread = new RecordThread(this, input, @@ -1698,9 +1784,15 @@ audio_io_handle_t AudioFlinger::openInput(audio_module_handle_t module, ); mRecordThreads.add(id, thread); ALOGV("openInput() created record thread: ID %d thread %p", id, thread); - if (pSamplingRate != NULL) *pSamplingRate = reqSamplingRate; - if (pFormat != NULL) *pFormat = config.format; - if (pChannelMask != NULL) *pChannelMask = reqChannels; + if (pSamplingRate != NULL) { + *pSamplingRate = reqSamplingRate; + } + if (pFormat != NULL) { + *pFormat = config.format; + } + if (pChannelMask != NULL) { + *pChannelMask = reqChannels; + } // notify client processes of the new input creation thread->audioConfigChanged_l(AudioSystem::INPUT_OPENED); @@ -1768,6 +1860,16 @@ void AudioFlinger::acquireAudioSessionId(int audioSession) Mutex::Autolock _l(mLock); pid_t caller = IPCThreadState::self()->getCallingPid(); ALOGV("acquiring %d from %d", audioSession, caller); + + // Ignore requests received from processes not known as notification client. The request + // is likely proxied by mediaserver (e.g CameraService) and releaseAudioSessionId() can be + // called from a different pid leaving a stale session reference. Also we don't know how + // to clear this reference if the client process dies. + if (mNotificationClients.indexOfKey(caller) < 0) { + ALOGV("acquireAudioSessionId() unknown client %d for session %d", caller, audioSession); + return; + } + size_t num = mAudioSessionRefs.size(); for (size_t i = 0; i< num; i++) { AudioSessionRef *ref = mAudioSessionRefs.editItemAt(i); @@ -1800,7 +1902,9 @@ void AudioFlinger::releaseAudioSessionId(int audioSession) return; } } - ALOGW("session id %d not found for pid %d", audioSession, caller); + // If the caller is mediaserver it is likely that the session being released was acquired + // on behalf of a process not in notification clients and we ignore the warning. + ALOGW_IF(caller != getpid_cached, "session id %d not found for pid %d", audioSession, caller); } void AudioFlinger::purgeStaleEffects_l() { @@ -2001,24 +2105,7 @@ sp<IEffect> AudioFlinger::createEffect( goto Exit; } - if (io == 0) { - if (sessionId == AUDIO_SESSION_OUTPUT_STAGE) { - // output must be specified by AudioPolicyManager when using session - // AUDIO_SESSION_OUTPUT_STAGE - lStatus = BAD_VALUE; - goto Exit; - } else if (sessionId == AUDIO_SESSION_OUTPUT_MIX) { - // if the output returned by getOutputForEffect() is removed before we lock the - // mutex below, the call to checkPlaybackThread_l(io) below will detect it - // and we will exit safely - io = AudioSystem::getOutputForEffect(&desc); - } - } - { - Mutex::Autolock _l(mLock); - - if (!EffectIsNullUuid(&pDesc->uuid)) { // if uuid is specified, request effect descriptor lStatus = EffectGetDescriptor(&pDesc->uuid, &desc); @@ -2091,6 +2178,15 @@ sp<IEffect> AudioFlinger::createEffect( // return effect descriptor *pDesc = desc; + if (io == 0 && sessionId == AUDIO_SESSION_OUTPUT_MIX) { + // if the output returned by getOutputForEffect() is removed before we lock the + // mutex below, the call to checkPlaybackThread_l(io) below will detect it + // and we will exit safely + io = AudioSystem::getOutputForEffect(&desc); + ALOGV("createEffect got output %d", io); + } + + Mutex::Autolock _l(mLock); // If output is not specified try to find a matching audio session ID in one of the // output threads. @@ -2098,6 +2194,12 @@ sp<IEffect> AudioFlinger::createEffect( // because of code checking output when entering the function. // Note: io is never 0 when creating an effect on an input if (io == 0) { + if (sessionId == AUDIO_SESSION_OUTPUT_STAGE) { + // output must be specified by AudioPolicyManager when using session + // AUDIO_SESSION_OUTPUT_STAGE + lStatus = BAD_VALUE; + goto Exit; + } // look for the thread where the specified audio session is present for (size_t i = 0; i < mPlaybackThreads.size(); i++) { if (mPlaybackThreads.valueAt(i)->hasAudioSession(sessionId) != 0) { @@ -2171,9 +2273,7 @@ status_t AudioFlinger::moveEffects(int sessionId, audio_io_handle_t srcOutput, Mutex::Autolock _dl(dstThread->mLock); Mutex::Autolock _sl(srcThread->mLock); - moveEffectChain_l(sessionId, srcThread, dstThread, false); - - return NO_ERROR; + return moveEffectChain_l(sessionId, srcThread, dstThread, false); } // moveEffectChain_l must be called with both srcThread and dstThread mLocks held @@ -2200,13 +2300,18 @@ status_t AudioFlinger::moveEffectChain_l(int sessionId, // transfer all effects one by one so that new effect chain is created on new thread with // correct buffer sizes and audio parameters and effect engines reconfigured accordingly - audio_io_handle_t dstOutput = dstThread->id(); sp<EffectChain> dstChain; uint32_t strategy = 0; // prevent compiler warning sp<EffectModule> effect = chain->getEffectFromId_l(0); + Vector< sp<EffectModule> > removed; + status_t status = NO_ERROR; while (effect != 0) { srcThread->removeEffect_l(effect); - dstThread->addEffect_l(effect); + removed.add(effect); + status = dstThread->addEffect_l(effect); + if (status != NO_ERROR) { + break; + } // removeEffect_l() has stopped the effect if it was active so it must be restarted if (effect->state() == EffectModule::ACTIVE || effect->state() == EffectModule::STOPPING) { @@ -2218,15 +2323,15 @@ status_t AudioFlinger::moveEffectChain_l(int sessionId, dstChain = effect->chain().promote(); if (dstChain == 0) { ALOGW("moveEffectChain_l() cannot get chain from effect %p", effect.get()); - srcThread->addEffect_l(effect); - return NO_INIT; + status = NO_INIT; + break; } strategy = dstChain->strategy(); } if (reRegister) { AudioSystem::unregisterEffect(effect->id()); AudioSystem::registerEffect(&effect->desc(), - dstOutput, + dstThread->id(), strategy, sessionId, effect->id()); @@ -2234,7 +2339,53 @@ status_t AudioFlinger::moveEffectChain_l(int sessionId, effect = chain->getEffectFromId_l(0); } - return NO_ERROR; + if (status != NO_ERROR) { + for (size_t i = 0; i < removed.size(); i++) { + srcThread->addEffect_l(removed[i]); + if (dstChain != 0 && reRegister) { + AudioSystem::unregisterEffect(removed[i]->id()); + AudioSystem::registerEffect(&removed[i]->desc(), + srcThread->id(), + strategy, + sessionId, + removed[i]->id()); + } + } + } + + return status; +} + +bool AudioFlinger::isNonOffloadableGlobalEffectEnabled_l() +{ + if (mGlobalEffectEnableTime != 0 && + ((systemTime() - mGlobalEffectEnableTime) < kMinGlobalEffectEnabletimeNs)) { + return true; + } + + for (size_t i = 0; i < mPlaybackThreads.size(); i++) { + sp<EffectChain> ec = + mPlaybackThreads.valueAt(i)->getEffectChain_l(AUDIO_SESSION_OUTPUT_MIX); + if (ec != 0 && ec->isNonOffloadableEnabled()) { + return true; + } + } + return false; +} + +void AudioFlinger::onNonOffloadableGlobalEffectEnable() +{ + Mutex::Autolock _l(mLock); + + mGlobalEffectEnableTime = systemTime(); + + for (size_t i = 0; i < mPlaybackThreads.size(); i++) { + sp<PlaybackThread> t = mPlaybackThreads.valueAt(i); + if (t->mType == ThreadBase::OFFLOAD) { + t->invalidateTracks(AUDIO_STREAM_MUSIC); + } + } + } struct Entry { diff --git a/services/audioflinger/AudioFlinger.h b/services/audioflinger/AudioFlinger.h index b0efef6..2aeb263 100644 --- a/services/audioflinger/AudioFlinger.h +++ b/services/audioflinger/AudioFlinger.h @@ -24,6 +24,8 @@ #include <common_time/cc_helper.h> +#include <cutils/compiler.h> + #include <media/IAudioFlinger.h> #include <media/IAudioFlingerClient.h> #include <media/IAudioTrack.h> @@ -54,6 +56,7 @@ #include <powermanager/IPowerManager.h> #include <media/nbaio/NBLog.h> +#include <private/media/AudioTrackShared.h> namespace android { @@ -89,7 +92,7 @@ class AudioFlinger : { friend class BinderService<AudioFlinger>; // for AudioFlinger() public: - static const char* getServiceName() { return "media.audio_flinger"; } + static const char* getServiceName() ANDROID_API { return "media.audio_flinger"; } virtual status_t dump(int fd, const Vector<String16>& args); @@ -105,6 +108,7 @@ public: audio_io_handle_t output, pid_t tid, int *sessionId, + String8& name, status_t *status); virtual sp<IAudioRecord> openRecord( @@ -113,7 +117,7 @@ public: audio_format_t format, audio_channel_mask_t channelMask, size_t frameCount, - IAudioFlinger::track_flags_t flags, + IAudioFlinger::track_flags_t *flags, pid_t tid, int *sessionId, status_t *status); @@ -157,7 +161,8 @@ public: audio_format_t *pFormat, audio_channel_mask_t *pChannelMask, uint32_t *pLatencyMs, - audio_output_flags_t flags); + audio_output_flags_t flags, + const audio_offload_info_t *offloadInfo); virtual audio_io_handle_t openDuplicateOutput(audio_io_handle_t output1, audio_io_handle_t output2); @@ -216,6 +221,8 @@ public: virtual uint32_t getPrimaryOutputSamplingRate(); virtual size_t getPrimaryOutputFrameCount(); + virtual status_t setLowRamDevice(bool isLowRamDevice); + virtual status_t onTransact( uint32_t code, const Parcel& data, @@ -278,7 +285,7 @@ private: bool btNrecIsOff() const { return mBtNrecIsOff; } - AudioFlinger(); + AudioFlinger() ANDROID_API; virtual ~AudioFlinger(); // call in any IAudioFlinger method that accesses mPrimaryHardwareDev @@ -359,7 +366,9 @@ private: class PlaybackThread; class MixerThread; class DirectOutputThread; + class OffloadThread; class DuplicatingThread; + class AsyncCallbackThread; class Track; class RecordTrack; class EffectModule; @@ -401,8 +410,13 @@ private: int64_t pts); virtual status_t setMediaTimeTransform(const LinearTransform& xform, int target); + virtual status_t setParameters(const String8& keyValuePairs); + virtual status_t getTimestamp(AudioTimestamp& timestamp); + virtual void signal(); // signal playback thread for a change in control block + virtual status_t onTransact( uint32_t code, const Parcel& data, Parcel* reply, uint32_t flags); + private: const sp<PlaybackThread::Track> mTrack; }; @@ -424,6 +438,7 @@ private: void stop_nonvirtual(); }; + PlaybackThread *checkPlaybackThread_l(audio_io_handle_t output) const; MixerThread *checkMixerThread_l(audio_io_handle_t output) const; RecordThread *checkRecordThread_l(audio_io_handle_t input) const; @@ -452,6 +467,9 @@ private: void removeClient_l(pid_t pid); void removeNotificationClient(pid_t pid); + bool isNonOffloadableGlobalEffectEnabled_l(); + void onNonOffloadableGlobalEffectEnable(); + class AudioHwDevice { public: enum Flags { @@ -490,11 +508,12 @@ private: struct AudioStreamOut { AudioHwDevice* const audioHwDev; audio_stream_out_t* const stream; + audio_output_flags_t flags; audio_hw_device_t* hwDev() const { return audioHwDev->hwDevice(); } - AudioStreamOut(AudioHwDevice *dev, audio_stream_out_t *out) : - audioHwDev(dev), stream(out) {} + AudioStreamOut(AudioHwDevice *dev, audio_stream_out_t *out, audio_output_flags_t flags) : + audioHwDev(dev), stream(out), flags(flags) {} }; struct AudioStreamIn { @@ -588,12 +607,11 @@ private: status_t closeOutput_nonvirtual(audio_io_handle_t output); status_t closeInput_nonvirtual(audio_io_handle_t input); -// do not use #ifdef here, since AudioFlinger.h is included by more than one module -//#ifdef TEE_SINK +#ifdef TEE_SINK // all record threads serially share a common tee sink, which is re-created on format change sp<NBAIO_Sink> mRecordTeeSink; sp<NBAIO_Source> mRecordTeeSource; -//#endif +#endif public: @@ -618,6 +636,16 @@ public: static const size_t kTeeSinkTrackFramesDefault = 0x1000; #endif + // This method reads from a variable without mLock, but the variable is updated under mLock. So + // we might read a stale value, or a value that's inconsistent with respect to other variables. + // In this case, it's safe because the return value isn't used for making an important decision. + // The reason we don't want to take mLock is because it could block the caller for a long time. + bool isLowRamDevice() const { return mIsLowRamDevice; } + +private: + bool mIsLowRamDevice; + bool mIsDeviceTypeKnown; + nsecs_t mGlobalEffectEnableTime; // when a global effect was last enabled }; #undef INCLUDING_FROM_AUDIOFLINGER_H diff --git a/services/audioflinger/AudioMixer.cpp b/services/audioflinger/AudioMixer.cpp index 7d38f80..df4e029 100644 --- a/services/audioflinger/AudioMixer.cpp +++ b/services/audioflinger/AudioMixer.cpp @@ -18,6 +18,7 @@ #define LOG_TAG "AudioMixer" //#define LOG_NDEBUG 0 +#include "Configuration.h" #include <stdint.h> #include <string.h> #include <stdlib.h> diff --git a/services/audioflinger/AudioPolicyService.cpp b/services/audioflinger/AudioPolicyService.cpp index 2706880..35e816b 100644 --- a/services/audioflinger/AudioPolicyService.cpp +++ b/services/audioflinger/AudioPolicyService.cpp @@ -17,6 +17,7 @@ #define LOG_TAG "AudioPolicyService" //#define LOG_NDEBUG 0 +#include "Configuration.h" #undef __STRICT_ANSI__ #define __STDINT_LIMITS #define __STDC_LIMIT_MACROS @@ -40,6 +41,7 @@ #include <system/audio_policy.h> #include <hardware/audio_policy.h> #include <audio_effects/audio_effects_conf.h> +#include <media/AudioParameter.h> namespace android { @@ -49,7 +51,7 @@ static const char kCmdDeadlockedString[] = "AudioPolicyService command thread ma static const int kDumpLockRetries = 50; static const int kDumpLockSleepUs = 20000; -static const nsecs_t kAudioCommandTimeout = 3000000000; // 3 seconds +static const nsecs_t kAudioCommandTimeout = 3000000000LL; // 3 seconds namespace { extern struct audio_policy_service_ops aps_ops; @@ -68,10 +70,11 @@ AudioPolicyService::AudioPolicyService() Mutex::Autolock _l(mLock); // start tone playback thread - mTonePlaybackThread = new AudioCommandThread(String8("")); + mTonePlaybackThread = new AudioCommandThread(String8("ApmTone"), this); // start audio commands thread - mAudioCommandThread = new AudioCommandThread(String8("ApmCommand")); - + mAudioCommandThread = new AudioCommandThread(String8("ApmAudio"), this); + // start output activity command thread + mOutputCommandThread = new AudioCommandThread(String8("ApmOutput"), this); /* instantiate the audio policy manager */ rc = hw_get_module(AUDIO_POLICY_HARDWARE_MODULE_ID, &module); if (rc) @@ -222,15 +225,16 @@ audio_io_handle_t AudioPolicyService::getOutput(audio_stream_type_t stream, uint32_t samplingRate, audio_format_t format, audio_channel_mask_t channelMask, - audio_output_flags_t flags) + audio_output_flags_t flags, + const audio_offload_info_t *offloadInfo) { if (mpAudioPolicy == NULL) { return 0; } ALOGV("getOutput()"); Mutex::Autolock _l(mLock); - return mpAudioPolicy->get_output(mpAudioPolicy, stream, samplingRate, format, channelMask, - flags); + return mpAudioPolicy->get_output(mpAudioPolicy, stream, samplingRate, + format, channelMask, flags, offloadInfo); } status_t AudioPolicyService::startOutput(audio_io_handle_t output, @@ -253,6 +257,15 @@ status_t AudioPolicyService::stopOutput(audio_io_handle_t output, return NO_INIT; } ALOGV("stopOutput()"); + mOutputCommandThread->stopOutputCommand(output, stream, session); + return NO_ERROR; +} + +status_t AudioPolicyService::doStopOutput(audio_io_handle_t output, + audio_stream_type_t stream, + int session) +{ + ALOGV("doStopOutput from tid %d", gettid()); Mutex::Autolock _l(mLock); return mpAudioPolicy->stop_output(mpAudioPolicy, output, stream, session); } @@ -263,6 +276,12 @@ void AudioPolicyService::releaseOutput(audio_io_handle_t output) return; } ALOGV("releaseOutput()"); + mOutputCommandThread->releaseOutputCommand(output); +} + +void AudioPolicyService::doReleaseOutput(audio_io_handle_t output) +{ + ALOGV("doReleaseOutput from tid %d", gettid()); Mutex::Autolock _l(mLock); mpAudioPolicy->release_output(mpAudioPolicy, output); } @@ -277,9 +296,14 @@ audio_io_handle_t AudioPolicyService::getInput(audio_source_t inputSource, return 0; } // already checked by client, but double-check in case the client wrapper is bypassed - if (uint32_t(inputSource) >= AUDIO_SOURCE_CNT) { + if (inputSource >= AUDIO_SOURCE_CNT && inputSource != AUDIO_SOURCE_HOTWORD) { return 0; } + + if ((inputSource == AUDIO_SOURCE_HOTWORD) && !captureHotwordAllowed()) { + return 0; + } + Mutex::Autolock _l(mLock); // the audio_in_acoustics_t parameter is ignored by get_input() audio_io_handle_t input = mpAudioPolicy->get_input(mpAudioPolicy, inputSource, samplingRate, @@ -289,7 +313,10 @@ audio_io_handle_t AudioPolicyService::getInput(audio_source_t inputSource, return input; } // create audio pre processors according to input source - ssize_t index = mInputSources.indexOfKey(inputSource); + audio_source_t aliasSource = (inputSource == AUDIO_SOURCE_HOTWORD) ? + AUDIO_SOURCE_VOICE_RECOGNITION : inputSource; + + ssize_t index = mInputSources.indexOfKey(aliasSource); if (index < 0) { return input; } @@ -638,8 +665,9 @@ status_t AudioPolicyService::onTransact( // ----------- AudioPolicyService::AudioCommandThread implementation ---------- -AudioPolicyService::AudioCommandThread::AudioCommandThread(String8 name) - : Thread(false), mName(name) +AudioPolicyService::AudioCommandThread::AudioCommandThread(String8 name, + const wp<AudioPolicyService>& service) + : Thread(false), mName(name), mService(service) { mpToneGenerator = NULL; } @@ -647,7 +675,7 @@ AudioPolicyService::AudioCommandThread::AudioCommandThread(String8 name) AudioPolicyService::AudioCommandThread::~AudioCommandThread() { - if (mName != "" && !mAudioCommands.isEmpty()) { + if (!mAudioCommands.isEmpty()) { release_wake_lock(mName.string()); } mAudioCommands.clear(); @@ -656,11 +684,7 @@ AudioPolicyService::AudioCommandThread::~AudioCommandThread() void AudioPolicyService::AudioCommandThread::onFirstRef() { - if (mName != "") { - run(mName.string(), ANDROID_PRIORITY_AUDIO); - } else { - run("AudioCommand", ANDROID_PRIORITY_AUDIO); - } + run(mName.string(), ANDROID_PRIORITY_AUDIO); } bool AudioPolicyService::AudioCommandThread::threadLoop() @@ -735,6 +759,32 @@ bool AudioPolicyService::AudioCommandThread::threadLoop() } delete data; }break; + case STOP_OUTPUT: { + StopOutputData *data = (StopOutputData *)command->mParam; + ALOGV("AudioCommandThread() processing stop output %d", + data->mIO); + sp<AudioPolicyService> svc = mService.promote(); + if (svc == 0) { + break; + } + mLock.unlock(); + svc->doStopOutput(data->mIO, data->mStream, data->mSession); + mLock.lock(); + delete data; + }break; + case RELEASE_OUTPUT: { + ReleaseOutputData *data = (ReleaseOutputData *)command->mParam; + ALOGV("AudioCommandThread() processing release output %d", + data->mIO); + sp<AudioPolicyService> svc = mService.promote(); + if (svc == 0) { + break; + } + mLock.unlock(); + svc->doReleaseOutput(data->mIO); + mLock.lock(); + delete data; + }break; default: ALOGW("AudioCommandThread() unknown command %d", command->mCommand); } @@ -746,7 +796,7 @@ bool AudioPolicyService::AudioCommandThread::threadLoop() } } // release delayed commands wake lock - if (mName != "" && mAudioCommands.isEmpty()) { + if (mAudioCommands.isEmpty()) { release_wake_lock(mName.string()); } ALOGV("AudioCommandThread() going to sleep"); @@ -890,17 +940,45 @@ status_t AudioPolicyService::AudioCommandThread::voiceVolumeCommand(float volume return status; } +void AudioPolicyService::AudioCommandThread::stopOutputCommand(audio_io_handle_t output, + audio_stream_type_t stream, + int session) +{ + AudioCommand *command = new AudioCommand(); + command->mCommand = STOP_OUTPUT; + StopOutputData *data = new StopOutputData(); + data->mIO = output; + data->mStream = stream; + data->mSession = session; + command->mParam = (void *)data; + Mutex::Autolock _l(mLock); + insertCommand_l(command); + ALOGV("AudioCommandThread() adding stop output %d", output); + mWaitWorkCV.signal(); +} + +void AudioPolicyService::AudioCommandThread::releaseOutputCommand(audio_io_handle_t output) +{ + AudioCommand *command = new AudioCommand(); + command->mCommand = RELEASE_OUTPUT; + ReleaseOutputData *data = new ReleaseOutputData(); + data->mIO = output; + command->mParam = (void *)data; + Mutex::Autolock _l(mLock); + insertCommand_l(command); + ALOGV("AudioCommandThread() adding release output %d", output); + mWaitWorkCV.signal(); +} + // insertCommand_l() must be called with mLock held void AudioPolicyService::AudioCommandThread::insertCommand_l(AudioCommand *command, int delayMs) { ssize_t i; // not size_t because i will count down to -1 Vector <AudioCommand *> removedCommands; - - nsecs_t time = 0; command->mTime = systemTime() + milliseconds(delayMs); // acquire wake lock to make sure delayed commands are processed - if (mName != "" && mAudioCommands.isEmpty()) { + if (mAudioCommands.isEmpty()) { acquire_wake_lock(PARTIAL_WAKE_LOCK, mName.string()); } @@ -942,7 +1020,10 @@ void AudioPolicyService::AudioCommandThread::insertCommand_l(AudioCommand *comma } else { data2->mKeyValuePairs = param2.toString(); } - time = command2->mTime; + command->mTime = command2->mTime; + // force delayMs to non 0 so that code below does not request to wait for + // command status as the command is now delayed + delayMs = 1; } break; case SET_VOLUME: { @@ -953,7 +1034,10 @@ void AudioPolicyService::AudioCommandThread::insertCommand_l(AudioCommand *comma ALOGV("Filtering out volume command on output %d for stream %d", data->mIO, data->mStream); removedCommands.add(command2); - time = command2->mTime; + command->mTime = command2->mTime; + // force delayMs to non 0 so that code below does not request to wait for + // command status as the command is now delayed + delayMs = 1; } break; case START_TONE: case STOP_TONE: @@ -975,16 +1059,12 @@ void AudioPolicyService::AudioCommandThread::insertCommand_l(AudioCommand *comma } removedCommands.clear(); - // wait for status only if delay is 0 and command time was not modified above - if (delayMs == 0 && time == 0) { + // wait for status only if delay is 0 + if (delayMs == 0) { command->mWaitStatus = true; } else { command->mWaitStatus = false; } - // update command time if modified above - if (time != 0) { - command->mTime = time; - } // insert command at the right place according to its time stamp ALOGV("inserting command: %d at index %d, num commands %d", @@ -1055,6 +1135,21 @@ int AudioPolicyService::setVoiceVolume(float volume, int delayMs) return (int)mAudioCommandThread->voiceVolumeCommand(volume, delayMs); } +bool AudioPolicyService::isOffloadSupported(const audio_offload_info_t& info) +{ + if (mpAudioPolicy == NULL) { + ALOGV("mpAudioPolicy == NULL"); + return false; + } + + if (mpAudioPolicy->is_offload_supported == NULL) { + ALOGV("HAL does not implement is_offload_supported"); + return false; + } + + return mpAudioPolicy->is_offload_supported(mpAudioPolicy, &info); +} + // ---------------------------------------------------------------------------- // Audio pre-processing configuration // ---------------------------------------------------------------------------- @@ -1387,7 +1482,8 @@ static audio_io_handle_t aps_open_output_on_module(void *service, audio_format_t *pFormat, audio_channel_mask_t *pChannelMask, uint32_t *pLatencyMs, - audio_output_flags_t flags) + audio_output_flags_t flags, + const audio_offload_info_t *offloadInfo) { sp<IAudioFlinger> af = AudioSystem::get_audio_flinger(); if (af == 0) { @@ -1395,7 +1491,7 @@ static audio_io_handle_t aps_open_output_on_module(void *service, return 0; } return af->openOutput(module, pDevices, pSamplingRate, pFormat, pChannelMask, - pLatencyMs, flags); + pLatencyMs, flags, offloadInfo); } static audio_io_handle_t aps_open_dup_output(void *service, diff --git a/services/audioflinger/AudioPolicyService.h b/services/audioflinger/AudioPolicyService.h index 35cf368..ae053a9 100644 --- a/services/audioflinger/AudioPolicyService.h +++ b/services/audioflinger/AudioPolicyService.h @@ -19,6 +19,7 @@ #include <cutils/misc.h> #include <cutils/config_utils.h> +#include <cutils/compiler.h> #include <utils/String8.h> #include <utils/Vector.h> #include <utils/SortedVector.h> @@ -44,7 +45,7 @@ class AudioPolicyService : public: // for BinderService - static const char *getServiceName() { return "media.audio_policy"; } + static const char *getServiceName() ANDROID_API { return "media.audio_policy"; } virtual status_t dump(int fd, const Vector<String16>& args); @@ -66,7 +67,8 @@ public: audio_format_t format = AUDIO_FORMAT_DEFAULT, audio_channel_mask_t channelMask = 0, audio_output_flags_t flags = - AUDIO_OUTPUT_FLAG_NONE); + AUDIO_OUTPUT_FLAG_NONE, + const audio_offload_info_t *offloadInfo = NULL); virtual status_t startOutput(audio_io_handle_t output, audio_stream_type_t stream, int session = 0); @@ -135,9 +137,15 @@ public: virtual status_t startTone(audio_policy_tone_t tone, audio_stream_type_t stream); virtual status_t stopTone(); virtual status_t setVoiceVolume(float volume, int delayMs = 0); + virtual bool isOffloadSupported(const audio_offload_info_t &config); + + status_t doStopOutput(audio_io_handle_t output, + audio_stream_type_t stream, + int session = 0); + void doReleaseOutput(audio_io_handle_t output); private: - AudioPolicyService(); + AudioPolicyService() ANDROID_API; virtual ~AudioPolicyService(); status_t dumpInternals(int fd); @@ -158,10 +166,12 @@ private: STOP_TONE, SET_VOLUME, SET_PARAMETERS, - SET_VOICE_VOLUME + SET_VOICE_VOLUME, + STOP_OUTPUT, + RELEASE_OUTPUT }; - AudioCommandThread (String8 name); + AudioCommandThread (String8 name, const wp<AudioPolicyService>& service); virtual ~AudioCommandThread(); status_t dump(int fd); @@ -179,6 +189,11 @@ private: status_t parametersCommand(audio_io_handle_t ioHandle, const char *keyValuePairs, int delayMs = 0); status_t voiceVolumeCommand(float volume, int delayMs = 0); + void stopOutputCommand(audio_io_handle_t output, + audio_stream_type_t stream, + int session); + void releaseOutputCommand(audio_io_handle_t output); + void insertCommand_l(AudioCommand *command, int delayMs = 0); private: @@ -223,12 +238,25 @@ private: float mVolume; }; + class StopOutputData { + public: + audio_io_handle_t mIO; + audio_stream_type_t mStream; + int mSession; + }; + + class ReleaseOutputData { + public: + audio_io_handle_t mIO; + }; + Mutex mLock; Condition mWaitWorkCV; Vector <AudioCommand *> mAudioCommands; // list of pending commands ToneGenerator *mpToneGenerator; // the tone generator AudioCommand mLastCommand; // last processed command (used by dump) String8 mName; // string used by wake lock fo delayed commands + wp<AudioPolicyService> mService; }; class EffectDesc { @@ -313,6 +341,7 @@ private: // device connection state or routing sp<AudioCommandThread> mAudioCommandThread; // audio commands thread sp<AudioCommandThread> mTonePlaybackThread; // tone playback thread + sp<AudioCommandThread> mOutputCommandThread; // process stop and release output struct audio_policy_device *mpAudioPolicyDev; struct audio_policy *mpAudioPolicy; KeyedVector< audio_source_t, InputSourceDesc* > mInputSources; diff --git a/services/audioflinger/AudioResampler.h b/services/audioflinger/AudioResampler.h index 2b8694f..33e64ce 100644 --- a/services/audioflinger/AudioResampler.h +++ b/services/audioflinger/AudioResampler.h @@ -19,13 +19,14 @@ #include <stdint.h> #include <sys/types.h> +#include <cutils/compiler.h> #include <media/AudioBufferProvider.h> namespace android { // ---------------------------------------------------------------------------- -class AudioResampler { +class ANDROID_API AudioResampler { public: // Determines quality of SRC. // LOW_QUALITY: linear interpolator (1st order) @@ -55,6 +56,14 @@ public: // set the PTS of the next buffer output by the resampler virtual void setPTS(int64_t pts); + // Resample int16_t samples from provider and accumulate into 'out'. + // A mono provider delivers a sequence of samples. + // A stereo provider delivers a sequence of interleaved pairs of samples. + // Multi-channel providers are not supported. + // In either case, 'out' holds interleaved pairs of fixed-point signed Q19.12. + // That is, for a mono provider, there is an implicit up-channeling. + // Since this method accumulates, the caller is responsible for clearing 'out' initially. + // FIXME assumes provider is always successful; it should return the actual frame count. virtual void resample(int32_t* out, size_t outFrameCount, AudioBufferProvider* provider) = 0; diff --git a/services/audioflinger/AudioWatchdog.cpp b/services/audioflinger/AudioWatchdog.cpp index 8f328ee..93d185e 100644 --- a/services/audioflinger/AudioWatchdog.cpp +++ b/services/audioflinger/AudioWatchdog.cpp @@ -17,9 +17,12 @@ #define LOG_TAG "AudioWatchdog" //#define LOG_NDEBUG 0 +#include "Configuration.h" #include <utils/Log.h> #include "AudioWatchdog.h" +#ifdef AUDIO_WATCHDOG + namespace android { void AudioWatchdogDump::dump(int fd) @@ -132,3 +135,5 @@ void AudioWatchdog::setDump(AudioWatchdogDump *dump) } } // namespace android + +#endif // AUDIO_WATCHDOG diff --git a/services/audioflinger/Configuration.h b/services/audioflinger/Configuration.h new file mode 100644 index 0000000..bc2038a --- /dev/null +++ b/services/audioflinger/Configuration.h @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Put build-time configuration options here rather than Android.mk, +// so that the instantiate for AudioFlinger service will pick up the same options. + +#ifndef ANDROID_AUDIOFLINGER_CONFIGURATION_H +#define ANDROID_AUDIOFLINGER_CONFIGURATION_H + +// uncomment to enable detailed battery usage reporting (not debugged) +//#define ADD_BATTERY_DATA + +// uncomment to enable the audio watchdog +//#define AUDIO_WATCHDOG + +// uncomment to display CPU load adjusted for CPU frequency +//#define CPU_FREQUENCY_STATISTICS + +// uncomment to enable fast mixer to take performance samples for later statistical analysis +#define FAST_MIXER_STATISTICS + +// uncomment to allow fast tracks at non-native sample rate +//#define FAST_TRACKS_AT_NON_NATIVE_SAMPLE_RATE + +// uncomment for debugging timing problems related to StateQueue::push() +//#define STATE_QUEUE_DUMP + +// uncomment to allow tee sink debugging to be enabled by property +//#define TEE_SINK + +// uncomment to log CPU statistics every n wall clock seconds +//#define DEBUG_CPU_USAGE 10 + +#endif // ANDROID_AUDIOFLINGER_CONFIGURATION_H diff --git a/services/audioflinger/Effects.cpp b/services/audioflinger/Effects.cpp index 942ea35..a8a5169 100644 --- a/services/audioflinger/Effects.cpp +++ b/services/audioflinger/Effects.cpp @@ -19,6 +19,7 @@ #define LOG_TAG "AudioFlinger" //#define LOG_NDEBUG 0 +#include "Configuration.h" #include <utils/Log.h> #include <audio_effects/effect_visualizer.h> #include <audio_utils/primitives.h> @@ -94,16 +95,7 @@ AudioFlinger::EffectModule::~EffectModule() { ALOGV("Destructor %p", this); if (mEffectInterface != NULL) { - if ((mDescriptor.flags & EFFECT_FLAG_TYPE_MASK) == EFFECT_FLAG_TYPE_PRE_PROC || - (mDescriptor.flags & EFFECT_FLAG_TYPE_MASK) == EFFECT_FLAG_TYPE_POST_PROC) { - sp<ThreadBase> thread = mThread.promote(); - if (thread != 0) { - audio_stream_t *stream = thread->stream(); - if (stream != NULL) { - stream->remove_audio_effect(stream, mEffectInterface); - } - } - } + remove_effect_from_hal_l(); // release effect engine EffectRelease(mEffectInterface); } @@ -487,7 +479,7 @@ status_t AudioFlinger::EffectModule::stop_l() if (mStatus != NO_ERROR) { return mStatus; } - status_t cmdStatus; + status_t cmdStatus = NO_ERROR; uint32_t size = sizeof(status_t); status_t status = (*mEffectInterface)->command(mEffectInterface, EFFECT_CMD_DISABLE, @@ -495,12 +487,19 @@ status_t AudioFlinger::EffectModule::stop_l() NULL, &size, &cmdStatus); - if (status == 0) { + if (status == NO_ERROR) { status = cmdStatus; } - if (status == 0 && - ((mDescriptor.flags & EFFECT_FLAG_TYPE_MASK) == EFFECT_FLAG_TYPE_PRE_PROC || - (mDescriptor.flags & EFFECT_FLAG_TYPE_MASK) == EFFECT_FLAG_TYPE_POST_PROC)) { + if (status == NO_ERROR) { + status = remove_effect_from_hal_l(); + } + return status; +} + +status_t AudioFlinger::EffectModule::remove_effect_from_hal_l() +{ + if ((mDescriptor.flags & EFFECT_FLAG_TYPE_MASK) == EFFECT_FLAG_TYPE_PRE_PROC || + (mDescriptor.flags & EFFECT_FLAG_TYPE_MASK) == EFFECT_FLAG_TYPE_POST_PROC) { sp<ThreadBase> thread = mThread.promote(); if (thread != 0) { audio_stream_t *stream = thread->stream(); @@ -509,7 +508,7 @@ status_t AudioFlinger::EffectModule::stop_l() } } } - return status; + return NO_ERROR; } status_t AudioFlinger::EffectModule::command(uint32_t cmdCode, @@ -765,6 +764,46 @@ bool AudioFlinger::EffectModule::purgeHandles() return enabled; } +status_t AudioFlinger::EffectModule::setOffloaded(bool offloaded, audio_io_handle_t io) +{ + Mutex::Autolock _l(mLock); + if (mStatus != NO_ERROR) { + return mStatus; + } + status_t status = NO_ERROR; + if ((mDescriptor.flags & EFFECT_FLAG_OFFLOAD_SUPPORTED) != 0) { + status_t cmdStatus; + uint32_t size = sizeof(status_t); + effect_offload_param_t cmd; + + cmd.isOffload = offloaded; + cmd.ioHandle = io; + status = (*mEffectInterface)->command(mEffectInterface, + EFFECT_CMD_OFFLOAD, + sizeof(effect_offload_param_t), + &cmd, + &size, + &cmdStatus); + if (status == NO_ERROR) { + status = cmdStatus; + } + mOffloaded = (status == NO_ERROR) ? offloaded : false; + } else { + if (offloaded) { + status = INVALID_OPERATION; + } + mOffloaded = false; + } + ALOGV("setOffloaded() offloaded %d io %d status %d", offloaded, io, status); + return status; +} + +bool AudioFlinger::EffectModule::isOffloaded() const +{ + Mutex::Autolock _l(mLock); + return mOffloaded; +} + void AudioFlinger::EffectModule::dump(int fd, const Vector<String16>& args) { const size_t SIZE = 256; @@ -932,6 +971,23 @@ status_t AudioFlinger::EffectHandle::enable() thread->checkSuspendOnEffectEnabled(mEffect, false, mEffect->sessionId()); } mEnabled = false; + } else { + if (thread != 0) { + if (thread->type() == ThreadBase::OFFLOAD) { + PlaybackThread *t = (PlaybackThread *)thread.get(); + Mutex::Autolock _l(t->mLock); + t->broadcast_l(); + } + if (!mEffect->isOffloadable()) { + if (thread->type() == ThreadBase::OFFLOAD) { + PlaybackThread *t = (PlaybackThread *)thread.get(); + t->invalidateTracks(AUDIO_STREAM_MUSIC); + } + if (mEffect->sessionId() == AUDIO_SESSION_OUTPUT_MIX) { + thread->mAudioFlinger->onNonOffloadableGlobalEffectEnable(); + } + } + } } return status; } @@ -960,6 +1016,11 @@ status_t AudioFlinger::EffectHandle::disable() sp<ThreadBase> thread = mEffect->thread().promote(); if (thread != 0) { thread->checkSuspendOnEffectEnabled(mEffect, false, mEffect->sessionId()); + if (thread->type() == ThreadBase::OFFLOAD) { + PlaybackThread *t = (PlaybackThread *)thread.get(); + Mutex::Autolock _l(t->mLock); + t->broadcast_l(); + } } return status; @@ -1217,9 +1278,7 @@ void AudioFlinger::EffectChain::clearInputBuffer() // Must be called with EffectChain::mLock locked void AudioFlinger::EffectChain::clearInputBuffer_l(sp<ThreadBase> thread) { - size_t numSamples = thread->frameCount() * thread->channelCount(); - memset(mInBuffer, 0, numSamples * sizeof(int16_t)); - + memset(mInBuffer, 0, thread->frameCount() * thread->frameSize()); } // Must be called with EffectChain::mLock locked @@ -1232,9 +1291,10 @@ void AudioFlinger::EffectChain::process_l() } bool isGlobalSession = (mSessionId == AUDIO_SESSION_OUTPUT_MIX) || (mSessionId == AUDIO_SESSION_OUTPUT_STAGE); - // always process effects unless no more tracks are on the session and the effect tail - // has been rendered - bool doProcess = true; + // never process effects when: + // - on an OFFLOAD thread + // - no more tracks are on the session and the effect tail has been rendered + bool doProcess = (thread->type() != ThreadBase::OFFLOAD); if (!isGlobalSession) { bool tracksOnSession = (trackCnt() != 0); @@ -1720,4 +1780,16 @@ void AudioFlinger::EffectChain::checkSuspendOnEffectEnabled(const sp<EffectModul } } +bool AudioFlinger::EffectChain::isNonOffloadableEnabled() +{ + Mutex::Autolock _l(mLock); + size_t size = mEffects.size(); + for (size_t i = 0; i < size; i++) { + if (mEffects[i]->isEnabled() && !mEffects[i]->isOffloadable()) { + return true; + } + } + return false; +} + }; // namespace android diff --git a/services/audioflinger/Effects.h b/services/audioflinger/Effects.h index 91303ee..b717857 100644 --- a/services/audioflinger/Effects.h +++ b/services/audioflinger/Effects.h @@ -25,6 +25,10 @@ // state changes or resource modifications. Always respect the following order // if multiple mutexes must be acquired to avoid cross deadlock: // AudioFlinger -> ThreadBase -> EffectChain -> EffectModule +// In addition, methods that lock the AudioPolicyService mutex (getOutputForEffect(), +// startOutput()...) should never be called with AudioFlinger or Threadbase mutex locked +// to avoid cross deadlock with other clients calling AudioPolicyService methods that in turn +// call AudioFlinger thus locking the same mutexes in the reverse order. // The EffectModule class is a wrapper object controlling the effect engine implementation // in the effect library. It prevents concurrent calls to process() and command() functions @@ -111,6 +115,10 @@ public: bool purgeHandles(); void lock() { mLock.lock(); } void unlock() { mLock.unlock(); } + bool isOffloadable() const + { return (mDescriptor.flags & EFFECT_FLAG_OFFLOAD_SUPPORTED) != 0; } + status_t setOffloaded(bool offloaded, audio_io_handle_t io); + bool isOffloaded() const; void dump(int fd, const Vector<String16>& args); @@ -126,6 +134,7 @@ protected: status_t start_l(); status_t stop_l(); + status_t remove_effect_from_hal_l(); mutable Mutex mLock; // mutex for process, commands and handles list protection wp<ThreadBase> mThread; // parent thread @@ -143,6 +152,7 @@ mutable Mutex mLock; // mutex for process, commands and handl // sending disable command. uint32_t mDisableWaitCnt; // current process() calls count during disable period. bool mSuspended; // effect is suspended: temporarily disabled by framework + bool mOffloaded; // effect is currently offloaded to the audio DSP }; // The EffectHandle class implements the IEffect interface. It provides resources @@ -302,6 +312,10 @@ public: void clearInputBuffer(); + // At least one non offloadable effect in the chain is enabled + bool isNonOffloadableEnabled(); + + void dump(int fd, const Vector<String16>& args); protected: diff --git a/services/audioflinger/FastMixer.cpp b/services/audioflinger/FastMixer.cpp index 21df1d7..f27ea17 100644 --- a/services/audioflinger/FastMixer.cpp +++ b/services/audioflinger/FastMixer.cpp @@ -25,6 +25,7 @@ #define ATRACE_TAG ATRACE_TAG_AUDIO +#include "Configuration.h" #include <sys/atomics.h> #include <time.h> #include <utils/Log.h> @@ -44,6 +45,8 @@ #define MIN_WARMUP_CYCLES 2 // minimum number of loop cycles to wait for warmup #define MAX_WARMUP_CYCLES 10 // maximum number of loop cycles to wait for warmup +#define FCC_2 2 // fixed channel count assumption + namespace android { // Fast mixer thread @@ -82,7 +85,7 @@ bool FastMixer::threadLoop() struct timespec oldLoad = {0, 0}; // previous value of clock_gettime(CLOCK_THREAD_CPUTIME_ID) bool oldLoadValid = false; // whether oldLoad is valid uint32_t bounds = 0; - bool full = false; // whether we have collected at least kSamplingN samples + bool full = false; // whether we have collected at least mSamplingN samples #ifdef CPU_FREQUENCY_STATISTICS ThreadCpuUsage tcu; // for reading the current CPU clock frequency in kHz #endif @@ -93,6 +96,12 @@ bool FastMixer::threadLoop() uint32_t warmupCycles = 0; // counter of number of loop cycles required to warmup NBAIO_Sink* teeSink = NULL; // if non-NULL, then duplicate write() to this non-blocking sink NBLog::Writer dummyLogWriter, *logWriter = &dummyLogWriter; + uint32_t totalNativeFramesWritten = 0; // copied to dumpState->mFramesWritten + + // next 2 fields are valid only when timestampStatus == NO_ERROR + AudioTimestamp timestamp; + uint32_t nativeFramesWrittenButNotPresented = 0; // the = 0 is to silence the compiler + status_t timestampStatus = INVALID_OPERATION; for (;;) { @@ -142,7 +151,9 @@ bool FastMixer::threadLoop() preIdle = *current; current = &preIdle; oldTsValid = false; +#ifdef FAST_MIXER_STATISTICS oldLoadValid = false; +#endif ignoreNextOverrun = true; } previous = current; @@ -182,9 +193,12 @@ bool FastMixer::threadLoop() warmupCycles = 0; sleepNs = -1; coldGen = current->mColdGen; +#ifdef FAST_MIXER_STATISTICS bounds = 0; full = false; +#endif oldTsValid = !clock_gettime(CLOCK_MONOTONIC, &oldTs); + timestampStatus = INVALID_OPERATION; } else { sleepNs = FAST_HOT_IDLE_NS; } @@ -220,7 +234,7 @@ bool FastMixer::threadLoop() } else { format = outputSink->format(); sampleRate = Format_sampleRate(format); - ALOG_ASSERT(Format_channelCount(format) == 2); + ALOG_ASSERT(Format_channelCount(format) == FCC_2); } dumpState->mSampleRate = sampleRate; } @@ -236,7 +250,7 @@ bool FastMixer::threadLoop() // implementation; it would be better to have normal mixer allocate for us // to avoid blocking here and to prevent possible priority inversion mixer = new AudioMixer(frameCount, sampleRate, FastMixerState::kMaxFastTracks); - mixBuffer = new short[frameCount * 2]; + mixBuffer = new short[frameCount * FCC_2]; periodNs = (frameCount * 1000000000LL) / sampleRate; // 1.00 underrunNs = (frameCount * 1750000000LL) / sampleRate; // 1.75 overrunNs = (frameCount * 500000000LL) / sampleRate; // 0.50 @@ -375,6 +389,31 @@ bool FastMixer::threadLoop() i = __builtin_ctz(currentTrackMask); currentTrackMask &= ~(1 << i); const FastTrack* fastTrack = ¤t->mFastTracks[i]; + + // Refresh the per-track timestamp + if (timestampStatus == NO_ERROR) { + uint32_t trackFramesWrittenButNotPresented; + uint32_t trackSampleRate = fastTrack->mSampleRate; + // There is currently no sample rate conversion for fast tracks currently + if (trackSampleRate != 0 && trackSampleRate != sampleRate) { + trackFramesWrittenButNotPresented = + ((int64_t) nativeFramesWrittenButNotPresented * trackSampleRate) / + sampleRate; + } else { + trackFramesWrittenButNotPresented = nativeFramesWrittenButNotPresented; + } + uint32_t trackFramesWritten = fastTrack->mBufferProvider->framesReleased(); + // Can't provide an AudioTimestamp before first frame presented, + // or during the brief 32-bit wraparound window + if (trackFramesWritten >= trackFramesWrittenButNotPresented) { + AudioTimestamp perTrackTimestamp; + perTrackTimestamp.mPosition = + trackFramesWritten - trackFramesWrittenButNotPresented; + perTrackTimestamp.mTime = timestamp.mTime; + fastTrack->mBufferProvider->onTimestamp(perTrackTimestamp); + } + } + int name = fastTrackNames[i]; ALOG_ASSERT(name >= 0); if (fastTrack->mVolumeProvider != NULL) { @@ -433,7 +472,7 @@ bool FastMixer::threadLoop() //bool didFullWrite = false; // dumpsys could display a count of partial writes if ((command & FastMixerState::WRITE) && (outputSink != NULL) && (mixBuffer != NULL)) { if (mixBufferState == UNDEFINED) { - memset(mixBuffer, 0, frameCount * 2 * sizeof(short)); + memset(mixBuffer, 0, frameCount * FCC_2 * sizeof(short)); mixBufferState = ZEROED; } if (teeSink != NULL) { @@ -448,7 +487,8 @@ bool FastMixer::threadLoop() dumpState->mWriteSequence++; if (framesWritten >= 0) { ALOG_ASSERT((size_t) framesWritten <= frameCount); - dumpState->mFramesWritten += framesWritten; + totalNativeFramesWritten += framesWritten; + dumpState->mFramesWritten = totalNativeFramesWritten; //if ((size_t) framesWritten == frameCount) { // didFullWrite = true; //} @@ -457,6 +497,18 @@ bool FastMixer::threadLoop() } attemptedWrite = true; // FIXME count # of writes blocked excessively, CPU usage, etc. for dump + + timestampStatus = outputSink->getTimestamp(timestamp); + if (timestampStatus == NO_ERROR) { + uint32_t totalNativeFramesPresented = timestamp.mPosition; + if (totalNativeFramesPresented <= totalNativeFramesWritten) { + nativeFramesWrittenButNotPresented = + totalNativeFramesWritten - totalNativeFramesPresented; + } else { + // HAL reported that more frames were presented than were written + timestampStatus = INVALID_OPERATION; + } + } } // To be exactly periodic, compute the next sleep time based on current time. @@ -498,91 +550,91 @@ bool FastMixer::threadLoop() } } sleepNs = -1; - if (isWarm) { - if (sec > 0 || nsec > underrunNs) { - ATRACE_NAME("underrun"); - // FIXME only log occasionally - ALOGV("underrun: time since last cycle %d.%03ld sec", - (int) sec, nsec / 1000000L); - dumpState->mUnderruns++; - ignoreNextOverrun = true; - } else if (nsec < overrunNs) { - if (ignoreNextOverrun) { - ignoreNextOverrun = false; - } else { + if (isWarm) { + if (sec > 0 || nsec > underrunNs) { + ATRACE_NAME("underrun"); // FIXME only log occasionally - ALOGV("overrun: time since last cycle %d.%03ld sec", + ALOGV("underrun: time since last cycle %d.%03ld sec", (int) sec, nsec / 1000000L); - dumpState->mOverruns++; + dumpState->mUnderruns++; + ignoreNextOverrun = true; + } else if (nsec < overrunNs) { + if (ignoreNextOverrun) { + ignoreNextOverrun = false; + } else { + // FIXME only log occasionally + ALOGV("overrun: time since last cycle %d.%03ld sec", + (int) sec, nsec / 1000000L); + dumpState->mOverruns++; + } + // This forces a minimum cycle time. It: + // - compensates for an audio HAL with jitter due to sample rate conversion + // - works with a variable buffer depth audio HAL that never pulls at a + // rate < than overrunNs per buffer. + // - recovers from overrun immediately after underrun + // It doesn't work with a non-blocking audio HAL. + sleepNs = forceNs - nsec; + } else { + ignoreNextOverrun = false; } - // This forces a minimum cycle time. It: - // - compensates for an audio HAL with jitter due to sample rate conversion - // - works with a variable buffer depth audio HAL that never pulls at a rate - // < than overrunNs per buffer. - // - recovers from overrun immediately after underrun - // It doesn't work with a non-blocking audio HAL. - sleepNs = forceNs - nsec; - } else { - ignoreNextOverrun = false; } - } #ifdef FAST_MIXER_STATISTICS - if (isWarm) { - // advance the FIFO queue bounds - size_t i = bounds & (FastMixerDumpState::kSamplingN - 1); - bounds = (bounds & 0xFFFF0000) | ((bounds + 1) & 0xFFFF); - if (full) { - bounds += 0x10000; - } else if (!(bounds & (FastMixerDumpState::kSamplingN - 1))) { - full = true; - } - // compute the delta value of clock_gettime(CLOCK_MONOTONIC) - uint32_t monotonicNs = nsec; - if (sec > 0 && sec < 4) { - monotonicNs += sec * 1000000000; - } - // compute the raw CPU load = delta value of clock_gettime(CLOCK_THREAD_CPUTIME_ID) - uint32_t loadNs = 0; - struct timespec newLoad; - rc = clock_gettime(CLOCK_THREAD_CPUTIME_ID, &newLoad); - if (rc == 0) { - if (oldLoadValid) { - sec = newLoad.tv_sec - oldLoad.tv_sec; - nsec = newLoad.tv_nsec - oldLoad.tv_nsec; - if (nsec < 0) { - --sec; - nsec += 1000000000; - } - loadNs = nsec; - if (sec > 0 && sec < 4) { - loadNs += sec * 1000000000; + if (isWarm) { + // advance the FIFO queue bounds + size_t i = bounds & (dumpState->mSamplingN - 1); + bounds = (bounds & 0xFFFF0000) | ((bounds + 1) & 0xFFFF); + if (full) { + bounds += 0x10000; + } else if (!(bounds & (dumpState->mSamplingN - 1))) { + full = true; + } + // compute the delta value of clock_gettime(CLOCK_MONOTONIC) + uint32_t monotonicNs = nsec; + if (sec > 0 && sec < 4) { + monotonicNs += sec * 1000000000; + } + // compute raw CPU load = delta value of clock_gettime(CLOCK_THREAD_CPUTIME_ID) + uint32_t loadNs = 0; + struct timespec newLoad; + rc = clock_gettime(CLOCK_THREAD_CPUTIME_ID, &newLoad); + if (rc == 0) { + if (oldLoadValid) { + sec = newLoad.tv_sec - oldLoad.tv_sec; + nsec = newLoad.tv_nsec - oldLoad.tv_nsec; + if (nsec < 0) { + --sec; + nsec += 1000000000; + } + loadNs = nsec; + if (sec > 0 && sec < 4) { + loadNs += sec * 1000000000; + } + } else { + // first time through the loop + oldLoadValid = true; } - } else { - // first time through the loop - oldLoadValid = true; + oldLoad = newLoad; } - oldLoad = newLoad; - } #ifdef CPU_FREQUENCY_STATISTICS - // get the absolute value of CPU clock frequency in kHz - int cpuNum = sched_getcpu(); - uint32_t kHz = tcu.getCpukHz(cpuNum); - kHz = (kHz << 4) | (cpuNum & 0xF); + // get the absolute value of CPU clock frequency in kHz + int cpuNum = sched_getcpu(); + uint32_t kHz = tcu.getCpukHz(cpuNum); + kHz = (kHz << 4) | (cpuNum & 0xF); #endif - // save values in FIFO queues for dumpsys - // these stores #1, #2, #3 are not atomic with respect to each other, - // or with respect to store #4 below - dumpState->mMonotonicNs[i] = monotonicNs; - dumpState->mLoadNs[i] = loadNs; + // save values in FIFO queues for dumpsys + // these stores #1, #2, #3 are not atomic with respect to each other, + // or with respect to store #4 below + dumpState->mMonotonicNs[i] = monotonicNs; + dumpState->mLoadNs[i] = loadNs; #ifdef CPU_FREQUENCY_STATISTICS - dumpState->mCpukHz[i] = kHz; + dumpState->mCpukHz[i] = kHz; #endif - // this store #4 is not atomic with respect to stores #1, #2, #3 above, but - // the newest open and oldest closed halves are atomic with respect to each other - dumpState->mBounds = bounds; - ATRACE_INT("cycle_ms", monotonicNs / 1000000); - ATRACE_INT("load_us", loadNs / 1000); - } + // this store #4 is not atomic with respect to stores #1, #2, #3 above, but + // the newest open & oldest closed halves are atomic with respect to each other + dumpState->mBounds = bounds; + ATRACE_INT("cycle_ms", monotonicNs / 1000000); + ATRACE_INT("load_us", loadNs / 1000); + } #endif } else { // first time through the loop @@ -603,25 +655,43 @@ bool FastMixer::threadLoop() // never return 'true'; Thread::_threadLoop() locks mutex which can result in priority inversion } -FastMixerDumpState::FastMixerDumpState() : +FastMixerDumpState::FastMixerDumpState( +#ifdef FAST_MIXER_STATISTICS + uint32_t samplingN +#endif + ) : mCommand(FastMixerState::INITIAL), mWriteSequence(0), mFramesWritten(0), mNumTracks(0), mWriteErrors(0), mUnderruns(0), mOverruns(0), mSampleRate(0), mFrameCount(0), /* mMeasuredWarmupTs({0, 0}), */ mWarmupCycles(0), mTrackMask(0) #ifdef FAST_MIXER_STATISTICS - , mBounds(0) + , mSamplingN(0), mBounds(0) #endif { mMeasuredWarmupTs.tv_sec = 0; mMeasuredWarmupTs.tv_nsec = 0; +#ifdef FAST_MIXER_STATISTICS + increaseSamplingN(samplingN); +#endif +} + +#ifdef FAST_MIXER_STATISTICS +void FastMixerDumpState::increaseSamplingN(uint32_t samplingN) +{ + if (samplingN <= mSamplingN || samplingN > kSamplingN || roundup(samplingN) != samplingN) { + return; + } + uint32_t additional = samplingN - mSamplingN; // sample arrays aren't accessed atomically with respect to the bounds, // so clearing reduces chance for dumpsys to read random uninitialized samples - memset(&mMonotonicNs, 0, sizeof(mMonotonicNs)); - memset(&mLoadNs, 0, sizeof(mLoadNs)); + memset(&mMonotonicNs[mSamplingN], 0, sizeof(mMonotonicNs[0]) * additional); + memset(&mLoadNs[mSamplingN], 0, sizeof(mLoadNs[0]) * additional); #ifdef CPU_FREQUENCY_STATISTICS - memset(&mCpukHz, 0, sizeof(mCpukHz)); + memset(&mCpukHz[mSamplingN], 0, sizeof(mCpukHz[0]) * additional); #endif + mSamplingN = samplingN; } +#endif FastMixerDumpState::~FastMixerDumpState() { @@ -641,7 +711,7 @@ static int compare_uint32_t(const void *pa, const void *pb) } } -void FastMixerDumpState::dump(int fd) +void FastMixerDumpState::dump(int fd) const { if (mCommand == FastMixerState::INITIAL) { fdprintf(fd, "FastMixer not initialized\n"); @@ -692,9 +762,9 @@ void FastMixerDumpState::dump(int fd) uint32_t newestOpen = bounds & 0xFFFF; uint32_t oldestClosed = bounds >> 16; uint32_t n = (newestOpen - oldestClosed) & 0xFFFF; - if (n > kSamplingN) { + if (n > mSamplingN) { ALOGE("too many samples %u", n); - n = kSamplingN; + n = mSamplingN; } // statistics for monotonic (wall clock) time, thread raw CPU load in time, CPU clock frequency, // and adjusted CPU load in MHz normalized for CPU clock frequency @@ -710,7 +780,7 @@ void FastMixerDumpState::dump(int fd) uint32_t *tail = n >= kTailDenominator ? new uint32_t[n] : NULL; // loop over all the samples for (uint32_t j = 0; j < n; ++j) { - size_t i = oldestClosed++ & (kSamplingN - 1); + size_t i = oldestClosed++ & (mSamplingN - 1); uint32_t wallNs = mMonotonicNs[i]; if (tail != NULL) { tail[j] = wallNs; diff --git a/services/audioflinger/FastMixer.h b/services/audioflinger/FastMixer.h index 2ab1d04..6158925 100644 --- a/services/audioflinger/FastMixer.h +++ b/services/audioflinger/FastMixer.h @@ -85,10 +85,14 @@ struct FastTrackDump { // Only POD types are permitted, and the contents shouldn't be trusted (i.e. do range checks). // It has a different lifetime than the FastMixer, and so it can't be a member of FastMixer. struct FastMixerDumpState { - FastMixerDumpState(); + FastMixerDumpState( +#ifdef FAST_MIXER_STATISTICS + uint32_t samplingN = kSamplingNforLowRamDevice +#endif + ); /*virtual*/ ~FastMixerDumpState(); - void dump(int fd); // should only be called on a stable copy, not the original + void dump(int fd) const; // should only be called on a stable copy, not the original FastMixerState::Command mCommand; // current command uint32_t mWriteSequence; // incremented before and after each write() @@ -106,8 +110,15 @@ struct FastMixerDumpState { #ifdef FAST_MIXER_STATISTICS // Recently collected samples of per-cycle monotonic time, thread CPU time, and CPU frequency. - // kSamplingN is the size of the sampling frame, and must be a power of 2 <= 0x8000. + // kSamplingN is max size of sampling frame (statistics), and must be a power of 2 <= 0x8000. + // The sample arrays are virtually allocated based on this compile-time constant, + // but are only initialized and used based on the runtime parameter mSamplingN. static const uint32_t kSamplingN = 0x8000; + // Compile-time constant for a "low RAM device", must be a power of 2 <= kSamplingN. + // This value was chosen such that each array uses 1 small page (4 Kbytes). + static const uint32_t kSamplingNforLowRamDevice = 0x400; + // Corresponding runtime maximum size of sample arrays, must be a power of 2 <= kSamplingN. + uint32_t mSamplingN; // The bounds define the interval of valid samples, and are represented as follows: // newest open (excluded) endpoint = lower 16 bits of bounds, modulo N // oldest closed (included) endpoint = upper 16 bits of bounds, modulo N @@ -119,6 +130,8 @@ struct FastMixerDumpState { #ifdef CPU_FREQUENCY_STATISTICS uint32_t mCpukHz[kSamplingN]; // absolute CPU clock frequency in kHz, bits 0-3 are CPU# #endif + // Increase sampling window after construction, must be a power of 2 <= kSamplingN + void increaseSamplingN(uint32_t samplingN); #endif }; diff --git a/services/audioflinger/FastMixerState.cpp b/services/audioflinger/FastMixerState.cpp index c45c81b..737de97 100644 --- a/services/audioflinger/FastMixerState.cpp +++ b/services/audioflinger/FastMixerState.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "Configuration.h" #include "FastMixerState.h" namespace android { diff --git a/services/audioflinger/PlaybackTracks.h b/services/audioflinger/PlaybackTracks.h index a749d7a..a2e2511 100644 --- a/services/audioflinger/PlaybackTracks.h +++ b/services/audioflinger/PlaybackTracks.h @@ -46,15 +46,21 @@ public: void destroy(); int name() const { return mName; } + virtual uint32_t sampleRate() const; + audio_stream_type_t streamType() const { return mStreamType; } + bool isOffloaded() const { return (mFlags & IAudioFlinger::TRACK_OFFLOAD) != 0; } + status_t setParameters(const String8& keyValuePairs); status_t attachAuxEffect(int EffectId); void setAuxBuffer(int EffectId, int32_t *buffer); int32_t *auxBuffer() const { return mAuxBuffer; } void setMainBuffer(int16_t *buffer) { mMainBuffer = buffer; } int16_t *mainBuffer() const { return mMainBuffer; } int auxEffectId() const { return mAuxEffectId; } + virtual status_t getTimestamp(AudioTimestamp& timestamp); + void signal(); // implement FastMixerState::VolumeProvider interface virtual uint32_t getVolumeLR(); @@ -66,6 +72,7 @@ protected: friend class PlaybackThread; friend class MixerThread; friend class DirectOutputThread; + friend class OffloadThread; Track(const Track&); Track& operator = (const Track&); @@ -75,7 +82,9 @@ protected: int64_t pts = kInvalidPTS); // releaseBuffer() not overridden + // ExtendedAudioBufferProvider interface virtual size_t framesReady() const; + virtual size_t framesReleased() const; bool isPausing() const { return mState == PAUSING; } bool isPaused() const { return mState == PAUSED; } @@ -101,6 +110,7 @@ public: bool isInvalid() const { return mIsInvalid; } virtual bool isTimedTrack() const { return false; } bool isFastTrack() const { return (mFlags & IAudioFlinger::TRACK_FAST) != 0; } + int fastIndex() const { return mFastIndex; } protected: @@ -108,7 +118,10 @@ protected: enum {FS_INVALID, FS_FILLING, FS_FILLED, FS_ACTIVE}; mutable uint8_t mFillingUpStatus; int8_t mRetryCount; - const sp<IMemory> mSharedBuffer; + + // see comment at AudioFlinger::PlaybackThread::Track::~Track for why this can't be const + sp<IMemory> mSharedBuffer; + bool mResetDone; const audio_stream_type_t mStreamType; int mName; // track name on the normal mixer, @@ -134,11 +147,12 @@ private: // but the slot is only used if track is active FastTrackUnderruns mObservedUnderruns; // Most recently observed value of // mFastMixerDumpState.mTracks[mFastIndex].mUnderruns - uint32_t mUnderrunCount; // Counter of total number of underruns, never reset volatile float mCachedVolume; // combined master volume and stream type volume; // 'volatile' means accessed without lock or // barrier, but is read/written atomically bool mIsInvalid; // non-resettable latch, set by invalidate() + AudioTrackServerProxy* mAudioTrackServerProxy; + bool mResumeToStopping; // track was paused in stopping state. }; // end of Track class TimedTrack : public Track { @@ -255,10 +269,6 @@ public: private: - enum { - NO_MORE_BUFFERS = 0x80000001, // same in AudioTrack.h, ok to be different value - }; - status_t obtainBuffer(AudioBufferProvider::Buffer* buffer, uint32_t waitTimeMs); void clearBufferQueue(); diff --git a/services/audioflinger/RecordTracks.h b/services/audioflinger/RecordTracks.h index 6c0d1d3..cd8f70c 100644 --- a/services/audioflinger/RecordTracks.h +++ b/services/audioflinger/RecordTracks.h @@ -36,6 +36,7 @@ public: void destroy(); + void invalidate(); // clear the buffer overflow flag void clearOverflow() { mOverflow = false; } // set the buffer overflow flag and return previous value @@ -57,4 +58,5 @@ private: // releaseBuffer() not overridden bool mOverflow; // overflow on most recent attempt to fill client buffer + AudioRecordServerProxy* mAudioRecordServerProxy; }; diff --git a/services/audioflinger/ServiceUtilities.cpp b/services/audioflinger/ServiceUtilities.cpp index d15bd04..152455d 100644 --- a/services/audioflinger/ServiceUtilities.cpp +++ b/services/audioflinger/ServiceUtilities.cpp @@ -34,6 +34,22 @@ bool recordingAllowed() { return ok; } +bool captureAudioOutputAllowed() { + if (getpid_cached == IPCThreadState::self()->getCallingPid()) return true; + static const String16 sCaptureAudioOutput("android.permission.CAPTURE_AUDIO_OUTPUT"); + // don't use PermissionCache; this is not a system permission + bool ok = checkCallingPermission(sCaptureAudioOutput); + if (!ok) ALOGE("Request requires android.permission.CAPTURE_AUDIO_OUTPUT"); + return ok; +} + +bool captureHotwordAllowed() { + static const String16 sCaptureHotwordAllowed("android.permission.CAPTURE_AUDIO_HOTWORD"); + bool ok = checkCallingPermission(sCaptureHotwordAllowed); + if (!ok) ALOGE("android.permission.CAPTURE_AUDIO_HOTWORD"); + return ok; +} + bool settingsAllowed() { if (getpid_cached == IPCThreadState::self()->getCallingPid()) return true; static const String16 sAudioSettings("android.permission.MODIFY_AUDIO_SETTINGS"); diff --git a/services/audioflinger/ServiceUtilities.h b/services/audioflinger/ServiceUtilities.h index 80cecba..531bc56 100644 --- a/services/audioflinger/ServiceUtilities.h +++ b/services/audioflinger/ServiceUtilities.h @@ -21,6 +21,8 @@ namespace android { extern pid_t getpid_cached; bool recordingAllowed(); +bool captureAudioOutputAllowed(); +bool captureHotwordAllowed(); bool settingsAllowed(); bool dumpAllowed(); diff --git a/services/audioflinger/StateQueue.cpp b/services/audioflinger/StateQueue.cpp index 3e891a5..c2d3bbd 100644 --- a/services/audioflinger/StateQueue.cpp +++ b/services/audioflinger/StateQueue.cpp @@ -17,6 +17,7 @@ #define LOG_TAG "StateQueue" //#define LOG_NDEBUG 0 +#include "Configuration.h" #include <time.h> #include <cutils/atomic.h> #include <utils/Log.h> diff --git a/services/audioflinger/StateQueue.h b/services/audioflinger/StateQueue.h index e33b3c6..9cde642 100644 --- a/services/audioflinger/StateQueue.h +++ b/services/audioflinger/StateQueue.h @@ -31,8 +31,14 @@ // and this may result in an audible artifact // needs read-only access to a recent stable state, // but not necessarily the most current one +// only allocate and free memory when configuration changes +// avoid conventional logging, as this is a form of I/O and could block +// defer computation to other threads when feasible; for example +// cycle times are collected by fast mixer thread but the floating-point +// statistical calculations on these cycle times are computed by normal mixer +// these requirements also apply to callouts such as AudioBufferProvider and VolumeProvider // Normal mixer thread: -// periodic with typical period ~40 ms +// periodic with typical period ~20 ms // SCHED_OTHER scheduling policy and nice priority == urgent audio // ok to block, but prefer to avoid as much as possible // needs read/write access to state diff --git a/services/audioflinger/StateQueueInstantiations.cpp b/services/audioflinger/StateQueueInstantiations.cpp index 077582f..0d5cd0c 100644 --- a/services/audioflinger/StateQueueInstantiations.cpp +++ b/services/audioflinger/StateQueueInstantiations.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "Configuration.h" #include "FastMixerState.h" #include "StateQueue.h" diff --git a/services/audioflinger/Threads.cpp b/services/audioflinger/Threads.cpp index 0b88c0e..47dcca6 100644 --- a/services/audioflinger/Threads.cpp +++ b/services/audioflinger/Threads.cpp @@ -20,11 +20,12 @@ //#define LOG_NDEBUG 0 #define ATRACE_TAG ATRACE_TAG_AUDIO +#include "Configuration.h" #include <math.h> #include <fcntl.h> #include <sys/stat.h> #include <cutils/properties.h> -#include <cutils/compiler.h> +#include <media/AudioParameter.h> #include <utils/Log.h> #include <utils/Trace.h> @@ -53,14 +54,11 @@ #include "ServiceUtilities.h" #include "SchedulingPolicyService.h" -#undef ADD_BATTERY_DATA - #ifdef ADD_BATTERY_DATA #include <media/IMediaPlayerService.h> #include <media/IMediaDeathNotifier.h> #endif -// #define DEBUG_CPU_USAGE 10 // log statistics every n wall clock seconds #ifdef DEBUG_CPU_USAGE #include <cpustats/CentralTendencyStatistics.h> #include <cpustats/ThreadCpuUsage.h> @@ -111,6 +109,9 @@ static const uint32_t kMinNormalMixBufferSizeMs = 20; // maximum normal mix buffer size static const uint32_t kMaxNormalMixBufferSizeMs = 24; +// Offloaded output thread standby delay: allows track transition without going to standby +static const nsecs_t kOffloadStandbyDelayNs = seconds(1); + // Whether to use fast mixer static const enum { FastMixer_Never, // never initialize or use: for debugging only @@ -139,7 +140,7 @@ static const int kPriorityFastMixer = 3; // FIXME It would be better for client to tell AudioFlinger whether it wants double-buffering or // N-buffering, so AudioFlinger could allocate the right amount of memory. // See the client's minBufCount and mNotificationFramesAct calculations for details. -static const int kFastTrackMultiplier = 2; +static const int kFastTrackMultiplier = 1; // ---------------------------------------------------------------------------- @@ -267,10 +268,9 @@ AudioFlinger::ThreadBase::ThreadBase(const sp<AudioFlinger>& audioFlinger, audio audio_devices_t outDevice, audio_devices_t inDevice, type_t type) : Thread(false /*canCallJava*/), mType(type), - mAudioFlinger(audioFlinger), mSampleRate(0), mFrameCount(0), mNormalFrameCount(0), - // mChannelMask - mChannelCount(0), - mFrameSize(1), mFormat(AUDIO_FORMAT_INVALID), + mAudioFlinger(audioFlinger), + // mSampleRate, mFrameCount, mChannelMask, mChannelCount, mFrameSize, and mFormat are + // set by PlaybackThread::readOutputParameters() or RecordThread::readInputParameters() mParamStatus(NO_ERROR), mStandby(false), mOutDevice(outDevice), mInDevice(inDevice), mAudioSource(AUDIO_SOURCE_DEFAULT), mId(id), @@ -281,6 +281,12 @@ AudioFlinger::ThreadBase::ThreadBase(const sp<AudioFlinger>& audioFlinger, audio AudioFlinger::ThreadBase::~ThreadBase() { + // mConfigEvents should be empty, but just in case it isn't, free the memory it owns + for (size_t i = 0; i < mConfigEvents.size(); i++) { + delete mConfigEvents[i]; + } + mConfigEvents.clear(); + mParamCond.broadcast(); // do not lock the mutex in destructor releaseWakeLock_l(); @@ -420,9 +426,7 @@ void AudioFlinger::ThreadBase::dumpBase(int fd, const Vector<String16>& args) result.append(buffer); snprintf(buffer, SIZE, "HAL frame count: %d\n", mFrameCount); result.append(buffer); - snprintf(buffer, SIZE, "Normal frame count: %d\n", mNormalFrameCount); - result.append(buffer); - snprintf(buffer, SIZE, "Channel Count: %d\n", mChannelCount); + snprintf(buffer, SIZE, "Channel Count: %u\n", mChannelCount); result.append(buffer); snprintf(buffer, SIZE, "Channel Mask: 0x%08x\n", mChannelMask); result.append(buffer); @@ -472,13 +476,32 @@ void AudioFlinger::ThreadBase::dumpEffectChains(int fd, const Vector<String16>& } } -void AudioFlinger::ThreadBase::acquireWakeLock() +void AudioFlinger::ThreadBase::acquireWakeLock(int uid) { Mutex::Autolock _l(mLock); - acquireWakeLock_l(); + acquireWakeLock_l(uid); } -void AudioFlinger::ThreadBase::acquireWakeLock_l() +String16 AudioFlinger::ThreadBase::getWakeLockTag() +{ + switch (mType) { + case MIXER: + return String16("AudioMix"); + case DIRECT: + return String16("AudioDirectOut"); + case DUPLICATING: + return String16("AudioDup"); + case RECORD: + return String16("AudioIn"); + case OFFLOAD: + return String16("AudioOffload"); + default: + ALOG_ASSERT(false); + return String16("AudioUnknown"); + } +} + +void AudioFlinger::ThreadBase::acquireWakeLock_l(int uid) { if (mPowerManager == 0) { // use checkService() to avoid blocking if power service is not up yet @@ -493,9 +516,19 @@ void AudioFlinger::ThreadBase::acquireWakeLock_l() } if (mPowerManager != 0) { sp<IBinder> binder = new BBinder(); - status_t status = mPowerManager->acquireWakeLock(POWERMANAGER_PARTIAL_WAKE_LOCK, - binder, - String16(mName)); + status_t status; + if (uid >= 0) { + status = mPowerManager->acquireWakeLockWithUid(POWERMANAGER_PARTIAL_WAKE_LOCK, + binder, + getWakeLockTag(), + String16("media"), + uid); + } else { + status = mPowerManager->acquireWakeLock(POWERMANAGER_PARTIAL_WAKE_LOCK, + binder, + getWakeLockTag(), + String16("media")); + } if (status == NO_ERROR) { mWakeLockToken = binder; } @@ -697,14 +730,22 @@ sp<AudioFlinger::EffectHandle> AudioFlinger::ThreadBase::createEffect_l( goto Exit; } - // Do not allow effects with session ID 0 on direct output or duplicating threads - // TODO: add rule for hw accelerated effects on direct outputs with non PCM format - if (sessionId == AUDIO_SESSION_OUTPUT_MIX && mType != MIXER) { - ALOGW("createEffect_l() Cannot add auxiliary effect %s to session %d", - desc->name, sessionId); - lStatus = BAD_VALUE; - goto Exit; + // Allow global effects only on offloaded and mixer threads + if (sessionId == AUDIO_SESSION_OUTPUT_MIX) { + switch (mType) { + case MIXER: + case OFFLOAD: + break; + case DIRECT: + case DUPLICATING: + case RECORD: + default: + ALOGW("createEffect_l() Cannot add global effect %s on thread %s", desc->name, mName); + lStatus = BAD_VALUE; + goto Exit; + } } + // Only Pre processor effects are allowed on input threads and only on input threads if ((mType == RECORD) != ((desc->flags & EFFECT_FLAG_TYPE_MASK) == EFFECT_FLAG_TYPE_PRE_PROC)) { ALOGW("createEffect_l() effect %s (flags %08x) created on wrong thread type %d", @@ -747,6 +788,8 @@ sp<AudioFlinger::EffectHandle> AudioFlinger::ThreadBase::createEffect_l( if (lStatus != NO_ERROR) { goto Exit; } + effect->setOffloaded(mType == OFFLOAD, mId); + lStatus = chain->addEffect_l(effect); if (lStatus != NO_ERROR) { goto Exit; @@ -808,6 +851,10 @@ status_t AudioFlinger::ThreadBase::addEffect_l(const sp<EffectModule>& effect) sp<EffectChain> chain = getEffectChain_l(sessionId); bool chainCreated = false; + ALOGD_IF((mType == OFFLOAD) && !effect->isOffloadable(), + "addEffect_l() on offloaded thread %p: effect %s does not support offload flags %x", + this, effect->desc().name, effect->desc().flags); + if (chain == 0) { // create a new chain for this session ALOGV("addEffect_l() new effect chain for session %d", sessionId); @@ -824,6 +871,8 @@ status_t AudioFlinger::ThreadBase::addEffect_l(const sp<EffectModule>& effect) return BAD_VALUE; } + effect->setOffloaded(mType == OFFLOAD, mId); + status_t status = chain->addEffect_l(effect); if (status != NO_ERROR) { if (chainCreated) { @@ -926,16 +975,25 @@ AudioFlinger::PlaybackThread::PlaybackThread(const sp<AudioFlinger>& audioFlinge audio_devices_t device, type_t type) : ThreadBase(audioFlinger, id, device, AUDIO_DEVICE_NONE, type), - mMixBuffer(NULL), mSuspended(0), mBytesWritten(0), + mNormalFrameCount(0), mMixBuffer(NULL), + mAllocMixBuffer(NULL), mSuspended(0), mBytesWritten(0), // mStreamTypes[] initialized in constructor body mOutput(output), mLastWriteTime(0), mNumWrites(0), mNumDelayedWrites(0), mInWrite(false), mMixerStatus(MIXER_IDLE), mMixerStatusIgnoringFastTracks(MIXER_IDLE), standbyDelay(AudioFlinger::mStandbyTimeInNsecs), + mBytesRemaining(0), + mCurrentWriteLength(0), + mUseAsyncWrite(false), + mWriteAckSequence(0), + mDrainSequence(0), + mSignalPending(false), mScreenState(AudioFlinger::mScreenState), // index 0 is reserved for normal mixer's submix - mFastTrackAvailMask(((1 << FastMixerState::kMaxFastTracks) - 1) & ~1) + mFastTrackAvailMask(((1 << FastMixerState::kMaxFastTracks) - 1) & ~1), + // mLatchD, mLatchQ, + mLatchDValid(false), mLatchQValid(false) { snprintf(mName, kNameLength, "AudioOut_%X", id); mNBLogWriter = audioFlinger->newWriter_l(kLogSize, mName); @@ -975,7 +1033,7 @@ AudioFlinger::PlaybackThread::PlaybackThread(const sp<AudioFlinger>& audioFlinge AudioFlinger::PlaybackThread::~PlaybackThread() { mAudioFlinger->unregisterWriter(mNBLogWriter); - delete [] mMixBuffer; + delete [] mAllocMixBuffer; } void AudioFlinger::PlaybackThread::dump(int fd, const Vector<String16>& args) @@ -1043,6 +1101,8 @@ void AudioFlinger::PlaybackThread::dumpInternals(int fd, const Vector<String16>& snprintf(buffer, SIZE, "\nOutput thread %p internals\n", this); result.append(buffer); + snprintf(buffer, SIZE, "Normal frame count: %d\n", mNormalFrameCount); + result.append(buffer); snprintf(buffer, SIZE, "last write occurred (msecs): %llu\n", ns2ms(systemTime() - mLastWriteTime)); result.append(buffer); @@ -1181,7 +1241,22 @@ sp<AudioFlinger::PlaybackThread::Track> AudioFlinger::PlaybackThread::createTrac goto Exit; } } + } else if (mType == OFFLOAD) { + if (sampleRate != mSampleRate || format != mFormat || channelMask != mChannelMask) { + ALOGE("createTrack_l() Bad parameter: sampleRate %d format %d, channelMask 0x%08x \"" + "for output %p with format %d", + sampleRate, format, channelMask, mOutput, mFormat); + lStatus = BAD_VALUE; + goto Exit; + } } else { + if ((format & AUDIO_FORMAT_MAIN_MASK) != AUDIO_FORMAT_PCM) { + ALOGE("createTrack_l() Bad parameter: format %d \"" + "for output %p with format %d", + format, mOutput, mFormat); + lStatus = BAD_VALUE; + goto Exit; + } // Resampler implementation limits input sampling rate to 2 x output sampling rate. if (sampleRate > mSampleRate*2) { ALOGE("Sample rate out of range: %u mSampleRate %u", sampleRate, mSampleRate); @@ -1227,6 +1302,7 @@ sp<AudioFlinger::PlaybackThread::Track> AudioFlinger::PlaybackThread::createTrac lStatus = NO_MEMORY; goto Exit; } + mTracks.add(track); sp<EffectChain> chain = getEffectChain_l(sessionId); @@ -1301,12 +1377,14 @@ void AudioFlinger::PlaybackThread::setStreamVolume(audio_stream_type_t stream, f { Mutex::Autolock _l(mLock); mStreamTypes[stream].volume = value; + broadcast_l(); } void AudioFlinger::PlaybackThread::setStreamMute(audio_stream_type_t stream, bool muted) { Mutex::Autolock _l(mLock); mStreamTypes[stream].mute = muted; + broadcast_l(); } float AudioFlinger::PlaybackThread::streamVolume(audio_stream_type_t stream) const @@ -1326,7 +1404,31 @@ status_t AudioFlinger::PlaybackThread::addTrack_l(const sp<Track>& track) // the track is newly added, make sure it fills up all its // buffers before playing. This is to ensure the client will // effectively get the latency it requested. - track->mFillingUpStatus = Track::FS_FILLING; + if (!track->isOutputTrack()) { + TrackBase::track_state state = track->mState; + mLock.unlock(); + status = AudioSystem::startOutput(mId, track->streamType(), track->sessionId()); + mLock.lock(); + // abort track was stopped/paused while we released the lock + if (state != track->mState) { + if (status == NO_ERROR) { + mLock.unlock(); + AudioSystem::stopOutput(mId, track->streamType(), track->sessionId()); + mLock.lock(); + } + return INVALID_OPERATION; + } + // abort if start is rejected by audio policy manager + if (status != NO_ERROR) { + return PERMISSION_DENIED; + } +#ifdef ADD_BATTERY_DATA + // to track the speaker usage + addBatteryData(IMediaPlayerService::kBatteryDataAudioFlingerStart); +#endif + } + + track->mFillingUpStatus = track->sharedBuffer() != 0 ? Track::FS_FILLED : Track::FS_FILLING; track->mResetDone = false; track->mPresentationCompleteFrames = 0; mActiveTracks.add(track); @@ -1340,20 +1442,25 @@ status_t AudioFlinger::PlaybackThread::addTrack_l(const sp<Track>& track) status = NO_ERROR; } - ALOGV("mWaitWorkCV.broadcast"); - mWaitWorkCV.broadcast(); + ALOGV("signal playback thread"); + broadcast_l(); return status; } -// destroyTrack_l() must be called with ThreadBase::mLock held -void AudioFlinger::PlaybackThread::destroyTrack_l(const sp<Track>& track) +bool AudioFlinger::PlaybackThread::destroyTrack_l(const sp<Track>& track) { - track->mState = TrackBase::TERMINATED; + track->terminate(); // active tracks are removed by threadLoop() - if (mActiveTracks.indexOf(track) < 0) { + bool trackActive = (mActiveTracks.indexOf(track) >= 0); + track->mState = TrackBase::STOPPED; + if (!trackActive) { removeTrack_l(track); + } else if (track->isFastTrack() || track->isOffloaded()) { + track->mState = TrackBase::STOPPING_1; } + + return trackActive; } void AudioFlinger::PlaybackThread::removeTrack_l(const sp<Track>& track) @@ -1377,18 +1484,25 @@ void AudioFlinger::PlaybackThread::removeTrack_l(const sp<Track>& track) } } -String8 AudioFlinger::PlaybackThread::getParameters(const String8& keys) +void AudioFlinger::PlaybackThread::broadcast_l() { - String8 out_s8 = String8(""); - char *s; + // Thread could be blocked waiting for async + // so signal it to handle state changes immediately + // If threadLoop is currently unlocked a signal of mWaitWorkCV will + // be lost so we also flag to prevent it blocking on mWaitWorkCV + mSignalPending = true; + mWaitWorkCV.broadcast(); +} +String8 AudioFlinger::PlaybackThread::getParameters(const String8& keys) +{ Mutex::Autolock _l(mLock); if (initCheck() != NO_ERROR) { - return out_s8; + return String8(); } - s = mOutput->stream->common.get_parameters(&mOutput->stream->common, keys.string()); - out_s8 = String8(s); + char *s = mOutput->stream->common.get_parameters(&mOutput->stream->common, keys.string()); + const String8 out_s8(s); free(s); return out_s8; } @@ -1404,7 +1518,7 @@ void AudioFlinger::PlaybackThread::audioConfigChanged_l(int event, int param) { switch (event) { case AudioSystem::OUTPUT_OPENED: case AudioSystem::OUTPUT_CONFIG_CHANGED: - desc.channels = mChannelMask; + desc.channelMask = mChannelMask; desc.samplingRate = mSampleRate; desc.format = mFormat; desc.frameCount = mNormalFrameCount; // FIXME see @@ -1422,12 +1536,80 @@ void AudioFlinger::PlaybackThread::audioConfigChanged_l(int event, int param) { mAudioFlinger->audioConfigChanged_l(event, mId, param2); } +void AudioFlinger::PlaybackThread::writeCallback() +{ + ALOG_ASSERT(mCallbackThread != 0); + mCallbackThread->resetWriteBlocked(); +} + +void AudioFlinger::PlaybackThread::drainCallback() +{ + ALOG_ASSERT(mCallbackThread != 0); + mCallbackThread->resetDraining(); +} + +void AudioFlinger::PlaybackThread::resetWriteBlocked(uint32_t sequence) +{ + Mutex::Autolock _l(mLock); + // reject out of sequence requests + if ((mWriteAckSequence & 1) && (sequence == mWriteAckSequence)) { + mWriteAckSequence &= ~1; + mWaitWorkCV.signal(); + } +} + +void AudioFlinger::PlaybackThread::resetDraining(uint32_t sequence) +{ + Mutex::Autolock _l(mLock); + // reject out of sequence requests + if ((mDrainSequence & 1) && (sequence == mDrainSequence)) { + mDrainSequence &= ~1; + mWaitWorkCV.signal(); + } +} + +// static +int AudioFlinger::PlaybackThread::asyncCallback(stream_callback_event_t event, + void *param, + void *cookie) +{ + AudioFlinger::PlaybackThread *me = (AudioFlinger::PlaybackThread *)cookie; + ALOGV("asyncCallback() event %d", event); + switch (event) { + case STREAM_CBK_EVENT_WRITE_READY: + me->writeCallback(); + break; + case STREAM_CBK_EVENT_DRAIN_READY: + me->drainCallback(); + break; + default: + ALOGW("asyncCallback() unknown event %d", event); + break; + } + return 0; +} + void AudioFlinger::PlaybackThread::readOutputParameters() { + // unfortunately we have no way of recovering from errors here, hence the LOG_FATAL mSampleRate = mOutput->stream->common.get_sample_rate(&mOutput->stream->common); mChannelMask = mOutput->stream->common.get_channels(&mOutput->stream->common); - mChannelCount = (uint16_t)popcount(mChannelMask); + if (!audio_is_output_channel(mChannelMask)) { + LOG_FATAL("HAL channel mask %#x not valid for output", mChannelMask); + } + if ((mType == MIXER || mType == DUPLICATING) && mChannelMask != AUDIO_CHANNEL_OUT_STEREO) { + LOG_FATAL("HAL channel mask %#x not supported for mixed output; " + "must be AUDIO_CHANNEL_OUT_STEREO", mChannelMask); + } + mChannelCount = popcount(mChannelMask); mFormat = mOutput->stream->common.get_format(&mOutput->stream->common); + if (!audio_is_valid_format(mFormat)) { + LOG_FATAL("HAL format %d not valid for output", mFormat); + } + if ((mType == MIXER || mType == DUPLICATING) && mFormat != AUDIO_FORMAT_PCM_16_BIT) { + LOG_FATAL("HAL format %d not supported for mixed output; must be AUDIO_FORMAT_PCM_16_BIT", + mFormat); + } mFrameSize = audio_stream_frame_size(&mOutput->stream->common); mFrameCount = mOutput->stream->common.get_buffer_size(&mOutput->stream->common) / mFrameSize; if (mFrameCount & 15) { @@ -1435,6 +1617,15 @@ void AudioFlinger::PlaybackThread::readOutputParameters() mFrameCount); } + if ((mOutput->flags & AUDIO_OUTPUT_FLAG_NON_BLOCKING) && + (mOutput->stream->set_callback != NULL)) { + if (mOutput->stream->set_callback(mOutput->stream, + AudioFlinger::PlaybackThread::asyncCallback, this) == 0) { + mUseAsyncWrite = true; + mCallbackThread = new AudioFlinger::AsyncCallbackThread(this); + } + } + // Calculate size of normal mix buffer relative to the HAL output buffer size double multiplier = 1.0; if (mType == MIXER && (kUseFastMixer == FastMixer_Static || @@ -1477,9 +1668,11 @@ void AudioFlinger::PlaybackThread::readOutputParameters() ALOGI("HAL output buffer size %u frames, normal mix buffer size %u frames", mFrameCount, mNormalFrameCount); - delete[] mMixBuffer; - mMixBuffer = new int16_t[mNormalFrameCount * mChannelCount]; - memset(mMixBuffer, 0, mNormalFrameCount * mChannelCount * sizeof(int16_t)); + delete[] mAllocMixBuffer; + size_t align = (mFrameSize < sizeof(int16_t)) ? sizeof(int16_t) : mFrameSize; + mAllocMixBuffer = new int8_t[mNormalFrameCount * mFrameSize + align - 1]; + mMixBuffer = (int16_t *) ((((size_t)mAllocMixBuffer + align - 1) / align) * align); + memset(mMixBuffer, 0, mNormalFrameCount * mFrameSize); // force reconfiguration of effect chains and engines to take new buffer size and audio // parameters into account @@ -1613,16 +1806,21 @@ void AudioFlinger::PlaybackThread::threadLoop_removeTracks( const Vector< sp<Track> >& tracksToRemove) { size_t count = tracksToRemove.size(); - if (CC_UNLIKELY(count)) { + if (count) { for (size_t i = 0 ; i < count ; i++) { const sp<Track>& track = tracksToRemove.itemAt(i); - if ((track->sharedBuffer() != 0) && - (track->mState == TrackBase::ACTIVE || track->mState == TrackBase::RESUMING)) { + if (!track->isOutputTrack()) { AudioSystem::stopOutput(mId, track->streamType(), track->sessionId()); +#ifdef ADD_BATTERY_DATA + // to track the speaker usage + addBatteryData(IMediaPlayerService::kBatteryDataAudioFlingerStop); +#endif + if (track->isTerminated()) { + AudioSystem::releaseOutput(mId); + } } } } - } void AudioFlinger::PlaybackThread::checkSilentMode_l() @@ -1643,17 +1841,18 @@ void AudioFlinger::PlaybackThread::checkSilentMode_l() } // shared by MIXER and DIRECT, overridden by DUPLICATING -void AudioFlinger::PlaybackThread::threadLoop_write() +ssize_t AudioFlinger::PlaybackThread::threadLoop_write() { // FIXME rewrite to reduce number of system calls mLastWriteTime = systemTime(); mInWrite = true; - int bytesWritten; + ssize_t bytesWritten; // If an NBAIO sink is present, use it to write the normal mixer's submix if (mNormalSink != 0) { #define mBitShift 2 // FIXME - size_t count = mixBufferSize >> mBitShift; + size_t count = mBytesRemaining >> mBitShift; + size_t offset = (mCurrentWriteLength - mBytesRemaining) >> 1; ATRACE_BEGIN("write"); // update the setpoint when AudioFlinger::mScreenState changes uint32_t screenState = AudioFlinger::mScreenState; @@ -1665,24 +1864,70 @@ void AudioFlinger::PlaybackThread::threadLoop_write() (pipe->maxFrames() * 7) / 8 : mNormalFrameCount * 2); } } - ssize_t framesWritten = mNormalSink->write(mMixBuffer, count); + ssize_t framesWritten = mNormalSink->write(mMixBuffer + offset, count); ATRACE_END(); if (framesWritten > 0) { bytesWritten = framesWritten << mBitShift; } else { bytesWritten = framesWritten; } + status_t status = mNormalSink->getTimestamp(mLatchD.mTimestamp); + if (status == NO_ERROR) { + size_t totalFramesWritten = mNormalSink->framesWritten(); + if (totalFramesWritten >= mLatchD.mTimestamp.mPosition) { + mLatchD.mUnpresentedFrames = totalFramesWritten - mLatchD.mTimestamp.mPosition; + mLatchDValid = true; + } + } // otherwise use the HAL / AudioStreamOut directly } else { - // Direct output thread. - bytesWritten = (int)mOutput->stream->write(mOutput->stream, mMixBuffer, mixBufferSize); + // Direct output and offload threads + size_t offset = (mCurrentWriteLength - mBytesRemaining) / sizeof(int16_t); + if (mUseAsyncWrite) { + ALOGW_IF(mWriteAckSequence & 1, "threadLoop_write(): out of sequence write request"); + mWriteAckSequence += 2; + mWriteAckSequence |= 1; + ALOG_ASSERT(mCallbackThread != 0); + mCallbackThread->setWriteBlocked(mWriteAckSequence); + } + // FIXME We should have an implementation of timestamps for direct output threads. + // They are used e.g for multichannel PCM playback over HDMI. + bytesWritten = mOutput->stream->write(mOutput->stream, + mMixBuffer + offset, mBytesRemaining); + if (mUseAsyncWrite && + ((bytesWritten < 0) || (bytesWritten == (ssize_t)mBytesRemaining))) { + // do not wait for async callback in case of error of full write + mWriteAckSequence &= ~1; + ALOG_ASSERT(mCallbackThread != 0); + mCallbackThread->setWriteBlocked(mWriteAckSequence); + } } - if (bytesWritten > 0) { - mBytesWritten += mixBufferSize; - } mNumWrites++; mInWrite = false; + + return bytesWritten; +} + +void AudioFlinger::PlaybackThread::threadLoop_drain() +{ + if (mOutput->stream->drain) { + ALOGV("draining %s", (mMixerStatus == MIXER_DRAIN_TRACK) ? "early" : "full"); + if (mUseAsyncWrite) { + ALOGW_IF(mDrainSequence & 1, "threadLoop_drain(): out of sequence drain request"); + mDrainSequence |= 1; + ALOG_ASSERT(mCallbackThread != 0); + mCallbackThread->setDraining(mDrainSequence); + } + mOutput->stream->drain(mOutput->stream, + (mMixerStatus == MIXER_DRAIN_TRACK) ? AUDIO_DRAIN_EARLY_NOTIFY + : AUDIO_DRAIN_ALL); + } +} + +void AudioFlinger::PlaybackThread::threadLoop_exit() +{ + // Default implementation has nothing to do } /* @@ -1713,7 +1958,7 @@ void AudioFlinger::PlaybackThread::cacheParameters_l() void AudioFlinger::PlaybackThread::invalidateTracks(audio_stream_type_t streamType) { - ALOGV ("MixerThread::invalidateTracks() mixer %p, streamType %d, mTracks.size %d", + ALOGV("MixerThread::invalidateTracks() mixer %p, streamType %d, mTracks.size %d", this, streamType, mTracks.size()); Mutex::Autolock _l(mLock); @@ -1899,6 +2144,8 @@ bool AudioFlinger::PlaybackThread::threadLoop() // and then that string will be logged at the next convenient opportunity. const char *logString = NULL; + checkSilentMode_l(); + while (!exitPending()) { cpuStats.sample(myName); @@ -1917,16 +2164,38 @@ bool AudioFlinger::PlaybackThread::threadLoop() logString = NULL; } + if (mLatchDValid) { + mLatchQ = mLatchD; + mLatchDValid = false; + mLatchQValid = true; + } + if (checkForNewParameters_l()) { cacheParameters_l(); } saveOutputTracks(); + if (mSignalPending) { + // A signal was raised while we were unlocked + mSignalPending = false; + } else if (waitingAsyncCallback_l()) { + if (exitPending()) { + break; + } + releaseWakeLock_l(); + ALOGV("wait async completion"); + mWaitWorkCV.wait(mLock); + ALOGV("async completion/wake"); + acquireWakeLock_l(); + standbyTime = systemTime() + standbyDelay; + sleepTime = 0; - // put audio hardware into standby after short delay - if (CC_UNLIKELY((!mActiveTracks.size() && systemTime() > standbyTime) || - isSuspended())) { - if (!mStandby) { + continue; + } + if ((!mActiveTracks.size() && systemTime() > standbyTime) || + isSuspended()) { + // put audio hardware into standby after short delay + if (shouldStandby_l()) { threadLoop_standby(); @@ -1953,7 +2222,7 @@ bool AudioFlinger::PlaybackThread::threadLoop() mMixerStatus = MIXER_IDLE; mMixerStatusIgnoringFastTracks = MIXER_IDLE; mBytesWritten = 0; - + mBytesRemaining = 0; checkSilentMode_l(); standbyTime = systemTime() + standbyDelay; @@ -1965,7 +2234,6 @@ bool AudioFlinger::PlaybackThread::threadLoop() continue; } } - // mMixerStatusIgnoringFastTracks is also updated internally mMixerStatus = prepareTracks_l(&tracksToRemove); @@ -1975,19 +2243,40 @@ bool AudioFlinger::PlaybackThread::threadLoop() lockEffectChains_l(effectChains); } - if (CC_LIKELY(mMixerStatus == MIXER_TRACKS_READY)) { - threadLoop_mix(); - } else { - threadLoop_sleepTime(); - } + if (mBytesRemaining == 0) { + mCurrentWriteLength = 0; + if (mMixerStatus == MIXER_TRACKS_READY) { + // threadLoop_mix() sets mCurrentWriteLength + threadLoop_mix(); + } else if ((mMixerStatus != MIXER_DRAIN_TRACK) + && (mMixerStatus != MIXER_DRAIN_ALL)) { + // threadLoop_sleepTime sets sleepTime to 0 if data + // must be written to HAL + threadLoop_sleepTime(); + if (sleepTime == 0) { + mCurrentWriteLength = mixBufferSize; + } + } + mBytesRemaining = mCurrentWriteLength; + if (isSuspended()) { + sleepTime = suspendSleepTimeUs(); + // simulate write to HAL when suspended + mBytesWritten += mixBufferSize; + mBytesRemaining = 0; + } - if (isSuspended()) { - sleepTime = suspendSleepTimeUs(); - mBytesWritten += mixBufferSize; + // only process effects if we're going to write + if (sleepTime == 0 && mType != OFFLOAD) { + for (size_t i = 0; i < effectChains.size(); i ++) { + effectChains[i]->process_l(); + } + } } - - // only process effects if we're going to write - if (sleepTime == 0) { + // Process effect chains for offloaded thread even if no audio + // was read from audio track: process only updates effect state + // and thus does have to be synchronized with audio writes but may have + // to be called while waiting for async write callback + if (mType == OFFLOAD) { for (size_t i = 0; i < effectChains.size(); i ++) { effectChains[i]->process_l(); } @@ -1996,29 +2285,40 @@ bool AudioFlinger::PlaybackThread::threadLoop() // enable changes in effect chain unlockEffectChains(effectChains); - // sleepTime == 0 means we must write to audio hardware - if (sleepTime == 0) { - - threadLoop_write(); - + if (!waitingAsyncCallback()) { + // sleepTime == 0 means we must write to audio hardware + if (sleepTime == 0) { + if (mBytesRemaining) { + ssize_t ret = threadLoop_write(); + if (ret < 0) { + mBytesRemaining = 0; + } else { + mBytesWritten += ret; + mBytesRemaining -= ret; + } + } else if ((mMixerStatus == MIXER_DRAIN_TRACK) || + (mMixerStatus == MIXER_DRAIN_ALL)) { + threadLoop_drain(); + } if (mType == MIXER) { - // write blocked detection - nsecs_t now = systemTime(); - nsecs_t delta = now - mLastWriteTime; - if (!mStandby && delta > maxPeriod) { - mNumDelayedWrites++; - if ((now - lastWarning) > kWarningThrottleNs) { - ATRACE_NAME("underrun"); - ALOGW("write blocked for %llu msecs, %d delayed writes, thread %p", - ns2ms(delta), mNumDelayedWrites, this); - lastWarning = now; + // write blocked detection + nsecs_t now = systemTime(); + nsecs_t delta = now - mLastWriteTime; + if (!mStandby && delta > maxPeriod) { + mNumDelayedWrites++; + if ((now - lastWarning) > kWarningThrottleNs) { + ATRACE_NAME("underrun"); + ALOGW("write blocked for %llu msecs, %d delayed writes, thread %p", + ns2ms(delta), mNumDelayedWrites, this); + lastWarning = now; + } } - } } - mStandby = false; - } else { - usleep(sleepTime); + mStandby = false; + } else { + usleep(sleepTime); + } } // Finally let go of removed track(s), without the lock held @@ -2040,8 +2340,10 @@ if (mType == MIXER) { // is now local to this block, but will keep it for now (at least until merge done). } + threadLoop_exit(); + // for DuplicatingThread, standby mode is handled by the outputTracks, otherwise ... - if (mType == MIXER || mType == DIRECT) { + if (mType == MIXER || mType == DIRECT || mType == OFFLOAD) { // put output stream into standby mode if (!mStandby) { mOutput->stream->common.standby(&mOutput->stream->common); @@ -2054,7 +2356,45 @@ if (mType == MIXER) { return false; } +// removeTracks_l() must be called with ThreadBase::mLock held +void AudioFlinger::PlaybackThread::removeTracks_l(const Vector< sp<Track> >& tracksToRemove) +{ + size_t count = tracksToRemove.size(); + if (count) { + for (size_t i=0 ; i<count ; i++) { + const sp<Track>& track = tracksToRemove.itemAt(i); + mActiveTracks.remove(track); + ALOGV("removeTracks_l removing track on session %d", track->sessionId()); + sp<EffectChain> chain = getEffectChain_l(track->sessionId()); + if (chain != 0) { + ALOGV("stopping track on chain %p for session Id: %d", chain.get(), + track->sessionId()); + chain->decActiveTrackCnt(); + } + if (track->isTerminated()) { + removeTrack_l(track); + } + } + } + +} +status_t AudioFlinger::PlaybackThread::getTimestamp_l(AudioTimestamp& timestamp) +{ + if (mNormalSink != 0) { + return mNormalSink->getTimestamp(timestamp); + } + if (mType == OFFLOAD && mOutput->stream->get_presentation_position) { + uint64_t position64; + int ret = mOutput->stream->get_presentation_position( + mOutput->stream, &position64, ×tamp.mTime); + if (ret == 0) { + timestamp.mPosition = (uint32_t)position64; + return NO_ERROR; + } + } + return INVALID_OPERATION; +} // ---------------------------------------------------------------------------- AudioFlinger::MixerThread::MixerThread(const sp<AudioFlinger>& audioFlinger, AudioStreamOut* output, @@ -2068,7 +2408,7 @@ AudioFlinger::MixerThread::MixerThread(const sp<AudioFlinger>& audioFlinger, Aud // mNormalSink below { ALOGV("MixerThread() id=%d device=%#x type=%d", id, device, type); - ALOGV("mSampleRate=%u, mChannelMask=%#x, mChannelCount=%d, mFormat=%d, mFrameSize=%u, " + ALOGV("mSampleRate=%u, mChannelMask=%#x, mChannelCount=%u, mFormat=%d, mFrameSize=%u, " "mFrameCount=%d, mNormalFrameCount=%d", mSampleRate, mChannelMask, mChannelCount, mFormat, mFrameSize, mFrameCount, mNormalFrameCount); @@ -2258,7 +2598,7 @@ void AudioFlinger::MixerThread::threadLoop_removeTracks(const Vector< sp<Track> PlaybackThread::threadLoop_removeTracks(tracksToRemove); } -void AudioFlinger::MixerThread::threadLoop_write() +ssize_t AudioFlinger::MixerThread::threadLoop_write() { // FIXME we should only do one push per cycle; confirm this is true // Start the fast mixer if it's not already running @@ -2279,6 +2619,8 @@ void AudioFlinger::MixerThread::threadLoop_write() #endif } state->mCommand = FastMixerState::MIX_WRITE; + mFastMixerDumpState.increaseSamplingN(mAudioFlinger->isLowRamDevice() ? + FastMixerDumpState::kSamplingNforLowRamDevice : FastMixerDumpState::kSamplingN); sq->end(); sq->push(FastMixerStateQueue::BLOCK_UNTIL_PUSHED); if (kUseFastMixer == FastMixer_Dynamic) { @@ -2288,7 +2630,7 @@ void AudioFlinger::MixerThread::threadLoop_write() sq->end(false /*didModify*/); } } - PlaybackThread::threadLoop_write(); + return PlaybackThread::threadLoop_write(); } void AudioFlinger::MixerThread::threadLoop_standby() @@ -2320,11 +2662,41 @@ void AudioFlinger::MixerThread::threadLoop_standby() PlaybackThread::threadLoop_standby(); } +// Empty implementation for standard mixer +// Overridden for offloaded playback +void AudioFlinger::PlaybackThread::flushOutput_l() +{ +} + +bool AudioFlinger::PlaybackThread::waitingAsyncCallback_l() +{ + return false; +} + +bool AudioFlinger::PlaybackThread::shouldStandby_l() +{ + return !mStandby; +} + +bool AudioFlinger::PlaybackThread::waitingAsyncCallback() +{ + Mutex::Autolock _l(mLock); + return waitingAsyncCallback_l(); +} + // shared by MIXER and DIRECT, overridden by DUPLICATING void AudioFlinger::PlaybackThread::threadLoop_standby() { ALOGV("Audio hardware entering standby, mixer %p, suspend count %d", this, mSuspended); mOutput->stream->common.standby(&mOutput->stream->common); + if (mUseAsyncWrite != 0) { + // discard any pending drain or write ack by incrementing sequence + mWriteAckSequence = (mWriteAckSequence + 2) & ~1; + mDrainSequence = (mDrainSequence + 2) & ~1; + ALOG_ASSERT(mCallbackThread != 0); + mCallbackThread->setWriteBlocked(mWriteAckSequence); + mCallbackThread->setDraining(mDrainSequence); + } } void AudioFlinger::MixerThread::threadLoop_mix() @@ -2345,6 +2717,7 @@ void AudioFlinger::MixerThread::threadLoop_mix() // mix buffers... mAudioMixer->process(pts); + mCurrentWriteLength = mixBufferSize; // increase sleep time progressively when application underrun condition clears. // Only increase sleep time if the mixer is ready for two consecutive times to avoid // that a steady state of alternating ready/not ready conditions keeps the sleep time @@ -2426,7 +2799,7 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac } for (size_t i=0 ; i<count ; i++) { - sp<Track> t = mActiveTracks[i].promote(); + const sp<Track> t = mActiveTracks[i].promote(); if (t == 0) { continue; } @@ -2462,8 +2835,10 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac track->mObservedUnderruns = underruns; // don't count underruns that occur while stopping or pausing // or stopped which can occur when flush() is called while active - if (!(track->isStopping() || track->isPausing() || track->isStopped())) { - track->mUnderrunCount += recentUnderruns; + if (!(track->isStopping() || track->isPausing() || track->isStopped()) && + recentUnderruns > 0) { + // FIXME fast mixer will pull & mix partial buffers, but we count as a full underrun + track->mAudioTrackServerProxy->tallyUnderrunFrames(recentUnderruns * mFrameCount); } // This is similar to the state machine for normal tracks, @@ -2472,7 +2847,7 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac switch (track->mState) { case TrackBase::STOPPING_1: // track stays active in STOPPING_1 state until first underrun - if (recentUnderruns > 0) { + if (recentUnderruns > 0 || track->isTerminated()) { track->mState = TrackBase::STOPPING_2; } break; @@ -2506,7 +2881,7 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac } // indicate to client process that the track was disabled because of underrun; // it will then automatically call start() when data is available - android_atomic_or(CBLK_DISABLED, &track->mCblk->flags); + android_atomic_or(CBLK_DISABLED, &track->mCblk->mFlags); // remove from active list, but state remains ACTIVE [confusing but true] isActive = false; break; @@ -2514,7 +2889,6 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac // fall through case TrackBase::STOPPING_2: case TrackBase::PAUSED: - case TrackBase::TERMINATED: case TrackBase::STOPPED: case TrackBase::FLUSHED: // flush() while active // Check for presentation complete if track is inactive @@ -2595,28 +2969,39 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac // app does not call stop() and relies on underrun to stop: // hence the test on (mMixerStatus == MIXER_TRACKS_READY) meaning the track was mixed // during last round + size_t desiredFrames; + uint32_t sr = track->sampleRate(); + if (sr == mSampleRate) { + desiredFrames = mNormalFrameCount; + } else { + // +1 for rounding and +1 for additional sample needed for interpolation + desiredFrames = (mNormalFrameCount * sr) / mSampleRate + 1 + 1; + // add frames already consumed but not yet released by the resampler + // because cblk->framesReady() will include these frames + desiredFrames += mAudioMixer->getUnreleasedFrames(track->name()); + // the minimum track buffer size is normally twice the number of frames necessary + // to fill one buffer and the resampler should not leave more than one buffer worth + // of unreleased frames after each pass, but just in case... + ALOG_ASSERT(desiredFrames <= cblk->frameCount_); + } uint32_t minFrames = 1; if ((track->sharedBuffer() == 0) && !track->isStopped() && !track->isPausing() && (mMixerStatusIgnoringFastTracks == MIXER_TRACKS_READY)) { - if (t->sampleRate() == mSampleRate) { - minFrames = mNormalFrameCount; - } else { - // +1 for rounding and +1 for additional sample needed for interpolation - minFrames = (mNormalFrameCount * t->sampleRate()) / mSampleRate + 1 + 1; - // add frames already consumed but not yet released by the resampler - // because cblk->framesReady() will include these frames - minFrames += mAudioMixer->getUnreleasedFrames(track->name()); - // the minimum track buffer size is normally twice the number of frames necessary - // to fill one buffer and the resampler should not leave more than one buffer worth - // of unreleased frames after each pass, but just in case... - ALOG_ASSERT(minFrames <= cblk->frameCount_); - } + minFrames = desiredFrames; } - if ((track->framesReady() >= minFrames) && track->isReady() && + // It's not safe to call framesReady() for a static buffer track, so assume it's ready + size_t framesReady; + if (track->sharedBuffer() == 0) { + framesReady = track->framesReady(); + } else if (track->isStopped()) { + framesReady = 0; + } else { + framesReady = 1; + } + if ((framesReady >= minFrames) && track->isReady() && !track->isPaused() && !track->isTerminated()) { - ALOGVV("track %d u=%08x, s=%08x [OK] on thread %p", name, cblk->user, cblk->server, - this); + ALOGVV("track %d s=%08x [OK] on thread %p", name, cblk->mServer, this); mixedTracks++; @@ -2645,7 +3030,8 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac param = AudioMixer::RAMP_VOLUME; } mAudioMixer->setParameter(name, AudioMixer::RESAMPLE, AudioMixer::RESET, NULL); - } else if (cblk->server != 0) { + // FIXME should not make a decision based on mServer + } else if (cblk->mServer != 0) { // If the track is stopped before the first frame was mixed, // do not apply ramp param = AudioMixer::RAMP_VOLUME; @@ -2663,7 +3049,7 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac // read original volumes with volume control float typeVolume = mStreamTypes[track->streamType()].volume; float v = masterVolume * typeVolume; - ServerProxy *proxy = track->mServerProxy; + AudioTrackServerProxy *proxy = track->mAudioTrackServerProxy; uint32_t vlr = proxy->getVolumeLR(); vl = vlr & 0xFFFF; vr = vlr >> 16; @@ -2690,6 +3076,7 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac } va = (uint32_t)(v * sendLevel); } + // Delegate volume control to effect in track effect chain if needed if (chain != 0 && chain->setVolume_l(&vl, &vr)) { // Do not ramp volume if volume is controlled by effect @@ -2736,7 +3123,7 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac AudioMixer::CHANNEL_MASK, (void *)track->channelMask()); // limit track sample rate to 2 x output sample rate, which changes at re-configuration uint32_t maxSampleRate = mSampleRate * 2; - uint32_t reqSampleRate = track->mServerProxy->getSampleRate(); + uint32_t reqSampleRate = track->mAudioTrackServerProxy->getSampleRate(); if (reqSampleRate == 0) { reqSampleRate = mSampleRate; } else if (reqSampleRate > maxSampleRate) { @@ -2767,6 +3154,9 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac mixerStatus = MIXER_TRACKS_READY; } } else { + if (framesReady < desiredFrames && !track->isStopped() && !track->isPaused()) { + track->mAudioTrackServerProxy->tallyUnderrunFrames(desiredFrames); + } // clear effect chain input buffer if an active track underruns to avoid sending // previous audio buffer again to effects chain = getEffectChain_l(track->sessionId()); @@ -2774,8 +3164,7 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac chain->clearInputBuffer(); } - ALOGVV("track %d u=%08x, s=%08x [NOT READY] on thread %p", name, cblk->user, - cblk->server, this); + ALOGVV("track %d s=%08x [NOT READY] on thread %p", name, cblk->mServer, this); if ((track->sharedBuffer() != 0) || track->isTerminated() || track->isStopped() || track->isPaused()) { // We have consumed all the buffers of this track. @@ -2791,7 +3180,6 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac tracksToRemove->add(track); } } else { - track->mUnderrunCount++; // No buffers for this track. Give it a few chances to // fill a buffer, then remove it from active list. if (--(track->mRetryCount) <= 0) { @@ -2799,7 +3187,7 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac tracksToRemove->add(track); // indicate to client process that the track was disabled because of underrun; // it will then automatically call start() when data is available - android_atomic_or(CBLK_DISABLED, &cblk->flags); + android_atomic_or(CBLK_DISABLED, &cblk->mFlags); // If one track is not ready, mark the mixer also not ready if: // - the mixer was ready during previous round OR // - no other track is ready @@ -2861,30 +3249,13 @@ track_is_ready: ; } // remove all the tracks that need to be... - count = tracksToRemove->size(); - if (CC_UNLIKELY(count)) { - for (size_t i=0 ; i<count ; i++) { - const sp<Track>& track = tracksToRemove->itemAt(i); - mActiveTracks.remove(track); - if (track->mainBuffer() != mMixBuffer) { - chain = getEffectChain_l(track->sessionId()); - if (chain != 0) { - ALOGV("stopping track on chain %p for session Id: %d", chain.get(), - track->sessionId()); - chain->decActiveTrackCnt(); - } - } - if (track->isTerminated()) { - removeTrack_l(track); - } - } - } + removeTracks_l(*tracksToRemove); // mix buffer must be cleared if all tracks are connected to an // effect chain as in this case the mixer will not write to // mix buffer and track effects will accumulate into it - if ((mixedTracks != 0 && mixedTracks == tracksWithEffect) || - (mixedTracks == 0 && fastTracks > 0)) { + if ((mBytesRemaining == 0) && ((mixedTracks != 0 && mixedTracks == tracksWithEffect) || + (mixedTracks == 0 && fastTracks > 0))) { // FIXME as a performance optimization, should remember previous zero status memset(mMixBuffer, 0, mNormalFrameCount * mChannelCount * sizeof(int16_t)); } @@ -2948,7 +3319,7 @@ bool AudioFlinger::MixerThread::checkForNewParameters_l() } } if (param.getInt(String8(AudioParameter::keyChannels), value) == NO_ERROR) { - if (value != AUDIO_CHANNEL_OUT_STEREO) { + if ((audio_channel_mask_t) value != AUDIO_CHANNEL_OUT_STEREO) { status = BAD_VALUE; } else { reconfig = true; @@ -3009,10 +3380,8 @@ bool AudioFlinger::MixerThread::checkForNewParameters_l() keyValuePair.string()); } if (status == NO_ERROR && reconfig) { - delete mAudioMixer; - // for safety in case readOutputParameters() accesses mAudioMixer (it doesn't) - mAudioMixer = NULL; readOutputParameters(); + delete mAudioMixer; mAudioMixer = new AudioMixer(mNormalFrameCount, mSampleRate); for (size_t i = 0; i < mTracks.size() ; i++) { int name = getTrackName_l(mTracks[i]->mChannelMask, mTracks[i]->mSessionId); @@ -3061,7 +3430,7 @@ void AudioFlinger::MixerThread::dumpInternals(int fd, const Vector<String16>& ar write(fd, result.string(), result.size()); // Make a non-atomic copy of fast mixer dump state so it won't change underneath us - FastMixerDumpState copy = mFastMixerDumpState; + const FastMixerDumpState copy(mFastMixerDumpState); copy.dump(fd); #ifdef STATE_QUEUE_DUMP @@ -3116,10 +3485,63 @@ AudioFlinger::DirectOutputThread::DirectOutputThread(const sp<AudioFlinger>& aud { } +AudioFlinger::DirectOutputThread::DirectOutputThread(const sp<AudioFlinger>& audioFlinger, + AudioStreamOut* output, audio_io_handle_t id, uint32_t device, + ThreadBase::type_t type) + : PlaybackThread(audioFlinger, output, id, device, type) + // mLeftVolFloat, mRightVolFloat +{ +} + AudioFlinger::DirectOutputThread::~DirectOutputThread() { } +void AudioFlinger::DirectOutputThread::processVolume_l(Track *track, bool lastTrack) +{ + audio_track_cblk_t* cblk = track->cblk(); + float left, right; + + if (mMasterMute || mStreamTypes[track->streamType()].mute) { + left = right = 0; + } else { + float typeVolume = mStreamTypes[track->streamType()].volume; + float v = mMasterVolume * typeVolume; + AudioTrackServerProxy *proxy = track->mAudioTrackServerProxy; + uint32_t vlr = proxy->getVolumeLR(); + float v_clamped = v * (vlr & 0xFFFF); + if (v_clamped > MAX_GAIN) v_clamped = MAX_GAIN; + left = v_clamped/MAX_GAIN; + v_clamped = v * (vlr >> 16); + if (v_clamped > MAX_GAIN) v_clamped = MAX_GAIN; + right = v_clamped/MAX_GAIN; + } + + if (lastTrack) { + if (left != mLeftVolFloat || right != mRightVolFloat) { + mLeftVolFloat = left; + mRightVolFloat = right; + + // Convert volumes from float to 8.24 + uint32_t vl = (uint32_t)(left * (1 << 24)); + uint32_t vr = (uint32_t)(right * (1 << 24)); + + // Delegate volume control to effect in track effect chain if needed + // only one effect chain can be present on DirectOutputThread, so if + // there is one, the track is connected to it + if (!mEffectChains.isEmpty()) { + mEffectChains[0]->setVolume_l(&vl, &vr); + left = (float)vl / (1 << 24); + right = (float)vr / (1 << 24); + } + if (mOutput->stream->set_volume) { + mOutput->stream->set_volume(mOutput->stream, left, right); + } + } + } +} + + AudioFlinger::PlaybackThread::mixer_state AudioFlinger::DirectOutputThread::prepareTracks_l( Vector< sp<Track> > *tracksToRemove ) @@ -3146,66 +3568,29 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::DirectOutputThread::prep } else { minFrames = 1; } + // Only consider last track started for volume and mixer state control. + // This is the last entry in mActiveTracks unless a track underruns. + // As we only care about the transition phase between two tracks on a + // direct output, it is not a problem to ignore the underrun case. + bool last = (i == (count - 1)); + if ((track->framesReady() >= minFrames) && track->isReady() && !track->isPaused() && !track->isTerminated()) { - ALOGVV("track %d u=%08x, s=%08x [OK]", track->name(), cblk->user, cblk->server); + ALOGVV("track %d s=%08x [OK]", track->name(), cblk->mServer); if (track->mFillingUpStatus == Track::FS_FILLED) { track->mFillingUpStatus = Track::FS_ACTIVE; - mLeftVolFloat = mRightVolFloat = 0; + // make sure processVolume_l() will apply new volume even if 0 + mLeftVolFloat = mRightVolFloat = -1.0; if (track->mState == TrackBase::RESUMING) { track->mState = TrackBase::ACTIVE; } } // compute volume for this track - float left, right; - if (mMasterMute || track->isPausing() || mStreamTypes[track->streamType()].mute) { - left = right = 0; - if (track->isPausing()) { - track->setPaused(); - } - } else { - float typeVolume = mStreamTypes[track->streamType()].volume; - float v = mMasterVolume * typeVolume; - uint32_t vlr = track->mServerProxy->getVolumeLR(); - float v_clamped = v * (vlr & 0xFFFF); - if (v_clamped > MAX_GAIN) { - v_clamped = MAX_GAIN; - } - left = v_clamped/MAX_GAIN; - v_clamped = v * (vlr >> 16); - if (v_clamped > MAX_GAIN) { - v_clamped = MAX_GAIN; - } - right = v_clamped/MAX_GAIN; - } - // Only consider last track started for volume and mixer state control. - // This is the last entry in mActiveTracks unless a track underruns. - // As we only care about the transition phase between two tracks on a - // direct output, it is not a problem to ignore the underrun case. - if (i == (count - 1)) { - if (left != mLeftVolFloat || right != mRightVolFloat) { - mLeftVolFloat = left; - mRightVolFloat = right; - - // Convert volumes from float to 8.24 - uint32_t vl = (uint32_t)(left * (1 << 24)); - uint32_t vr = (uint32_t)(right * (1 << 24)); - - // Delegate volume control to effect in track effect chain if needed - // only one effect chain can be present on DirectOutputThread, so if - // there is one, the track is connected to it - if (!mEffectChains.isEmpty()) { - // Do not ramp volume if volume is controlled by effect - mEffectChains[0]->setVolume_l(&vl, &vr); - left = (float)vl / (1 << 24); - right = (float)vr / (1 << 24); - } - mOutput->stream->set_volume(mOutput->stream, left, right); - } - + processVolume_l(track, last); + if (last) { // reset retry count track->mRetryCount = kMaxTrackRetriesDirect; mActiveTrack = t; @@ -3218,7 +3603,7 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::DirectOutputThread::prep mEffectChains[0]->clearInputBuffer(); } - ALOGVV("track %d u=%08x, s=%08x [NOT READY]", track->name(), cblk->user, cblk->server); + ALOGVV("track %d s=%08x [NOT READY]", track->name(), cblk->mServer); if ((track->sharedBuffer() != 0) || track->isTerminated() || track->isStopped() || track->isPaused()) { // We have consumed all the buffers of this track. @@ -3239,7 +3624,7 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::DirectOutputThread::prep if (--(track->mRetryCount) <= 0) { ALOGV("BUFFER TIMEOUT: remove(%d) from active list", track->name()); tracksToRemove->add(track); - } else if (i == (count -1)){ + } else if (last) { mixerStatus = MIXER_TRACKS_ENABLED; } } @@ -3247,35 +3632,21 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::DirectOutputThread::prep } // remove all the tracks that need to be... - count = tracksToRemove->size(); - if (CC_UNLIKELY(count)) { - for (size_t i = 0 ; i < count ; i++) { - const sp<Track>& track = tracksToRemove->itemAt(i); - mActiveTracks.remove(track); - if (!mEffectChains.isEmpty()) { - ALOGV("stopping track on chain %p for session Id: %d", mEffectChains[0].get(), - track->sessionId()); - mEffectChains[0]->decActiveTrackCnt(); - } - if (track->isTerminated()) { - removeTrack_l(track); - } - } - } + removeTracks_l(*tracksToRemove); return mixerStatus; } void AudioFlinger::DirectOutputThread::threadLoop_mix() { - AudioBufferProvider::Buffer buffer; size_t frameCount = mFrameCount; int8_t *curBuf = (int8_t *)mMixBuffer; // output audio to hardware while (frameCount) { + AudioBufferProvider::Buffer buffer; buffer.frameCount = frameCount; mActiveTrack->getNextBuffer(&buffer); - if (CC_UNLIKELY(buffer.raw == NULL)) { + if (buffer.raw == NULL) { memset(curBuf, 0, frameCount * mFrameSize); break; } @@ -3284,10 +3655,10 @@ void AudioFlinger::DirectOutputThread::threadLoop_mix() curBuf += buffer.frameCount * mFrameSize; mActiveTrack->releaseBuffer(&buffer); } + mCurrentWriteLength = curBuf - (int8_t *)mMixBuffer; sleepTime = 0; standbyTime = systemTime() + standbyDelay; mActiveTrack.clear(); - } void AudioFlinger::DirectOutputThread::threadLoop_sleepTime() @@ -3403,7 +3774,358 @@ void AudioFlinger::DirectOutputThread::cacheParameters_l() // use shorter standby delay as on normal output to release // hardware resources as soon as possible - standbyDelay = microseconds(activeSleepTime*2); + if (audio_is_linear_pcm(mFormat)) { + standbyDelay = microseconds(activeSleepTime*2); + } else { + standbyDelay = kOffloadStandbyDelayNs; + } +} + +// ---------------------------------------------------------------------------- + +AudioFlinger::AsyncCallbackThread::AsyncCallbackThread( + const wp<AudioFlinger::PlaybackThread>& playbackThread) + : Thread(false /*canCallJava*/), + mPlaybackThread(playbackThread), + mWriteAckSequence(0), + mDrainSequence(0) +{ +} + +AudioFlinger::AsyncCallbackThread::~AsyncCallbackThread() +{ +} + +void AudioFlinger::AsyncCallbackThread::onFirstRef() +{ + run("Offload Cbk", ANDROID_PRIORITY_URGENT_AUDIO); +} + +bool AudioFlinger::AsyncCallbackThread::threadLoop() +{ + while (!exitPending()) { + uint32_t writeAckSequence; + uint32_t drainSequence; + + { + Mutex::Autolock _l(mLock); + mWaitWorkCV.wait(mLock); + if (exitPending()) { + break; + } + ALOGV("AsyncCallbackThread mWriteAckSequence %d mDrainSequence %d", + mWriteAckSequence, mDrainSequence); + writeAckSequence = mWriteAckSequence; + mWriteAckSequence &= ~1; + drainSequence = mDrainSequence; + mDrainSequence &= ~1; + } + { + sp<AudioFlinger::PlaybackThread> playbackThread = mPlaybackThread.promote(); + if (playbackThread != 0) { + if (writeAckSequence & 1) { + playbackThread->resetWriteBlocked(writeAckSequence >> 1); + } + if (drainSequence & 1) { + playbackThread->resetDraining(drainSequence >> 1); + } + } + } + } + return false; +} + +void AudioFlinger::AsyncCallbackThread::exit() +{ + ALOGV("AsyncCallbackThread::exit"); + Mutex::Autolock _l(mLock); + requestExit(); + mWaitWorkCV.broadcast(); +} + +void AudioFlinger::AsyncCallbackThread::setWriteBlocked(uint32_t sequence) +{ + Mutex::Autolock _l(mLock); + // bit 0 is cleared + mWriteAckSequence = sequence << 1; +} + +void AudioFlinger::AsyncCallbackThread::resetWriteBlocked() +{ + Mutex::Autolock _l(mLock); + // ignore unexpected callbacks + if (mWriteAckSequence & 2) { + mWriteAckSequence |= 1; + mWaitWorkCV.signal(); + } +} + +void AudioFlinger::AsyncCallbackThread::setDraining(uint32_t sequence) +{ + Mutex::Autolock _l(mLock); + // bit 0 is cleared + mDrainSequence = sequence << 1; +} + +void AudioFlinger::AsyncCallbackThread::resetDraining() +{ + Mutex::Autolock _l(mLock); + // ignore unexpected callbacks + if (mDrainSequence & 2) { + mDrainSequence |= 1; + mWaitWorkCV.signal(); + } +} + + +// ---------------------------------------------------------------------------- +AudioFlinger::OffloadThread::OffloadThread(const sp<AudioFlinger>& audioFlinger, + AudioStreamOut* output, audio_io_handle_t id, uint32_t device) + : DirectOutputThread(audioFlinger, output, id, device, OFFLOAD), + mHwPaused(false), + mFlushPending(false), + mPausedBytesRemaining(0) +{ +} + +AudioFlinger::OffloadThread::~OffloadThread() +{ + mPreviousTrack.clear(); +} + +void AudioFlinger::OffloadThread::threadLoop_exit() +{ + if (mFlushPending || mHwPaused) { + // If a flush is pending or track was paused, just discard buffered data + flushHw_l(); + } else { + mMixerStatus = MIXER_DRAIN_ALL; + threadLoop_drain(); + } + mCallbackThread->exit(); + PlaybackThread::threadLoop_exit(); +} + +AudioFlinger::PlaybackThread::mixer_state AudioFlinger::OffloadThread::prepareTracks_l( + Vector< sp<Track> > *tracksToRemove +) +{ + size_t count = mActiveTracks.size(); + + mixer_state mixerStatus = MIXER_IDLE; + bool doHwPause = false; + bool doHwResume = false; + + ALOGV("OffloadThread::prepareTracks_l active tracks %d", count); + + // find out which tracks need to be processed + for (size_t i = 0; i < count; i++) { + sp<Track> t = mActiveTracks[i].promote(); + // The track died recently + if (t == 0) { + continue; + } + Track* const track = t.get(); + audio_track_cblk_t* cblk = track->cblk(); + if (mPreviousTrack != NULL) { + if (t != mPreviousTrack) { + // Flush any data still being written from last track + mBytesRemaining = 0; + if (mPausedBytesRemaining) { + // Last track was paused so we also need to flush saved + // mixbuffer state and invalidate track so that it will + // re-submit that unwritten data when it is next resumed + mPausedBytesRemaining = 0; + // Invalidate is a bit drastic - would be more efficient + // to have a flag to tell client that some of the + // previously written data was lost + mPreviousTrack->invalidate(); + } + } + } + mPreviousTrack = t; + bool last = (i == (count - 1)); + if (track->isPausing()) { + track->setPaused(); + if (last) { + if (!mHwPaused) { + doHwPause = true; + mHwPaused = true; + } + // If we were part way through writing the mixbuffer to + // the HAL we must save this until we resume + // BUG - this will be wrong if a different track is made active, + // in that case we want to discard the pending data in the + // mixbuffer and tell the client to present it again when the + // track is resumed + mPausedWriteLength = mCurrentWriteLength; + mPausedBytesRemaining = mBytesRemaining; + mBytesRemaining = 0; // stop writing + } + tracksToRemove->add(track); + } else if (track->framesReady() && track->isReady() && + !track->isPaused() && !track->isTerminated() && !track->isStopping_2()) { + ALOGVV("OffloadThread: track %d s=%08x [OK]", track->name(), cblk->mServer); + if (track->mFillingUpStatus == Track::FS_FILLED) { + track->mFillingUpStatus = Track::FS_ACTIVE; + // make sure processVolume_l() will apply new volume even if 0 + mLeftVolFloat = mRightVolFloat = -1.0; + if (track->mState == TrackBase::RESUMING) { + track->mState = TrackBase::ACTIVE; + if (last) { + if (mPausedBytesRemaining) { + // Need to continue write that was interrupted + mCurrentWriteLength = mPausedWriteLength; + mBytesRemaining = mPausedBytesRemaining; + mPausedBytesRemaining = 0; + } + if (mHwPaused) { + doHwResume = true; + mHwPaused = false; + // threadLoop_mix() will handle the case that we need to + // resume an interrupted write + } + // enable write to audio HAL + sleepTime = 0; + } + } + } + + if (last) { + // reset retry count + track->mRetryCount = kMaxTrackRetriesOffload; + mActiveTrack = t; + mixerStatus = MIXER_TRACKS_READY; + } + } else { + ALOGVV("OffloadThread: track %d s=%08x [NOT READY]", track->name(), cblk->mServer); + if (track->isStopping_1()) { + // Hardware buffer can hold a large amount of audio so we must + // wait for all current track's data to drain before we say + // that the track is stopped. + if (mBytesRemaining == 0) { + // Only start draining when all data in mixbuffer + // has been written + ALOGV("OffloadThread: underrun and STOPPING_1 -> draining, STOPPING_2"); + track->mState = TrackBase::STOPPING_2; // so presentation completes after drain + if (last) { + sleepTime = 0; + standbyTime = systemTime() + standbyDelay; + mixerStatus = MIXER_DRAIN_TRACK; + mDrainSequence += 2; + if (mHwPaused) { + // It is possible to move from PAUSED to STOPPING_1 without + // a resume so we must ensure hardware is running + mOutput->stream->resume(mOutput->stream); + mHwPaused = false; + } + } + } + } else if (track->isStopping_2()) { + // Drain has completed, signal presentation complete + if (!(mDrainSequence & 1) || !last) { + track->mState = TrackBase::STOPPED; + size_t audioHALFrames = + (mOutput->stream->get_latency(mOutput->stream)*mSampleRate) / 1000; + size_t framesWritten = + mBytesWritten / audio_stream_frame_size(&mOutput->stream->common); + track->presentationComplete(framesWritten, audioHALFrames); + track->reset(); + tracksToRemove->add(track); + } + } else { + // No buffers for this track. Give it a few chances to + // fill a buffer, then remove it from active list. + if (--(track->mRetryCount) <= 0) { + ALOGV("OffloadThread: BUFFER TIMEOUT: remove(%d) from active list", + track->name()); + tracksToRemove->add(track); + } else if (last){ + mixerStatus = MIXER_TRACKS_ENABLED; + } + } + } + // compute volume for this track + processVolume_l(track, last); + } + + // make sure the pause/flush/resume sequence is executed in the right order. + // If a flush is pending and a track is active but the HW is not paused, force a HW pause + // before flush and then resume HW. This can happen in case of pause/flush/resume + // if resume is received before pause is executed. + if (doHwPause || (mFlushPending && !mHwPaused && (count != 0))) { + mOutput->stream->pause(mOutput->stream); + if (!doHwPause) { + doHwResume = true; + } + } + if (mFlushPending) { + flushHw_l(); + mFlushPending = false; + } + if (doHwResume) { + mOutput->stream->resume(mOutput->stream); + } + + // remove all the tracks that need to be... + removeTracks_l(*tracksToRemove); + + return mixerStatus; +} + +void AudioFlinger::OffloadThread::flushOutput_l() +{ + mFlushPending = true; +} + +// must be called with thread mutex locked +bool AudioFlinger::OffloadThread::waitingAsyncCallback_l() +{ + ALOGVV("waitingAsyncCallback_l mWriteAckSequence %d mDrainSequence %d", + mWriteAckSequence, mDrainSequence); + if (mUseAsyncWrite && ((mWriteAckSequence & 1) || (mDrainSequence & 1))) { + return true; + } + return false; +} + +// must be called with thread mutex locked +bool AudioFlinger::OffloadThread::shouldStandby_l() +{ + bool TrackPaused = false; + + // do not put the HAL in standby when paused. AwesomePlayer clear the offloaded AudioTrack + // after a timeout and we will enter standby then. + if (mTracks.size() > 0) { + TrackPaused = mTracks[mTracks.size() - 1]->isPaused(); + } + + return !mStandby && !TrackPaused; +} + + +bool AudioFlinger::OffloadThread::waitingAsyncCallback() +{ + Mutex::Autolock _l(mLock); + return waitingAsyncCallback_l(); +} + +void AudioFlinger::OffloadThread::flushHw_l() +{ + mOutput->stream->flush(mOutput->stream); + // Flush anything still waiting in the mixbuffer + mCurrentWriteLength = 0; + mBytesRemaining = 0; + mPausedWriteLength = 0; + mPausedBytesRemaining = 0; + if (mUseAsyncWrite) { + // discard any pending drain or write ack by incrementing sequence + mWriteAckSequence = (mWriteAckSequence + 2) & ~1; + mDrainSequence = (mDrainSequence + 2) & ~1; + ALOG_ASSERT(mCallbackThread != 0); + mCallbackThread->setWriteBlocked(mWriteAckSequence); + mCallbackThread->setDraining(mDrainSequence); + } } // ---------------------------------------------------------------------------- @@ -3434,6 +4156,7 @@ void AudioFlinger::DuplicatingThread::threadLoop_mix() } sleepTime = 0; writeFrames = mNormalFrameCount; + mCurrentWriteLength = mixBufferSize; standbyTime = systemTime() + standbyDelay; } @@ -3457,12 +4180,12 @@ void AudioFlinger::DuplicatingThread::threadLoop_sleepTime() } } -void AudioFlinger::DuplicatingThread::threadLoop_write() +ssize_t AudioFlinger::DuplicatingThread::threadLoop_write() { for (size_t i = 0; i < outputTracks.size(); i++) { outputTracks[i]->write(mMixBuffer, writeFrames); } - mBytesWritten += mixBufferSize; + return (ssize_t)mixBufferSize; } void AudioFlinger::DuplicatingThread::threadLoop_standby() @@ -3583,7 +4306,7 @@ AudioFlinger::RecordThread::RecordThread(const sp<AudioFlinger>& audioFlinger, ) : ThreadBase(audioFlinger, id, outDevice, inDevice, RECORD), mInput(input), mResampler(NULL), mRsmpOutBuffer(NULL), mRsmpInBuffer(NULL), - // mRsmpInIndex and mInputBytes set by readInputParameters() + // mRsmpInIndex and mBufferSize set by readInputParameters() mReqChannelCount(popcount(channelMask)), mReqSampleRate(sampleRate) // mBytesRead is only meaningful while active, and so is cleared in start() @@ -3595,7 +4318,7 @@ AudioFlinger::RecordThread::RecordThread(const sp<AudioFlinger>& audioFlinger, snprintf(mName, kNameLength, "AudioIn_%X", id); readInputParameters(); - + mClientUid = IPCThreadState::self()->getCallingUid(); } @@ -3627,7 +4350,7 @@ bool AudioFlinger::RecordThread::threadLoop() nsecs_t lastWarning = 0; inputStandBy(); - acquireWakeLock(); + acquireWakeLock(mClientUid); // used to verify we've read at least once before evaluating how many bytes were read bool readOnce = false; @@ -3652,11 +4375,14 @@ bool AudioFlinger::RecordThread::threadLoop() // go to sleep mWaitWorkCV.wait(mLock); ALOGV("RecordThread: loop starting"); - acquireWakeLock_l(); + acquireWakeLock_l(mClientUid); continue; } if (mActiveTrack != 0) { - if (mActiveTrack->mState == TrackBase::PAUSING) { + if (mActiveTrack->isTerminated()) { + removeTrack_l(mActiveTrack); + mActiveTrack.clear(); + } else if (mActiveTrack->mState == TrackBase::PAUSING) { standby(); mActiveTrack.clear(); mStartStopCond.broadcast(); @@ -3675,11 +4401,9 @@ bool AudioFlinger::RecordThread::threadLoop() mStartStopCond.broadcast(); } mStandby = false; - } else if (mActiveTrack->mState == TrackBase::TERMINATED) { - removeTrack_l(mActiveTrack); - mActiveTrack.clear(); } } + lockEffectChains_l(effectChains); } @@ -3695,7 +4419,8 @@ bool AudioFlinger::RecordThread::threadLoop() } buffer.frameCount = mFrameCount; - if (CC_LIKELY(mActiveTrack->getNextBuffer(&buffer) == NO_ERROR)) { + status_t status = mActiveTrack->getNextBuffer(&buffer); + if (status == NO_ERROR) { readOnce = true; size_t framesOut = buffer.frameCount; if (mResampler == NULL) { @@ -3710,8 +4435,7 @@ bool AudioFlinger::RecordThread::threadLoop() framesIn = framesOut; mRsmpInIndex += framesIn; framesOut -= framesIn; - if (mChannelCount == mReqChannelCount || - mFormat != AUDIO_FORMAT_PCM_16_BIT) { + if (mChannelCount == mReqChannelCount) { memcpy(dst, src, framesIn * mFrameSize); } else { if (mChannelCount == 1) { @@ -3725,9 +4449,7 @@ bool AudioFlinger::RecordThread::threadLoop() } if (framesOut && mFrameCount == mRsmpInIndex) { void *readInto; - if (framesOut == mFrameCount && - (mChannelCount == mReqChannelCount || - mFormat != AUDIO_FORMAT_PCM_16_BIT)) { + if (framesOut == mFrameCount && mChannelCount == mReqChannelCount) { readInto = buffer.raw; framesOut = 0; } else { @@ -3735,7 +4457,7 @@ bool AudioFlinger::RecordThread::threadLoop() mRsmpInIndex = 0; } mBytesRead = mInput->stream->read(mInput->stream, readInto, - mInputBytes); + mBufferSize); if (mBytesRead <= 0) { if ((mBytesRead < 0) && (mActiveTrack->mState == TrackBase::ACTIVE)) { @@ -3760,7 +4482,8 @@ bool AudioFlinger::RecordThread::threadLoop() } else { // resampling - memset(mRsmpOutBuffer, 0, framesOut * 2 * sizeof(int32_t)); + // resampler accumulates, but we only have one source track + memset(mRsmpOutBuffer, 0, framesOut * FCC_2 * sizeof(int32_t)); // alter output frame count as if we were expecting stereo samples if (mChannelCount == 1 && mReqChannelCount == 1) { framesOut >>= 1; @@ -3770,6 +4493,7 @@ bool AudioFlinger::RecordThread::threadLoop() // ditherAndClamp() works as long as all buffers returned by // mActiveTrack->getNextBuffer() are 32 bit aligned which should be always true. if (mChannelCount == 2 && mReqChannelCount == 1) { + // temporarily type pun mRsmpOutBuffer from Q19.12 to int16_t ditherAndClamp(mRsmpOutBuffer, mRsmpOutBuffer, framesOut); // the resampler always outputs stereo samples: // do post stereo to mono conversion @@ -3778,6 +4502,7 @@ bool AudioFlinger::RecordThread::threadLoop() } else { ditherAndClamp((int32_t *)buffer.raw, mRsmpOutBuffer, framesOut); } + // now done with mRsmpOutBuffer } if (mFramestoDrop == 0) { @@ -3826,6 +4551,10 @@ bool AudioFlinger::RecordThread::threadLoop() { Mutex::Autolock _l(mLock); + for (size_t i = 0; i < mTracks.size(); i++) { + sp<RecordTrack> track = mTracks[i]; + track->invalidate(); + } mActiveTrack.clear(); mStartStopCond.broadcast(); } @@ -3856,7 +4585,7 @@ sp<AudioFlinger::RecordThread::RecordTrack> AudioFlinger::RecordThread::createR audio_channel_mask_t channelMask, size_t frameCount, int sessionId, - IAudioFlinger::track_flags_t flags, + IAudioFlinger::track_flags_t *flags, pid_t tid, status_t *status) { @@ -3865,9 +4594,59 @@ sp<AudioFlinger::RecordThread::RecordTrack> AudioFlinger::RecordThread::createR lStatus = initCheck(); if (lStatus != NO_ERROR) { - ALOGE("Audio driver not initialized."); + ALOGE("createRecordTrack_l() audio driver not initialized"); goto Exit; } + // client expresses a preference for FAST, but we get the final say + if (*flags & IAudioFlinger::TRACK_FAST) { + if ( + // use case: callback handler and frame count is default or at least as large as HAL + ( + (tid != -1) && + ((frameCount == 0) || + (frameCount >= (mFrameCount * kFastTrackMultiplier))) + ) && + // FIXME when record supports non-PCM data, also check for audio_is_linear_pcm(format) + // mono or stereo + ( (channelMask == AUDIO_CHANNEL_OUT_MONO) || + (channelMask == AUDIO_CHANNEL_OUT_STEREO) ) && + // hardware sample rate + (sampleRate == mSampleRate) && + // record thread has an associated fast recorder + hasFastRecorder() + // FIXME test that RecordThread for this fast track has a capable output HAL + // FIXME add a permission test also? + ) { + // if frameCount not specified, then it defaults to fast recorder (HAL) frame count + if (frameCount == 0) { + frameCount = mFrameCount * kFastTrackMultiplier; + } + ALOGV("AUDIO_INPUT_FLAG_FAST accepted: frameCount=%d mFrameCount=%d", + frameCount, mFrameCount); + } else { + ALOGV("AUDIO_INPUT_FLAG_FAST denied: frameCount=%d " + "mFrameCount=%d format=%d isLinear=%d channelMask=%#x sampleRate=%u mSampleRate=%u " + "hasFastRecorder=%d tid=%d", + frameCount, mFrameCount, format, + audio_is_linear_pcm(format), + channelMask, sampleRate, mSampleRate, hasFastRecorder(), tid); + *flags &= ~IAudioFlinger::TRACK_FAST; + // For compatibility with AudioRecord calculation, buffer depth is forced + // to be at least 2 x the record thread frame count and cover audio hardware latency. + // This is probably too conservative, but legacy application code may depend on it. + // If you change this calculation, also review the start threshold which is related. + uint32_t latencyMs = 50; // FIXME mInput->stream->get_latency(mInput->stream); + size_t mNormalFrameCount = 2048; // FIXME + uint32_t minBufCount = latencyMs / ((1000 * mNormalFrameCount) / mSampleRate); + if (minBufCount < 2) { + minBufCount = 2; + } + size_t minFrameCount = mNormalFrameCount * minBufCount; + if (frameCount < minFrameCount) { + frameCount = minFrameCount; + } + } + } // FIXME use flags and tid similar to createTrack_l() @@ -3878,7 +4657,9 @@ sp<AudioFlinger::RecordThread::RecordTrack> AudioFlinger::RecordThread::createR format, channelMask, frameCount, sessionId); if (track->getCblk() == 0) { + ALOGE("createRecordTrack_l() no control block"); lStatus = NO_MEMORY; + track.clear(); goto Exit; } mTracks.add(track); @@ -3888,6 +4669,13 @@ sp<AudioFlinger::RecordThread::RecordTrack> AudioFlinger::RecordThread::createR mAudioFlinger->btNrecIsOff(); setEffectSuspended_l(FX_IID_AEC, suspend, sessionId); setEffectSuspended_l(FX_IID_NS, suspend, sessionId); + + if ((*flags & IAudioFlinger::TRACK_FAST) && (tid != -1)) { + pid_t callingPid = IPCThreadState::self()->getCallingPid(); + // we don't have CAP_SYS_NICE, nor do we want to have it as it's too powerful, + // so ask activity manager to do this on our behalf + sendPrioConfigEvent_l(callingPid, tid, kPriorityAudioApp); + } } lStatus = NO_ERROR; @@ -3969,6 +4757,7 @@ status_t AudioFlinger::RecordThread::start(RecordThread::RecordTrack* recordTrac ALOGV("Record started OK"); return status; } + startError: AudioSystem::stopInput(mId); clearSyncStartEvent(); @@ -4003,8 +4792,9 @@ void AudioFlinger::RecordThread::handleSyncStartEvent(const sp<SyncEvent>& event } } -bool AudioFlinger::RecordThread::stop_l(RecordThread::RecordTrack* recordTrack) { +bool AudioFlinger::RecordThread::stop(RecordThread::RecordTrack* recordTrack) { ALOGV("RecordThread::stop"); + AutoMutex _l(mLock); if (recordTrack != mActiveTrack.get() || recordTrack->mState == TrackBase::PAUSING) { return false; } @@ -4055,7 +4845,8 @@ status_t AudioFlinger::RecordThread::setSyncEvent(const sp<SyncEvent>& event) // destroyTrack_l() must be called with ThreadBase::mLock held void AudioFlinger::RecordThread::destroyTrack_l(const sp<RecordTrack>& track) { - track->mState = TrackBase::TERMINATED; + track->terminate(); + track->mState = TrackBase::STOPPED; // active tracks are removed by threadLoop() if (mActiveTrack != track) { removeTrack_l(track); @@ -4087,7 +4878,7 @@ void AudioFlinger::RecordThread::dumpInternals(int fd, const Vector<String16>& a if (mActiveTrack != 0) { snprintf(buffer, SIZE, "In index: %d\n", mRsmpInIndex); result.append(buffer); - snprintf(buffer, SIZE, "In size: %d\n", mInputBytes); + snprintf(buffer, SIZE, "Buffer size: %u bytes\n", mBufferSize); result.append(buffer); snprintf(buffer, SIZE, "Resampling: %d\n", (mResampler != NULL)); result.append(buffer); @@ -4140,7 +4931,7 @@ status_t AudioFlinger::RecordThread::getNextBuffer(AudioBufferProvider::Buffer* int channelCount; if (framesReady == 0) { - mBytesRead = mInput->stream->read(mInput->stream, mRsmpInBuffer, mInputBytes); + mBytesRead = mInput->stream->read(mInput->stream, mRsmpInBuffer, mBufferSize); if (mBytesRead <= 0) { if ((mBytesRead < 0) && (mActiveTrack->mState == TrackBase::ACTIVE)) { ALOGE("RecordThread::getNextBuffer() Error reading audio input"); @@ -4196,8 +4987,12 @@ bool AudioFlinger::RecordThread::checkForNewParameters_l() reconfig = true; } if (param.getInt(String8(AudioParameter::keyFormat), value) == NO_ERROR) { - reqFormat = (audio_format_t) value; - reconfig = true; + if ((audio_format_t) value != AUDIO_FORMAT_PCM_16_BIT) { + status = BAD_VALUE; + } else { + reqFormat = (audio_format_t) value; + reconfig = true; + } } if (param.getInt(String8(AudioParameter::keyChannels), value) == NO_ERROR) { reqChannelCount = popcount(value); @@ -4289,16 +5084,13 @@ bool AudioFlinger::RecordThread::checkForNewParameters_l() String8 AudioFlinger::RecordThread::getParameters(const String8& keys) { - char *s; - String8 out_s8 = String8(); - Mutex::Autolock _l(mLock); if (initCheck() != NO_ERROR) { - return out_s8; + return String8(); } - s = mInput->stream->common.get_parameters(&mInput->stream->common, keys.string()); - out_s8 = String8(s); + char *s = mInput->stream->common.get_parameters(&mInput->stream->common, keys.string()); + const String8 out_s8(s); free(s); return out_s8; } @@ -4310,7 +5102,7 @@ void AudioFlinger::RecordThread::audioConfigChanged_l(int event, int param) { switch (event) { case AudioSystem::INPUT_OPENED: case AudioSystem::INPUT_CONFIG_CHANGED: - desc.channels = mChannelMask; + desc.channelMask = mChannelMask; desc.samplingRate = mSampleRate; desc.format = mFormat; desc.frameCount = mFrameCount; @@ -4336,12 +5128,14 @@ void AudioFlinger::RecordThread::readInputParameters() mSampleRate = mInput->stream->common.get_sample_rate(&mInput->stream->common); mChannelMask = mInput->stream->common.get_channels(&mInput->stream->common); - mChannelCount = (uint16_t)popcount(mChannelMask); + mChannelCount = popcount(mChannelMask); mFormat = mInput->stream->common.get_format(&mInput->stream->common); + if (mFormat != AUDIO_FORMAT_PCM_16_BIT) { + ALOGE("HAL format %d not supported; must be AUDIO_FORMAT_PCM_16_BIT", mFormat); + } mFrameSize = audio_stream_frame_size(&mInput->stream->common); - mInputBytes = mInput->stream->common.get_buffer_size(&mInput->stream->common); - mFrameCount = mInputBytes / mFrameSize; - mNormalFrameCount = mFrameCount; // not used by record, but used by input effects + mBufferSize = mInput->stream->common.get_buffer_size(&mInput->stream->common); + mFrameCount = mBufferSize / mFrameSize; mRsmpInBuffer = new int16_t[mFrameCount * mChannelCount]; if (mSampleRate != mReqSampleRate && mChannelCount <= FCC_2 && mReqChannelCount <= FCC_2) @@ -4357,7 +5151,7 @@ void AudioFlinger::RecordThread::readInputParameters() mResampler = AudioResampler::create(16, channelCount, mReqSampleRate); mResampler->setSampleRate(mSampleRate); mResampler->setVolume(AudioMixer::UNITY_GAIN, AudioMixer::UNITY_GAIN); - mRsmpOutBuffer = new int32_t[mFrameCount * 2]; + mRsmpOutBuffer = new int32_t[mFrameCount * FCC_2]; // optmization: if mono to mono, alter input frame count as if we were inputing // stereo samples diff --git a/services/audioflinger/Threads.h b/services/audioflinger/Threads.h index 7de6872..802b784 100644 --- a/services/audioflinger/Threads.h +++ b/services/audioflinger/Threads.h @@ -28,7 +28,8 @@ public: MIXER, // Thread class is MixerThread DIRECT, // Thread class is DirectOutputThread DUPLICATING, // Thread class is DuplicatingThread - RECORD // Thread class is RecordThread + RECORD, // Thread class is RecordThread + OFFLOAD // Thread class is OffloadThread }; ThreadBase(const sp<AudioFlinger>& audioFlinger, audio_io_handle_t id, @@ -125,10 +126,9 @@ public: audio_channel_mask_t channelMask() const { return mChannelMask; } audio_format_t format() const { return mFormat; } // Called by AudioFlinger::frameCount(audio_io_handle_t output) and effects, - // and returns the normal mix buffer's frame count. - size_t frameCount() const { return mNormalFrameCount; } - // Return's the HAL's frame count i.e. fast mixer buffer size. - size_t frameCountHAL() const { return mFrameCount; } + // and returns the [normal mix] buffer's frame count. + virtual size_t frameCount() const = 0; + size_t frameSize() const { return mFrameSize; } // Should be "virtual status_t requestExitAndWait()" and override same // method in Thread, but Thread::requestExitAndWait() is not yet virtual. @@ -184,6 +184,8 @@ public: void lockEffectChains_l(Vector< sp<EffectChain> >& effectChains); // unlock effect chains after process void unlockEffectChains(const Vector< sp<EffectChain> >& effectChains); + // get a copy of mEffectChains vector + Vector< sp<EffectChain> > getEffectChains_l() const { return mEffectChains; }; // set audio mode to all effect chains void setMode(audio_mode_t mode); // get effect module with corresponding ID on specified audio session @@ -235,8 +237,8 @@ protected: effect_uuid_t mType; // effect type UUID }; - void acquireWakeLock(); - void acquireWakeLock_l(); + void acquireWakeLock(int uid = -1); + void acquireWakeLock_l(int uid = -1); void releaseWakeLock(); void releaseWakeLock_l(); void setEffectSuspended_l(const effect_uuid_t *type, @@ -249,6 +251,8 @@ protected: // check if some effects must be suspended when an effect chain is added void checkSuspendOnAddEffectChain_l(const sp<EffectChain>& chain); + String16 getWakeLockTag(); + virtual void preExit() { } friend class AudioFlinger; // for mEffectChains @@ -259,11 +263,13 @@ protected: Condition mWaitWorkCV; const sp<AudioFlinger> mAudioFlinger; + + // updated by PlaybackThread::readOutputParameters() or + // RecordThread::readInputParameters() uint32_t mSampleRate; size_t mFrameCount; // output HAL, direct output, record - size_t mNormalFrameCount; // normal mixer and effects audio_channel_mask_t mChannelMask; - uint16_t mChannelCount; + uint32_t mChannelCount; size_t mFrameSize; audio_format_t mFormat; @@ -290,6 +296,7 @@ protected: Vector<String8> mNewParameters; status_t mParamStatus; + // vector owns each ConfigEvent *, so must delete after removing Vector<ConfigEvent *> mConfigEvents; // These fields are written and read by thread itself without lock or barrier, @@ -328,11 +335,19 @@ public: enum mixer_state { MIXER_IDLE, // no active tracks MIXER_TRACKS_ENABLED, // at least one active track, but no track has any data ready - MIXER_TRACKS_READY // at least one active track, and at least one track has data + MIXER_TRACKS_READY, // at least one active track, and at least one track has data + MIXER_DRAIN_TRACK, // drain currently playing track + MIXER_DRAIN_ALL, // fully drain the hardware // standby mode does not have an enum value // suspend by audio policy manager is orthogonal to mixer state }; + // retry count before removing active track in case of underrun on offloaded thread: + // we need to make sure that AudioTrack client has enough time to send large buffers +//FIXME may be more appropriate if expressed in time units. Need to revise how underrun is handled + // for offloaded tracks + static const int8_t kMaxTrackRetriesOffload = 20; + PlaybackThread(const sp<AudioFlinger>& audioFlinger, AudioStreamOut* output, audio_io_handle_t id, audio_devices_t device, type_t type); virtual ~PlaybackThread(); @@ -350,8 +365,10 @@ protected: // Code snippets that were lifted up out of threadLoop() virtual void threadLoop_mix() = 0; virtual void threadLoop_sleepTime() = 0; - virtual void threadLoop_write(); + virtual ssize_t threadLoop_write(); + virtual void threadLoop_drain(); virtual void threadLoop_standby(); + virtual void threadLoop_exit(); virtual void threadLoop_removeTracks(const Vector< sp<Track> >& tracksToRemove); // prepareTracks_l reads and writes mActiveTracks, and returns @@ -359,6 +376,19 @@ protected: // is responsible for clearing or destroying this Vector later on, when it // is safe to do so. That will drop the final ref count and destroy the tracks. virtual mixer_state prepareTracks_l(Vector< sp<Track> > *tracksToRemove) = 0; + void removeTracks_l(const Vector< sp<Track> >& tracksToRemove); + + void writeCallback(); + void resetWriteBlocked(uint32_t sequence); + void drainCallback(); + void resetDraining(uint32_t sequence); + + static int asyncCallback(stream_callback_event_t event, void *param, void *cookie); + + virtual bool waitingAsyncCallback(); + virtual bool waitingAsyncCallback_l(); + virtual bool shouldStandby_l(); + // ThreadBase virtuals virtual void preExit(); @@ -429,11 +459,23 @@ public: virtual status_t setSyncEvent(const sp<SyncEvent>& event); virtual bool isValidSyncEvent(const sp<SyncEvent>& event) const; + + // called with AudioFlinger lock held void invalidateTracks(audio_stream_type_t streamType); + virtual size_t frameCount() const { return mNormalFrameCount; } + + // Return's the HAL's frame count i.e. fast mixer buffer size. + size_t frameCountHAL() const { return mFrameCount; } + + status_t getTimestamp_l(AudioTimestamp& timestamp); protected: - int16_t* mMixBuffer; + // updated by readOutputParameters() + size_t mNormalFrameCount; // normal mixer and effects + + int16_t* mMixBuffer; // frame size aligned mix buffer + int8_t* mAllocMixBuffer; // mixer buffer allocation address // suspend count, > 0 means suspended. While suspended, the thread continues to pull from // tracks and mix, but doesn't write to HAL. A2DP and SCO HAL implementations can't handle @@ -486,8 +528,9 @@ private: PlaybackThread& operator = (const PlaybackThread&); status_t addTrack_l(const sp<Track>& track); - void destroyTrack_l(const sp<Track>& track); + bool destroyTrack_l(const sp<Track>& track); void removeTrack_l(const sp<Track>& track); + void broadcast_l(); void readOutputParameters(); @@ -535,6 +578,27 @@ private: // DUPLICATING only uint32_t writeFrames; + size_t mBytesRemaining; + size_t mCurrentWriteLength; + bool mUseAsyncWrite; + // mWriteAckSequence contains current write sequence on bits 31-1. The write sequence is + // incremented each time a write(), a flush() or a standby() occurs. + // Bit 0 is set when a write blocks and indicates a callback is expected. + // Bit 0 is reset by the async callback thread calling resetWriteBlocked(). Out of sequence + // callbacks are ignored. + uint32_t mWriteAckSequence; + // mDrainSequence contains current drain sequence on bits 31-1. The drain sequence is + // incremented each time a drain is requested or a flush() or standby() occurs. + // Bit 0 is set when the drain() command is called at the HAL and indicates a callback is + // expected. + // Bit 0 is reset by the async callback thread calling resetDraining(). Out of sequence + // callbacks are ignored. + uint32_t mDrainSequence; + // A condition that must be evaluated by prepareTrack_l() has changed and we must not wait + // for async write callback in the thread loop before evaluating it + bool mSignalPending; + sp<AsyncCallbackThread> mCallbackThread; + private: // The HAL output sink is treated as non-blocking, but current implementation is blocking sp<NBAIO_Sink> mOutputSink; @@ -558,7 +622,18 @@ public: protected: // accessed by both binder threads and within threadLoop(), lock on mutex needed unsigned mFastTrackAvailMask; // bit i set if fast track [i] is available + virtual void flushOutput_l(); +private: + // timestamp latch: + // D input is written by threadLoop_write while mutex is unlocked, and read while locked + // Q output is written while locked, and read while locked + struct { + AudioTimestamp mTimestamp; + uint32_t mUnpresentedFrames; + } mLatchD, mLatchQ; + bool mLatchDValid; // true means mLatchD is valid, and clock it into latch at next opportunity + bool mLatchQValid; // true means mLatchQ is valid }; class MixerThread : public PlaybackThread { @@ -584,7 +659,7 @@ protected: virtual void cacheParameters_l(); // threadLoop snippets - virtual void threadLoop_write(); + virtual ssize_t threadLoop_write(); virtual void threadLoop_standby(); virtual void threadLoop_mix(); virtual void threadLoop_sleepTime(); @@ -641,17 +716,81 @@ protected: virtual void threadLoop_mix(); virtual void threadLoop_sleepTime(); -private: // volumes last sent to audio HAL with stream->set_volume() float mLeftVolFloat; float mRightVolFloat; + DirectOutputThread(const sp<AudioFlinger>& audioFlinger, AudioStreamOut* output, + audio_io_handle_t id, uint32_t device, ThreadBase::type_t type); + void processVolume_l(Track *track, bool lastTrack); + // prepareTracks_l() tells threadLoop_mix() the name of the single active track sp<Track> mActiveTrack; public: virtual bool hasFastMixer() const { return false; } }; +class OffloadThread : public DirectOutputThread { +public: + + OffloadThread(const sp<AudioFlinger>& audioFlinger, AudioStreamOut* output, + audio_io_handle_t id, uint32_t device); + virtual ~OffloadThread(); + +protected: + // threadLoop snippets + virtual mixer_state prepareTracks_l(Vector< sp<Track> > *tracksToRemove); + virtual void threadLoop_exit(); + virtual void flushOutput_l(); + + virtual bool waitingAsyncCallback(); + virtual bool waitingAsyncCallback_l(); + virtual bool shouldStandby_l(); + +private: + void flushHw_l(); + +private: + bool mHwPaused; + bool mFlushPending; + size_t mPausedWriteLength; // length in bytes of write interrupted by pause + size_t mPausedBytesRemaining; // bytes still waiting in mixbuffer after resume + sp<Track> mPreviousTrack; // used to detect track switch +}; + +class AsyncCallbackThread : public Thread { +public: + + AsyncCallbackThread(const wp<PlaybackThread>& playbackThread); + + virtual ~AsyncCallbackThread(); + + // Thread virtuals + virtual bool threadLoop(); + + // RefBase + virtual void onFirstRef(); + + void exit(); + void setWriteBlocked(uint32_t sequence); + void resetWriteBlocked(); + void setDraining(uint32_t sequence); + void resetDraining(); + +private: + const wp<PlaybackThread> mPlaybackThread; + // mWriteAckSequence corresponds to the last write sequence passed by the offload thread via + // setWriteBlocked(). The sequence is shifted one bit to the left and the lsb is used + // to indicate that the callback has been received via resetWriteBlocked() + uint32_t mWriteAckSequence; + // mDrainSequence corresponds to the last drain sequence passed by the offload thread via + // setDraining(). The sequence is shifted one bit to the left and the lsb is used + // to indicate that the callback has been received via resetDraining() + uint32_t mDrainSequence; + Condition mWaitWorkCV; + Mutex mLock; +}; + class DuplicatingThread : public MixerThread { public: DuplicatingThread(const sp<AudioFlinger>& audioFlinger, MixerThread* mainThread, @@ -671,7 +810,7 @@ protected: // threadLoop snippets virtual void threadLoop_mix(); virtual void threadLoop_sleepTime(); - virtual void threadLoop_write(); + virtual ssize_t threadLoop_write(); virtual void threadLoop_standby(); virtual void cacheParameters_l(); @@ -734,7 +873,7 @@ public: audio_channel_mask_t channelMask, size_t frameCount, int sessionId, - IAudioFlinger::track_flags_t flags, + IAudioFlinger::track_flags_t *flags, pid_t tid, status_t *status); @@ -744,7 +883,7 @@ public: // ask the thread to stop the specified track, and // return true if the caller should then do it's part of the stopping process - bool stop_l(RecordTrack* recordTrack); + bool stop(RecordTrack* recordTrack); void dump(int fd, const Vector<String16>& args); AudioStreamIn* clearInput(); @@ -775,6 +914,9 @@ public: static void syncStartEventCallback(const wp<SyncEvent>& event); void handleSyncStartEvent(const sp<SyncEvent>& event); + virtual size_t frameCount() const { return mFrameCount; } + bool hasFastRecorder() const { return false; } + private: void clearSyncStartEvent(); @@ -790,11 +932,14 @@ private: // is used together with mStartStopCond to indicate start()/stop() progress sp<RecordTrack> mActiveTrack; Condition mStartStopCond; + + // updated by RecordThread::readInputParameters() AudioResampler *mResampler; + // interleaved stereo pairs of fixed-point signed Q19.12 int32_t *mRsmpOutBuffer; - int16_t *mRsmpInBuffer; + int16_t *mRsmpInBuffer; // [mFrameCount * mChannelCount] size_t mRsmpInIndex; - size_t mInputBytes; + size_t mBufferSize; // stream buffer size for read() const uint32_t mReqChannelCount; const uint32_t mReqSampleRate; ssize_t mBytesRead; @@ -808,4 +953,5 @@ private: // For dumpsys const sp<NBAIO_Sink> mTeeSink; + int mClientUid; }; diff --git a/services/audioflinger/TrackBase.h b/services/audioflinger/TrackBase.h index fac7071..523e4b2 100644 --- a/services/audioflinger/TrackBase.h +++ b/services/audioflinger/TrackBase.h @@ -25,10 +25,10 @@ class TrackBase : public ExtendedAudioBufferProvider, public RefBase { public: enum track_state { IDLE, - TERMINATED, FLUSHED, STOPPED, - // next 2 states are currently used for fast tracks only + // next 2 states are currently used for fast tracks + // and offloaded tracks only STOPPING_1, // waiting for first underrun STOPPING_2, // waiting for presentation complete RESUMING, @@ -74,7 +74,7 @@ protected: audio_channel_mask_t channelMask() const { return mChannelMask; } - uint32_t sampleRate() const; // FIXME inline after cblk sr moved + virtual uint32_t sampleRate() const { return mSampleRate; } // Return a pointer to the start of a contiguous slice of the track buffer. // Parameter 'offset' is the requested start position, expressed in @@ -89,7 +89,7 @@ protected: return (mState == STOPPED || mState == FLUSHED); } - // for fast tracks only + // for fast tracks and offloaded tracks only bool isStopping() const { return mState == STOPPING_1 || mState == STOPPING_2; } @@ -101,11 +101,12 @@ protected: } bool isTerminated() const { - return mState == TERMINATED; + return mTerminated; } - bool step(); // mStepCount is an implicit input - void reset(); + void terminate() { + mTerminated = true; + } bool isOut() const { return mIsOut; } // true for Track and TimedTrack, false for RecordTrack, @@ -117,24 +118,19 @@ protected: audio_track_cblk_t* mCblk; void* mBuffer; // start of track buffer, typically in shared memory // except for OutputTrack when it is in local memory - void* mBufferEnd; // &mBuffer[mFrameCount * frameSize], where frameSize - // is based on mChannelCount and 16-bit samples - uint32_t mStepCount; // saves AudioBufferProvider::Buffer::frameCount as of - // time of releaseBuffer() for later use by step() // we don't really need a lock for these track_state mState; const uint32_t mSampleRate; // initial sample rate only; for tracks which // support dynamic rates, the current value is in control block const audio_format_t mFormat; const audio_channel_mask_t mChannelMask; - const uint8_t mChannelCount; + const uint32_t mChannelCount; const size_t mFrameSize; // AudioFlinger's view of frame size in shared memory, // where for AudioTrack (but not AudioRecord), // 8-bit PCM samples are stored as 16-bit const size_t mFrameCount;// size of track buffer given at createTrack() or // openRecord(), and then adjusted as needed - bool mStepServerFailed; const int mSessionId; Vector < sp<SyncEvent> >mSyncEvents; const bool mIsOut; @@ -142,4 +138,5 @@ protected: const int mId; sp<NBAIO_Sink> mTeeSink; sp<NBAIO_Source> mTeeSource; + bool mTerminated; }; diff --git a/services/audioflinger/Tracks.cpp b/services/audioflinger/Tracks.cpp index 5ac3129..9c6e724 100644 --- a/services/audioflinger/Tracks.cpp +++ b/services/audioflinger/Tracks.cpp @@ -19,8 +19,8 @@ #define LOG_TAG "AudioFlinger" //#define LOG_NDEBUG 0 +#include "Configuration.h" #include <math.h> -#include <cutils/compiler.h> #include <utils/Log.h> #include <private/media/AudioTrackShared.h> @@ -74,8 +74,6 @@ AudioFlinger::ThreadBase::TrackBase::TrackBase( mClient(client), mCblk(NULL), // mBuffer - // mBufferEnd - mStepCount(0), mState(IDLE), mSampleRate(sampleRate), mFormat(format), @@ -84,11 +82,11 @@ AudioFlinger::ThreadBase::TrackBase::TrackBase( mFrameSize(audio_is_linear_pcm(format) ? mChannelCount * audio_bytes_per_sample(format) : sizeof(int8_t)), mFrameCount(frameCount), - mStepServerFailed(false), mSessionId(sessionId), mIsOut(isOut), mServerProxy(NULL), - mId(android_atomic_inc(&nextTrackId)) + mId(android_atomic_inc(&nextTrackId)), + mTerminated(false) { // client == 0 implies sharedBuffer == 0 ALOG_ASSERT(!(client == 0 && sharedBuffer != 0)); @@ -98,7 +96,7 @@ AudioFlinger::ThreadBase::TrackBase::TrackBase( // ALOGD("Creating track with %d buffers @ %d bytes", bufferCount, bufferSize); size_t size = sizeof(audio_track_cblk_t); - size_t bufferSize = frameCount * mFrameSize; + size_t bufferSize = (sharedBuffer == 0 ? roundup(frameCount) : frameCount) * mFrameSize; if (sharedBuffer == 0) { size += bufferSize; } @@ -124,22 +122,15 @@ AudioFlinger::ThreadBase::TrackBase::TrackBase( new(mCblk) audio_track_cblk_t(); // clear all buffers mCblk->frameCount_ = frameCount; -// uncomment the following lines to quickly test 32-bit wraparound -// mCblk->user = 0xffff0000; -// mCblk->server = 0xffff0000; -// mCblk->userBase = 0xffff0000; -// mCblk->serverBase = 0xffff0000; if (sharedBuffer == 0) { mBuffer = (char*)mCblk + sizeof(audio_track_cblk_t); memset(mBuffer, 0, bufferSize); - // Force underrun condition to avoid false underrun callback until first data is - // written to buffer (other flags are cleared) - mCblk->flags = CBLK_UNDERRUN; } else { mBuffer = sharedBuffer->pointer(); +#if 0 + mCblk->mFlags = CBLK_FORCEREADY; // FIXME hack, need to fix the track ready logic +#endif } - mBufferEnd = (uint8_t *)mBuffer + bufferSize; - mServerProxy = new ServerProxy(mCblk, mBuffer, frameCount, mFrameSize, isOut); #ifdef TEE_SINK if (mTeeSinkTrackEnabled) { @@ -199,51 +190,12 @@ void AudioFlinger::ThreadBase::TrackBase::releaseBuffer(AudioBufferProvider::Buf } #endif - buffer->raw = NULL; - mStepCount = buffer->frameCount; - // FIXME See note at getNextBuffer() - (void) step(); // ignore return value of step() + ServerProxy::Buffer buf; + buf.mFrameCount = buffer->frameCount; + buf.mRaw = buffer->raw; buffer->frameCount = 0; -} - -bool AudioFlinger::ThreadBase::TrackBase::step() { - bool result = mServerProxy->step(mStepCount); - if (!result) { - ALOGV("stepServer failed acquiring cblk mutex"); - mStepServerFailed = true; - } - return result; -} - -void AudioFlinger::ThreadBase::TrackBase::reset() { - audio_track_cblk_t* cblk = this->cblk(); - - cblk->user = 0; - cblk->server = 0; - cblk->userBase = 0; - cblk->serverBase = 0; - mStepServerFailed = false; - ALOGV("TrackBase::reset"); -} - -uint32_t AudioFlinger::ThreadBase::TrackBase::sampleRate() const { - return mServerProxy->getSampleRate(); -} - -void* AudioFlinger::ThreadBase::TrackBase::getBuffer(uint32_t offset, uint32_t frames) const { - audio_track_cblk_t* cblk = this->cblk(); - int8_t *bufferStart = (int8_t *)mBuffer + (offset-cblk->serverBase) * mFrameSize; - int8_t *bufferEnd = bufferStart + frames * mFrameSize; - - // Check validity of returned pointer in case the track control block would have been corrupted. - ALOG_ASSERT(!(bufferStart < mBuffer || bufferStart > bufferEnd || bufferEnd > mBufferEnd), - "TrackBase::getBuffer buffer out of range:\n" - " start: %p, end %p , mBuffer %p mBufferEnd %p\n" - " server %u, serverBase %u, user %u, userBase %u, frameSize %u", - bufferStart, bufferEnd, mBuffer, mBufferEnd, - cblk->server, cblk->serverBase, cblk->user, cblk->userBase, mFrameSize); - - return bufferStart; + buffer->raw = NULL; + mServerProxy->releaseBuffer(&buf); } status_t AudioFlinger::ThreadBase::TrackBase::setSyncEvent(const sp<SyncEvent>& event) @@ -327,6 +279,21 @@ status_t AudioFlinger::TrackHandle::setMediaTimeTransform( xform, static_cast<TimedAudioTrack::TargetTimeline>(target)); } +status_t AudioFlinger::TrackHandle::setParameters(const String8& keyValuePairs) { + return mTrack->setParameters(keyValuePairs); +} + +status_t AudioFlinger::TrackHandle::getTimestamp(AudioTimestamp& timestamp) +{ + return mTrack->getTimestamp(timestamp); +} + + +void AudioFlinger::TrackHandle::signal() +{ + return mTrack->signal(); +} + status_t AudioFlinger::TrackHandle::onTransact( uint32_t code, const Parcel& data, Parcel* reply, uint32_t flags) { @@ -360,20 +327,29 @@ AudioFlinger::PlaybackThread::Track::Track( mPresentationCompleteFrames(0), mFlags(flags), mFastIndex(-1), - mUnderrunCount(0), mCachedVolume(1.0), - mIsInvalid(false) + mIsInvalid(false), + mAudioTrackServerProxy(NULL), + mResumeToStopping(false) { if (mCblk != NULL) { + if (sharedBuffer == 0) { + mAudioTrackServerProxy = new AudioTrackServerProxy(mCblk, mBuffer, frameCount, + mFrameSize); + } else { + mAudioTrackServerProxy = new StaticAudioTrackServerProxy(mCblk, mBuffer, frameCount, + mFrameSize); + } + mServerProxy = mAudioTrackServerProxy; // to avoid leaking a track name, do not allocate one unless there is an mCblk mName = thread->getTrackName_l(channelMask, sessionId); - mCblk->mName = mName; if (mName < 0) { ALOGE("no more track names available"); return; } // only allocate a fast track index if we were able to allocate a normal track name if (flags & IAudioFlinger::TRACK_FAST) { + mAudioTrackServerProxy->framesReadyIsCalledByMultipleThreads(); ALOG_ASSERT(thread->mFastTrackAvailMask != 0); int i = __builtin_ctz(thread->mFastTrackAvailMask); ALOG_ASSERT(0 < i && i < (int)FastMixerState::kMaxFastTracks); @@ -382,7 +358,6 @@ AudioFlinger::PlaybackThread::Track::Track( // this means we are potentially denying other more important fast tracks from // being created. It would be better to allocate the index dynamically. mFastIndex = i; - mCblk->mName = i; // Read the initial underruns because this field is never cleared by the fast mixer mObservedUnderruns = thread->getFastTrackUnderruns(i); thread->mFastTrackAvailMask &= ~(1 << i); @@ -395,6 +370,16 @@ AudioFlinger::PlaybackThread::Track::Track( AudioFlinger::PlaybackThread::Track::~Track() { ALOGV("PlaybackThread::Track destructor"); + + // The destructor would clear mSharedBuffer, + // but it will not push the decremented reference count, + // leaving the client's IMemory dangling indefinitely. + // This prevents that leak. + if (mSharedBuffer != 0) { + mSharedBuffer.clear(); + // flush the binder command buffer + IPCThreadState::self()->flushCommands(); + } } void AudioFlinger::PlaybackThread::Track::destroy() @@ -411,33 +396,25 @@ void AudioFlinger::PlaybackThread::Track::destroy() { // scope for mLock sp<ThreadBase> thread = mThread.promote(); if (thread != 0) { - if (!isOutputTrack()) { - if (mState == ACTIVE || mState == RESUMING) { - AudioSystem::stopOutput(thread->id(), mStreamType, mSessionId); - -#ifdef ADD_BATTERY_DATA - // to track the speaker usage - addBatteryData(IMediaPlayerService::kBatteryDataAudioFlingerStop); -#endif - } - AudioSystem::releaseOutput(thread->id()); - } Mutex::Autolock _l(thread->mLock); PlaybackThread *playbackThread = (PlaybackThread *)thread.get(); - playbackThread->destroyTrack_l(this); + bool wasActive = playbackThread->destroyTrack_l(this); + if (!isOutputTrack() && !wasActive) { + AudioSystem::releaseOutput(thread->id()); + } } } } /*static*/ void AudioFlinger::PlaybackThread::Track::appendDumpHeader(String8& result) { - result.append(" Name Client Type Fmt Chn mask Session StpCnt fCount S F SRate " - "L dB R dB Server User Main buf Aux Buf Flags Underruns\n"); + result.append(" Name Client Type Fmt Chn mask Session fCount S F SRate " + "L dB R dB Server Main buf Aux Buf Flags UndFrmCnt\n"); } void AudioFlinger::PlaybackThread::Track::dump(char* buffer, size_t size) { - uint32_t vlr = mServerProxy->getVolumeLR(); + uint32_t vlr = mAudioTrackServerProxy->getVolumeLR(); if (isFastTrack()) { sprintf(buffer, " F %2d", mFastIndex); } else { @@ -445,40 +422,41 @@ void AudioFlinger::PlaybackThread::Track::dump(char* buffer, size_t size) } track_state state = mState; char stateChar; - switch (state) { - case IDLE: - stateChar = 'I'; - break; - case TERMINATED: + if (isTerminated()) { stateChar = 'T'; - break; - case STOPPING_1: - stateChar = 's'; - break; - case STOPPING_2: - stateChar = '5'; - break; - case STOPPED: - stateChar = 'S'; - break; - case RESUMING: - stateChar = 'R'; - break; - case ACTIVE: - stateChar = 'A'; - break; - case PAUSING: - stateChar = 'p'; - break; - case PAUSED: - stateChar = 'P'; - break; - case FLUSHED: - stateChar = 'F'; - break; - default: - stateChar = '?'; - break; + } else { + switch (state) { + case IDLE: + stateChar = 'I'; + break; + case STOPPING_1: + stateChar = 's'; + break; + case STOPPING_2: + stateChar = '5'; + break; + case STOPPED: + stateChar = 'S'; + break; + case RESUMING: + stateChar = 'R'; + break; + case ACTIVE: + stateChar = 'A'; + break; + case PAUSING: + stateChar = 'p'; + break; + case PAUSED: + stateChar = 'P'; + break; + case FLUSHED: + stateChar = 'F'; + break; + default: + stateChar = '?'; + break; + } } char nowInUnderrun; switch (mObservedUnderruns.mBitFields.mMostRecent) { @@ -495,77 +473,50 @@ void AudioFlinger::PlaybackThread::Track::dump(char* buffer, size_t size) nowInUnderrun = '?'; break; } - snprintf(&buffer[7], size-7, " %6d %4u %3u 0x%08x %7u %6u %6u %1c %1d %5u %5.2g %5.2g " - "0x%08x 0x%08x 0x%08x 0x%08x %#5x %9u%c\n", + snprintf(&buffer[7], size-7, " %6u %4u %08X %08X %7u %6u %1c %1d %5u %5.2g %5.2g " + "%08X %08X %08X 0x%03X %9u%c\n", (mClient == 0) ? getpid_cached : mClient->pid(), mStreamType, mFormat, mChannelMask, mSessionId, - mStepCount, mFrameCount, stateChar, mFillingUpStatus, - mServerProxy->getSampleRate(), + mAudioTrackServerProxy->getSampleRate(), 20.0 * log10((vlr & 0xFFFF) / 4096.0), 20.0 * log10((vlr >> 16) / 4096.0), - mCblk->server, - mCblk->user, + mCblk->mServer, (int)mMainBuffer, (int)mAuxBuffer, - mCblk->flags, - mUnderrunCount, + mCblk->mFlags, + mAudioTrackServerProxy->getUnderrunFrames(), nowInUnderrun); } +uint32_t AudioFlinger::PlaybackThread::Track::sampleRate() const { + return mAudioTrackServerProxy->getSampleRate(); +} + // AudioBufferProvider interface status_t AudioFlinger::PlaybackThread::Track::getNextBuffer( AudioBufferProvider::Buffer* buffer, int64_t pts) { - audio_track_cblk_t* cblk = this->cblk(); - uint32_t framesReady; - uint32_t framesReq = buffer->frameCount; - - // Check if last stepServer failed, try to step now - if (mStepServerFailed) { - // FIXME When called by fast mixer, this takes a mutex with tryLock(). - // Since the fast mixer is higher priority than client callback thread, - // it does not result in priority inversion for client. - // But a non-blocking solution would be preferable to avoid - // fast mixer being unable to tryLock(), and - // to avoid the extra context switches if the client wakes up, - // discovers the mutex is locked, then has to wait for fast mixer to unlock. - if (!step()) goto getNextBuffer_exit; - ALOGV("stepServer recovered"); - mStepServerFailed = false; + ServerProxy::Buffer buf; + size_t desiredFrames = buffer->frameCount; + buf.mFrameCount = desiredFrames; + status_t status = mServerProxy->obtainBuffer(&buf); + buffer->frameCount = buf.mFrameCount; + buffer->raw = buf.mRaw; + if (buf.mFrameCount == 0) { + mAudioTrackServerProxy->tallyUnderrunFrames(desiredFrames); } + return status; +} - // FIXME Same as above - framesReady = mServerProxy->framesReady(); - - if (CC_LIKELY(framesReady)) { - uint32_t s = cblk->server; - uint32_t bufferEnd = cblk->serverBase + mFrameCount; - - bufferEnd = (cblk->loopEnd < bufferEnd) ? cblk->loopEnd : bufferEnd; - if (framesReq > framesReady) { - framesReq = framesReady; - } - if (framesReq > bufferEnd - s) { - framesReq = bufferEnd - s; - } +// releaseBuffer() is not overridden - buffer->raw = getBuffer(s, framesReq); - buffer->frameCount = framesReq; - return NO_ERROR; - } - -getNextBuffer_exit: - buffer->raw = NULL; - buffer->frameCount = 0; - ALOGV("getNextBuffer() no more data for track %d on thread %p", mName, mThread.unsafe_get()); - return NOT_ENOUGH_DATA; -} +// ExtendedAudioBufferProvider interface // Note that framesReady() takes a mutex on the control block using tryLock(). // This could result in priority inversion if framesReady() is called by the normal mixer, @@ -576,7 +527,12 @@ getNextBuffer_exit: // the tryLock() could block for up to 1 ms, and a sequence of these could delay fast mixer. // FIXME Replace AudioTrackShared control block implementation by a non-blocking FIFO queue. size_t AudioFlinger::PlaybackThread::Track::framesReady() const { - return mServerProxy->framesReady(); + return mAudioTrackServerProxy->framesReady(); +} + +size_t AudioFlinger::PlaybackThread::Track::framesReleased() const +{ + return mAudioTrackServerProxy->framesReleased(); } // Don't call for fast tracks; the framesReady() could result in priority inversion @@ -586,9 +542,9 @@ bool AudioFlinger::PlaybackThread::Track::isReady() const { } if (framesReady() >= mFrameCount || - (mCblk->flags & CBLK_FORCEREADY)) { + (mCblk->mFlags & CBLK_FORCEREADY)) { mFillingUpStatus = FS_FILLED; - android_atomic_and(~CBLK_FORCEREADY, &mCblk->flags); + android_atomic_and(~CBLK_FORCEREADY, &mCblk->mFlags); return true; } return false; @@ -603,36 +559,47 @@ status_t AudioFlinger::PlaybackThread::Track::start(AudioSystem::sync_event_t ev sp<ThreadBase> thread = mThread.promote(); if (thread != 0) { - Mutex::Autolock _l(thread->mLock); + if (isOffloaded()) { + Mutex::Autolock _laf(thread->mAudioFlinger->mLock); + Mutex::Autolock _lth(thread->mLock); + sp<EffectChain> ec = thread->getEffectChain_l(mSessionId); + if (thread->mAudioFlinger->isNonOffloadableGlobalEffectEnabled_l() || + (ec != 0 && ec->isNonOffloadableEnabled())) { + invalidate(); + return PERMISSION_DENIED; + } + } + Mutex::Autolock _lth(thread->mLock); track_state state = mState; // here the track could be either new, or restarted // in both cases "unstop" the track + if (state == PAUSED) { - mState = TrackBase::RESUMING; - ALOGV("PAUSED => RESUMING (%d) on thread %p", mName, this); + if (mResumeToStopping) { + // happened we need to resume to STOPPING_1 + mState = TrackBase::STOPPING_1; + ALOGV("PAUSED => STOPPING_1 (%d) on thread %p", mName, this); + } else { + mState = TrackBase::RESUMING; + ALOGV("PAUSED => RESUMING (%d) on thread %p", mName, this); + } } else { mState = TrackBase::ACTIVE; ALOGV("? => ACTIVE (%d) on thread %p", mName, this); } - if (!isOutputTrack() && state != ACTIVE && state != RESUMING) { - thread->mLock.unlock(); - status = AudioSystem::startOutput(thread->id(), mStreamType, mSessionId); - thread->mLock.lock(); - -#ifdef ADD_BATTERY_DATA - // to track the speaker usage - if (status == NO_ERROR) { - addBatteryData(IMediaPlayerService::kBatteryDataAudioFlingerStart); + PlaybackThread *playbackThread = (PlaybackThread *)thread.get(); + status = playbackThread->addTrack_l(this); + if (status == INVALID_OPERATION || status == PERMISSION_DENIED) { + triggerEvents(AudioSystem::SYNC_EVENT_PRESENTATION_COMPLETE); + // restore previous state if start was rejected by policy manager + if (status == PERMISSION_DENIED) { + mState = state; } -#endif } - if (status == NO_ERROR) { - PlaybackThread *playbackThread = (PlaybackThread *)thread.get(); - playbackThread->addTrack_l(this); - } else { - mState = state; - triggerEvents(AudioSystem::SYNC_EVENT_PRESENTATION_COMPLETE); + // track was already in the active list, not a problem + if (status == ALREADY_EXISTS) { + status = NO_ERROR; } } else { status = BAD_VALUE; @@ -653,26 +620,18 @@ void AudioFlinger::PlaybackThread::Track::stop() if (playbackThread->mActiveTracks.indexOf(this) < 0) { reset(); mState = STOPPED; - } else if (!isFastTrack()) { + } else if (!isFastTrack() && !isOffloaded()) { mState = STOPPED; } else { - // prepareTracks_l() will set state to STOPPING_2 after next underrun, - // and then to STOPPED and reset() when presentation is complete + // For fast tracks prepareTracks_l() will set state to STOPPING_2 + // presentation is complete + // For an offloaded track this starts a drain and state will + // move to STOPPING_2 when drain completes and then STOPPED mState = STOPPING_1; } ALOGV("not stopping/stopped => stopping/stopped (%d) on thread %p", mName, playbackThread); } - if (!isOutputTrack() && (state == ACTIVE || state == RESUMING)) { - thread->mLock.unlock(); - AudioSystem::stopOutput(thread->id(), mStreamType, mSessionId); - thread->mLock.lock(); - -#ifdef ADD_BATTERY_DATA - // to track the speaker usage - addBatteryData(IMediaPlayerService::kBatteryDataAudioFlingerStop); -#endif - } } } @@ -682,19 +641,27 @@ void AudioFlinger::PlaybackThread::Track::pause() sp<ThreadBase> thread = mThread.promote(); if (thread != 0) { Mutex::Autolock _l(thread->mLock); - if (mState == ACTIVE || mState == RESUMING) { + PlaybackThread *playbackThread = (PlaybackThread *)thread.get(); + switch (mState) { + case STOPPING_1: + case STOPPING_2: + if (!isOffloaded()) { + /* nothing to do if track is not offloaded */ + break; + } + + // Offloaded track was draining, we need to carry on draining when resumed + mResumeToStopping = true; + // fall through... + case ACTIVE: + case RESUMING: mState = PAUSING; ALOGV("ACTIVE/RESUMING => PAUSING (%d) on thread %p", mName, thread.get()); - if (!isOutputTrack()) { - thread->mLock.unlock(); - AudioSystem::stopOutput(thread->id(), mStreamType, mSessionId); - thread->mLock.lock(); - -#ifdef ADD_BATTERY_DATA - // to track the speaker usage - addBatteryData(IMediaPlayerService::kBatteryDataAudioFlingerStop); -#endif - } + playbackThread->broadcast_l(); + break; + + default: + break; } } } @@ -705,21 +672,52 @@ void AudioFlinger::PlaybackThread::Track::flush() sp<ThreadBase> thread = mThread.promote(); if (thread != 0) { Mutex::Autolock _l(thread->mLock); - if (mState != STOPPING_1 && mState != STOPPING_2 && mState != STOPPED && mState != PAUSED && - mState != PAUSING && mState != IDLE && mState != FLUSHED) { - return; - } - // No point remaining in PAUSED state after a flush => go to - // FLUSHED state - mState = FLUSHED; - // do not reset the track if it is still in the process of being stopped or paused. - // this will be done by prepareTracks_l() when the track is stopped. - // prepareTracks_l() will see mState == FLUSHED, then - // remove from active track list, reset(), and trigger presentation complete PlaybackThread *playbackThread = (PlaybackThread *)thread.get(); - if (playbackThread->mActiveTracks.indexOf(this) < 0) { + + if (isOffloaded()) { + // If offloaded we allow flush during any state except terminated + // and keep the track active to avoid problems if user is seeking + // rapidly and underlying hardware has a significant delay handling + // a pause + if (isTerminated()) { + return; + } + + ALOGV("flush: offload flush"); reset(); + + if (mState == STOPPING_1 || mState == STOPPING_2) { + ALOGV("flushed in STOPPING_1 or 2 state, change state to ACTIVE"); + mState = ACTIVE; + } + + if (mState == ACTIVE) { + ALOGV("flush called in active state, resetting buffer time out retry count"); + mRetryCount = PlaybackThread::kMaxTrackRetriesOffload; + } + + mResumeToStopping = false; + } else { + if (mState != STOPPING_1 && mState != STOPPING_2 && mState != STOPPED && + mState != PAUSED && mState != PAUSING && mState != IDLE && mState != FLUSHED) { + return; + } + // No point remaining in PAUSED state after a flush => go to + // FLUSHED state + mState = FLUSHED; + // do not reset the track if it is still in the process of being stopped or paused. + // this will be done by prepareTracks_l() when the track is stopped. + // prepareTracks_l() will see mState == FLUSHED, then + // remove from active track list, reset(), and trigger presentation complete + if (playbackThread->mActiveTracks.indexOf(this) < 0) { + reset(); + } } + // Prevent flush being lost if the track is flushed and then resumed + // before mixer thread can run. This is important when offloading + // because the hardware buffer could hold a large amount of audio + playbackThread->flushOutput_l(); + playbackThread->broadcast_l(); } } @@ -728,11 +726,9 @@ void AudioFlinger::PlaybackThread::Track::reset() // Do not reset twice to avoid discarding data written just after a flush and before // the audioflinger thread detects the track is stopped. if (!mResetDone) { - TrackBase::reset(); // Force underrun condition to avoid false underrun callback until first data is // written to buffer - android_atomic_and(~CBLK_FORCEREADY, &mCblk->flags); - android_atomic_or(CBLK_UNDERRUN, &mCblk->flags); + android_atomic_and(~CBLK_FORCEREADY, &mCblk->mFlags); mFillingUpStatus = FS_FILLING; mResetDone = true; if (mState == FLUSHED) { @@ -741,6 +737,51 @@ void AudioFlinger::PlaybackThread::Track::reset() } } +status_t AudioFlinger::PlaybackThread::Track::setParameters(const String8& keyValuePairs) +{ + sp<ThreadBase> thread = mThread.promote(); + if (thread == 0) { + ALOGE("thread is dead"); + return FAILED_TRANSACTION; + } else if ((thread->type() == ThreadBase::DIRECT) || + (thread->type() == ThreadBase::OFFLOAD)) { + return thread->setParameters(keyValuePairs); + } else { + return PERMISSION_DENIED; + } +} + +status_t AudioFlinger::PlaybackThread::Track::getTimestamp(AudioTimestamp& timestamp) +{ + // Client should implement this using SSQ; the unpresented frame count in latch is irrelevant + if (isFastTrack()) { + return INVALID_OPERATION; + } + sp<ThreadBase> thread = mThread.promote(); + if (thread == 0) { + return INVALID_OPERATION; + } + Mutex::Autolock _l(thread->mLock); + PlaybackThread *playbackThread = (PlaybackThread *)thread.get(); + if (!isOffloaded()) { + if (!playbackThread->mLatchQValid) { + return INVALID_OPERATION; + } + uint32_t unpresentedFrames = + ((int64_t) playbackThread->mLatchQ.mUnpresentedFrames * mSampleRate) / + playbackThread->mSampleRate; + uint32_t framesWritten = mAudioTrackServerProxy->framesReleased(); + if (framesWritten < unpresentedFrames) { + return INVALID_OPERATION; + } + timestamp.mPosition = framesWritten - unpresentedFrames; + timestamp.mTime = playbackThread->mLatchQ.mTimestamp.mTime; + return NO_ERROR; + } + + return playbackThread->getTimestamp_l(timestamp); +} + status_t AudioFlinger::PlaybackThread::Track::attachAuxEffect(int EffectId) { status_t status = DEAD_OBJECT; @@ -766,7 +807,11 @@ status_t AudioFlinger::PlaybackThread::Track::attachAuxEffect(int EffectId) return INVALID_OPERATION; } srcThread->removeEffect_l(effect); - playbackThread->addEffect_l(effect); + status = playbackThread->addEffect_l(effect); + if (status != NO_ERROR) { + srcThread->addEffect_l(effect); + return INVALID_OPERATION; + } // removeEffect_l() has stopped the effect if it was active so it must be restarted if (effect->state() == EffectModule::ACTIVE || effect->state() == EffectModule::STOPPING) { @@ -802,15 +847,23 @@ bool AudioFlinger::PlaybackThread::Track::presentationComplete(size_t framesWrit // a track is considered presented when the total number of frames written to audio HAL // corresponds to the number of frames written when presentationComplete() is called for the // first time (mPresentationCompleteFrames == 0) plus the buffer filling status at that time. + // For an offloaded track the HAL+h/w delay is variable so a HAL drain() is used + // to detect when all frames have been played. In this case framesWritten isn't + // useful because it doesn't always reflect whether there is data in the h/w + // buffers, particularly if a track has been paused and resumed during draining + ALOGV("presentationComplete() mPresentationCompleteFrames %d framesWritten %d", + mPresentationCompleteFrames, framesWritten); if (mPresentationCompleteFrames == 0) { mPresentationCompleteFrames = framesWritten + audioHalFrames; ALOGV("presentationComplete() reset: mPresentationCompleteFrames %d audioHalFrames %d", mPresentationCompleteFrames, audioHalFrames); } - if (framesWritten >= mPresentationCompleteFrames) { + + if (framesWritten >= mPresentationCompleteFrames || isOffloaded()) { ALOGV("presentationComplete() session %d complete: framesWritten %d", mSessionId, framesWritten); triggerEvents(AudioSystem::SYNC_EVENT_PRESENTATION_COMPLETE); + mAudioTrackServerProxy->setStreamEndDone(); return true; } return false; @@ -833,7 +886,7 @@ uint32_t AudioFlinger::PlaybackThread::Track::getVolumeLR() { // called by FastMixer, so not allowed to take any locks, block, or do I/O including logs ALOG_ASSERT(isFastTrack() && (mCblk != NULL)); - uint32_t vlr = mServerProxy->getVolumeLR(); + uint32_t vlr = mAudioTrackServerProxy->getVolumeLR(); uint32_t vl = vlr & 0xFFFF; uint32_t vr = vlr >> 16; // track volumes come from shared memory, so can't be trusted and must be clamped @@ -856,7 +909,7 @@ uint32_t AudioFlinger::PlaybackThread::Track::getVolumeLR() status_t AudioFlinger::PlaybackThread::Track::setSyncEvent(const sp<SyncEvent>& event) { - if (mState == TERMINATED || mState == PAUSED || + if (isTerminated() || mState == PAUSED || ((framesReady() == 0) && ((mSharedBuffer != 0) || (mState == STOPPED)))) { ALOGW("Track::setSyncEvent() in invalid state %d on session %d %s mode, framesReady %d ", @@ -870,12 +923,25 @@ status_t AudioFlinger::PlaybackThread::Track::setSyncEvent(const sp<SyncEvent>& void AudioFlinger::PlaybackThread::Track::invalidate() { - // FIXME should use proxy - android_atomic_or(CBLK_INVALID, &mCblk->flags); - mCblk->cv.signal(); + // FIXME should use proxy, and needs work + audio_track_cblk_t* cblk = mCblk; + android_atomic_or(CBLK_INVALID, &cblk->mFlags); + android_atomic_release_store(0x40000000, &cblk->mFutex); + // client is not in server, so FUTEX_WAKE is needed instead of FUTEX_WAKE_PRIVATE + (void) __futex_syscall3(&cblk->mFutex, FUTEX_WAKE, INT_MAX); mIsInvalid = true; } +void AudioFlinger::PlaybackThread::Track::signal() +{ + sp<ThreadBase> thread = mThread.promote(); + if (thread != 0) { + PlaybackThread *t = (PlaybackThread *)thread.get(); + Mutex::Autolock _l(t->mLock); + t->broadcast_l(); + } +} + // ---------------------------------------------------------------------------- sp<AudioFlinger::PlaybackThread::TimedTrack> @@ -1185,10 +1251,12 @@ status_t AudioFlinger::PlaybackThread::TimedTrack::getNextBuffer( } } + uint32_t sr = sampleRate(); + // adjust the head buffer's PTS to reflect the portion of the head buffer // that has already been consumed int64_t effectivePTS = headLocalPTS + - ((head.position() / mFrameSize) * mLocalTimeFreq / sampleRate()); + ((head.position() / mFrameSize) * mLocalTimeFreq / sr); // Calculate the delta in samples between the head of the input buffer // queue and the start of the next output buffer that will be written. @@ -1220,7 +1288,7 @@ status_t AudioFlinger::PlaybackThread::TimedTrack::getNextBuffer( // the current output position is within this threshold, then we will // concatenate the next input samples to the previous output const int64_t kSampleContinuityThreshold = - (static_cast<int64_t>(sampleRate()) << 32) / 250; + (static_cast<int64_t>(sr) << 32) / 250; // if this is the first buffer of audio that we're emitting from this track // then it should be almost exactly on time. @@ -1409,15 +1477,17 @@ AudioFlinger::PlaybackThread::OutputTrack::OutputTrack( mOutBuffer.frameCount = 0; playbackThread->mTracks.add(this); ALOGV("OutputTrack constructor mCblk %p, mBuffer %p, " - "mCblk->frameCount_ %u, mChannelMask 0x%08x mBufferEnd %p", + "mCblk->frameCount_ %u, mChannelMask 0x%08x", mCblk, mBuffer, - mCblk->frameCount_, mChannelMask, mBufferEnd); + mCblk->frameCount_, mChannelMask); // since client and server are in the same process, // the buffer has the same virtual address on both sides mClientProxy = new AudioTrackClientProxy(mCblk, mBuffer, mFrameCount, mFrameSize); mClientProxy->setVolumeLR((uint32_t(uint16_t(0x1000)) << 16) | uint16_t(0x1000)); mClientProxy->setSendLevel(0.0); mClientProxy->setSampleRate(sampleRate); + mClientProxy = new AudioTrackClientProxy(mCblk, mBuffer, mFrameCount, mFrameSize, + true /*clientInServer*/); } else { ALOGW("Error creating output track on thread %p", playbackThread); } @@ -1477,7 +1547,7 @@ bool AudioFlinger::PlaybackThread::OutputTrack::write(int16_t* data, uint32_t fr memset(pInBuffer->raw, 0, startFrames * channelCount * sizeof(int16_t)); mBufferQueue.add(pInBuffer); } else { - ALOGW ("OutputTrack::write() %p no more buffers in queue", this); + ALOGW("OutputTrack::write() %p no more buffers in queue", this); } } } @@ -1498,9 +1568,10 @@ bool AudioFlinger::PlaybackThread::OutputTrack::write(int16_t* data, uint32_t fr if (mOutBuffer.frameCount == 0) { mOutBuffer.frameCount = pInBuffer->frameCount; nsecs_t startTime = systemTime(); - if (obtainBuffer(&mOutBuffer, waitTimeLeftMs) == (status_t)NO_MORE_BUFFERS) { - ALOGV ("OutputTrack::write() %p thread %p no more output buffers", this, - mThread.unsafe_get()); + status_t status = obtainBuffer(&mOutBuffer, waitTimeLeftMs); + if (status != NO_ERROR) { + ALOGV("OutputTrack::write() %p thread %p no more output buffers; status %d", this, + mThread.unsafe_get(), status); outputBufferFull = true; break; } @@ -1515,7 +1586,10 @@ bool AudioFlinger::PlaybackThread::OutputTrack::write(int16_t* data, uint32_t fr uint32_t outFrames = pInBuffer->frameCount > mOutBuffer.frameCount ? mOutBuffer.frameCount : pInBuffer->frameCount; memcpy(mOutBuffer.raw, pInBuffer->raw, outFrames * channelCount * sizeof(int16_t)); - mClientProxy->stepUser(outFrames); + Proxy::Buffer buf; + buf.mFrameCount = outFrames; + buf.mRaw = NULL; + mClientProxy->releaseBuffer(&buf); pInBuffer->frameCount -= outFrames; pInBuffer->i16 += outFrames * channelCount; mOutBuffer.frameCount -= outFrames; @@ -1559,8 +1633,10 @@ bool AudioFlinger::PlaybackThread::OutputTrack::write(int16_t* data, uint32_t fr // If no more buffers are pending, fill output track buffer to make sure it is started // by output mixer. if (frames == 0 && mBufferQueue.size() == 0) { - if (mCblk->user < mFrameCount) { - frames = mFrameCount - mCblk->user; + // FIXME borken, replace by getting framesReady() from proxy + size_t user = 0; // was mCblk->user + if (user < mFrameCount) { + frames = mFrameCount - user; pInBuffer = new Buffer; pInBuffer->mBuffer = new int16_t[frames * channelCount]; pInBuffer->frameCount = frames; @@ -1578,46 +1654,17 @@ bool AudioFlinger::PlaybackThread::OutputTrack::write(int16_t* data, uint32_t fr status_t AudioFlinger::PlaybackThread::OutputTrack::obtainBuffer( AudioBufferProvider::Buffer* buffer, uint32_t waitTimeMs) { - audio_track_cblk_t* cblk = mCblk; - uint32_t framesReq = buffer->frameCount; - - ALOGVV("OutputTrack::obtainBuffer user %d, server %d", cblk->user, cblk->server); - buffer->frameCount = 0; - - size_t framesAvail; - { - Mutex::Autolock _l(cblk->lock); - - // read the server count again - while (!(framesAvail = mClientProxy->framesAvailable_l())) { - if (CC_UNLIKELY(!mActive)) { - ALOGV("Not active and NO_MORE_BUFFERS"); - return NO_MORE_BUFFERS; - } - status_t result = cblk->cv.waitRelative(cblk->lock, milliseconds(waitTimeMs)); - if (result != NO_ERROR) { - return NO_MORE_BUFFERS; - } - } - } - - if (framesReq > framesAvail) { - framesReq = framesAvail; - } - - uint32_t u = cblk->user; - uint32_t bufferEnd = cblk->userBase + mFrameCount; - - if (framesReq > bufferEnd - u) { - framesReq = bufferEnd - u; - } - - buffer->frameCount = framesReq; - buffer->raw = mClientProxy->buffer(u); - return NO_ERROR; + ClientProxy::Buffer buf; + buf.mFrameCount = buffer->frameCount; + struct timespec timeout; + timeout.tv_sec = waitTimeMs / 1000; + timeout.tv_nsec = (int) (waitTimeMs % 1000) * 1000000; + status_t status = mClientProxy->obtainBuffer(&buf, &timeout); + buffer->frameCount = buf.mFrameCount; + buffer->raw = buf.mRaw; + return status; } - void AudioFlinger::PlaybackThread::OutputTrack::clearBufferQueue() { size_t size = mBufferQueue.size(); @@ -1687,7 +1734,12 @@ AudioFlinger::RecordThread::RecordTrack::RecordTrack( channelMask, frameCount, 0 /*sharedBuffer*/, sessionId, false /*isOut*/), mOverflow(false) { - ALOGV("RecordTrack constructor, size %d", (int)mBufferEnd - (int)mBuffer); + ALOGV("RecordTrack constructor"); + if (mCblk != NULL) { + mAudioRecordServerProxy = new AudioRecordServerProxy(mCblk, mBuffer, frameCount, + mFrameSize); + mServerProxy = mAudioRecordServerProxy; + } } AudioFlinger::RecordThread::RecordTrack::~RecordTrack() @@ -1699,42 +1751,16 @@ AudioFlinger::RecordThread::RecordTrack::~RecordTrack() status_t AudioFlinger::RecordThread::RecordTrack::getNextBuffer(AudioBufferProvider::Buffer* buffer, int64_t pts) { - audio_track_cblk_t* cblk = this->cblk(); - uint32_t framesAvail; - uint32_t framesReq = buffer->frameCount; - - // Check if last stepServer failed, try to step now - if (mStepServerFailed) { - if (!step()) { - goto getNextBuffer_exit; - } - ALOGV("stepServer recovered"); - mStepServerFailed = false; + ServerProxy::Buffer buf; + buf.mFrameCount = buffer->frameCount; + status_t status = mServerProxy->obtainBuffer(&buf); + buffer->frameCount = buf.mFrameCount; + buffer->raw = buf.mRaw; + if (buf.mFrameCount == 0) { + // FIXME also wake futex so that overrun is noticed more quickly + (void) android_atomic_or(CBLK_OVERRUN, &mCblk->mFlags); } - - // FIXME lock is not actually held, so overrun is possible - framesAvail = mServerProxy->framesAvailableIn_l(); - - if (CC_LIKELY(framesAvail)) { - uint32_t s = cblk->server; - uint32_t bufferEnd = cblk->serverBase + mFrameCount; - - if (framesReq > framesAvail) { - framesReq = framesAvail; - } - if (framesReq > bufferEnd - s) { - framesReq = bufferEnd - s; - } - - buffer->raw = getBuffer(s, framesReq); - buffer->frameCount = framesReq; - return NO_ERROR; - } - -getNextBuffer_exit: - buffer->raw = NULL; - buffer->frameCount = 0; - return NOT_ENOUGH_DATA; + return status; } status_t AudioFlinger::RecordThread::RecordTrack::start(AudioSystem::sync_event_t event, @@ -1754,16 +1780,7 @@ void AudioFlinger::RecordThread::RecordTrack::stop() sp<ThreadBase> thread = mThread.promote(); if (thread != 0) { RecordThread *recordThread = (RecordThread *)thread.get(); - recordThread->mLock.lock(); - bool doStop = recordThread->stop_l(this); - if (doStop) { - TrackBase::reset(); - // Force overrun condition to avoid false overrun callback until first data is - // read from buffer - android_atomic_or(CBLK_UNDERRUN, &mCblk->flags); - } - recordThread->mLock.unlock(); - if (doStop) { + if (recordThread->stop(this)) { AudioSystem::stopInput(recordThread->id()); } } @@ -1787,23 +1804,31 @@ void AudioFlinger::RecordThread::RecordTrack::destroy() } } +void AudioFlinger::RecordThread::RecordTrack::invalidate() +{ + // FIXME should use proxy, and needs work + audio_track_cblk_t* cblk = mCblk; + android_atomic_or(CBLK_INVALID, &cblk->mFlags); + android_atomic_release_store(0x40000000, &cblk->mFutex); + // client is not in server, so FUTEX_WAKE is needed instead of FUTEX_WAKE_PRIVATE + (void) __futex_syscall3(&cblk->mFutex, FUTEX_WAKE, INT_MAX); +} + /*static*/ void AudioFlinger::RecordThread::RecordTrack::appendDumpHeader(String8& result) { - result.append(" Clien Fmt Chn mask Session Step S Serv User FrameCount\n"); + result.append("Client Fmt Chn mask Session S Server fCount\n"); } void AudioFlinger::RecordThread::RecordTrack::dump(char* buffer, size_t size) { - snprintf(buffer, size, " %05d %03u 0x%08x %05d %04u %01d %08x %08x %05d\n", + snprintf(buffer, size, "%6u %3u %08X %7u %1d %08X %6u\n", (mClient == 0) ? getpid_cached : mClient->pid(), mFormat, mChannelMask, mSessionId, - mStepCount, mState, - mCblk->server, - mCblk->user, + mCblk->mServer, mFrameCount); } diff --git a/services/camera/libcameraservice/Android.mk b/services/camera/libcameraservice/Android.mk index 83d9ccd..d23f8b9 100644 --- a/services/camera/libcameraservice/Android.mk +++ b/services/camera/libcameraservice/Android.mk @@ -8,29 +8,32 @@ include $(CLEAR_VARS) LOCAL_SRC_FILES:= \ CameraService.cpp \ - CameraClient.cpp \ - Camera2Client.cpp \ - ProCamera2Client.cpp \ - Camera2ClientBase.cpp \ - CameraDeviceBase.cpp \ - Camera2Device.cpp \ - Camera3Device.cpp \ - camera2/Parameters.cpp \ - camera2/FrameProcessor.cpp \ - camera2/StreamingProcessor.cpp \ - camera2/JpegProcessor.cpp \ - camera2/CallbackProcessor.cpp \ - camera2/ZslProcessor.cpp \ - camera2/BurstCapture.cpp \ - camera2/JpegCompressor.cpp \ - camera2/CaptureSequencer.cpp \ - camera2/ProFrameProcessor.cpp \ - camera2/ZslProcessor3.cpp \ - camera3/Camera3Stream.cpp \ - camera3/Camera3IOStreamBase.cpp \ - camera3/Camera3InputStream.cpp \ - camera3/Camera3OutputStream.cpp \ - camera3/Camera3ZslStream.cpp \ + CameraDeviceFactory.cpp \ + common/Camera2ClientBase.cpp \ + common/CameraDeviceBase.cpp \ + common/FrameProcessorBase.cpp \ + api1/CameraClient.cpp \ + api1/Camera2Client.cpp \ + api1/client2/Parameters.cpp \ + api1/client2/FrameProcessor.cpp \ + api1/client2/StreamingProcessor.cpp \ + api1/client2/JpegProcessor.cpp \ + api1/client2/CallbackProcessor.cpp \ + api1/client2/ZslProcessor.cpp \ + api1/client2/BurstCapture.cpp \ + api1/client2/JpegCompressor.cpp \ + api1/client2/CaptureSequencer.cpp \ + api1/client2/ZslProcessor3.cpp \ + api2/CameraDeviceClient.cpp \ + api_pro/ProCamera2Client.cpp \ + device2/Camera2Device.cpp \ + device3/Camera3Device.cpp \ + device3/Camera3Stream.cpp \ + device3/Camera3IOStreamBase.cpp \ + device3/Camera3InputStream.cpp \ + device3/Camera3OutputStream.cpp \ + device3/Camera3ZslStream.cpp \ + device3/StatusTracker.cpp \ gui/RingBufferConsumer.cpp \ LOCAL_SHARED_LIBRARIES:= \ diff --git a/services/camera/libcameraservice/CameraDeviceFactory.cpp b/services/camera/libcameraservice/CameraDeviceFactory.cpp new file mode 100644 index 0000000..7fdf304 --- /dev/null +++ b/services/camera/libcameraservice/CameraDeviceFactory.cpp @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// #define LOG_NDEBUG 0 +#define LOG_TAG "CameraDeviceFactory" +#include <utils/Log.h> + +#include "CameraService.h" +#include "CameraDeviceFactory.h" +#include "common/CameraDeviceBase.h" +#include "device2/Camera2Device.h" +#include "device3/Camera3Device.h" + +namespace android { + +wp<CameraService> CameraDeviceFactory::sService; + +sp<CameraDeviceBase> CameraDeviceFactory::createDevice(int cameraId) { + + sp<CameraService> svc = sService.promote(); + if (svc == 0) { + ALOGE("%s: No service registered", __FUNCTION__); + return NULL; + } + + int deviceVersion = svc->getDeviceVersion(cameraId, /*facing*/NULL); + + sp<CameraDeviceBase> device; + + switch (deviceVersion) { + case CAMERA_DEVICE_API_VERSION_2_0: + case CAMERA_DEVICE_API_VERSION_2_1: + device = new Camera2Device(cameraId); + break; + case CAMERA_DEVICE_API_VERSION_3_0: + device = new Camera3Device(cameraId); + break; + default: + ALOGE("%s: Camera %d: Unknown HAL device version %d", + __FUNCTION__, cameraId, deviceVersion); + device = NULL; + break; + } + + ALOGV_IF(device != 0, "Created a new camera device for version %d", + deviceVersion); + + return device; +} + +void CameraDeviceFactory::registerService(wp<CameraService> service) { + ALOGV("%s: Registered service %p", __FUNCTION__, + service.promote().get()); + + sService = service; +} + +}; // namespace android diff --git a/services/camera/libcameraservice/CameraDeviceFactory.h b/services/camera/libcameraservice/CameraDeviceFactory.h new file mode 100644 index 0000000..236dc56 --- /dev/null +++ b/services/camera/libcameraservice/CameraDeviceFactory.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_SERVERS_CAMERA_CAMERADEVICEFACTORY_H +#define ANDROID_SERVERS_CAMERA_CAMERADEVICEFACTORY_H + +#include <utils/RefBase.h> + +namespace android { + +class CameraDeviceBase; +class CameraService; + +/** + * Create the right instance of Camera2Device or Camera3Device + * automatically based on the device version. + */ +class CameraDeviceFactory : public virtual RefBase { + public: + static void registerService(wp<CameraService> service); + + // Prerequisite: Call registerService. + static sp<CameraDeviceBase> createDevice(int cameraId); + private: + CameraDeviceFactory(wp<CameraService> service); + + static wp<CameraService> sService; +}; + +}; // namespace android + +#endif diff --git a/services/camera/libcameraservice/CameraService.cpp b/services/camera/libcameraservice/CameraService.cpp index 757a781..34a5b15 100644 --- a/services/camera/libcameraservice/CameraService.cpp +++ b/services/camera/libcameraservice/CameraService.cpp @@ -38,9 +38,11 @@ #include <utils/String16.h> #include "CameraService.h" -#include "CameraClient.h" -#include "Camera2Client.h" -#include "ProCamera2Client.h" +#include "api1/CameraClient.h" +#include "api1/Camera2Client.h" +#include "api_pro/ProCamera2Client.h" +#include "api2/CameraDeviceClient.h" +#include "CameraDeviceFactory.h" namespace android { @@ -126,6 +128,8 @@ void CameraService::onFirstRef() CAMERA_MODULE_API_VERSION_2_1) { mModule->set_callbacks(this); } + + CameraDeviceFactory::registerService(this); } } @@ -164,7 +168,7 @@ void CameraService::onDeviceStatusChanged(int cameraId, Mutex::Autolock al(mServiceLock); /* Find all clients that we need to disconnect */ - sp<Client> client = mClient[cameraId].promote(); + sp<BasicClient> client = mClient[cameraId].promote(); if (client.get() != NULL) { clientsToDisconnect.push_back(client); } @@ -207,7 +211,7 @@ int32_t CameraService::getNumberOfCameras() { status_t CameraService::getCameraInfo(int cameraId, struct CameraInfo* cameraInfo) { if (!mModule) { - return NO_INIT; + return -ENODEV; } if (cameraId < 0 || cameraId >= mNumberOfCameras) { @@ -221,6 +225,49 @@ status_t CameraService::getCameraInfo(int cameraId, return rc; } +status_t CameraService::getCameraCharacteristics(int cameraId, + CameraMetadata* cameraInfo) { + if (!cameraInfo) { + ALOGE("%s: cameraInfo is NULL", __FUNCTION__); + return BAD_VALUE; + } + + if (!mModule) { + ALOGE("%s: camera hardware module doesn't exist", __FUNCTION__); + return -ENODEV; + } + + if (mModule->common.module_api_version < CAMERA_MODULE_API_VERSION_2_0) { + // TODO: Remove this check once HAL1 shim is in place. + ALOGE("%s: Only HAL module version V2 or higher supports static metadata", __FUNCTION__); + return BAD_VALUE; + } + + if (cameraId < 0 || cameraId >= mNumberOfCameras) { + ALOGE("%s: Invalid camera id: %d", __FUNCTION__, cameraId); + return BAD_VALUE; + } + + int facing; + if (getDeviceVersion(cameraId, &facing) == CAMERA_DEVICE_API_VERSION_1_0) { + // TODO: Remove this check once HAL1 shim is in place. + ALOGE("%s: HAL1 doesn't support static metadata yet", __FUNCTION__); + return BAD_VALUE; + } + + if (getDeviceVersion(cameraId, &facing) <= CAMERA_DEVICE_API_VERSION_2_1) { + // Disable HAL2.x support for camera2 API for now. + ALOGW("%s: HAL2.x doesn't support getCameraCharacteristics for now", __FUNCTION__); + return BAD_VALUE; + } + + struct camera_info info; + status_t ret = mModule->get_camera_info(cameraId, &info); + *cameraInfo = info.static_camera_characteristics; + + return ret; +} + int CameraService::getDeviceVersion(int cameraId, int* facing) { struct camera_info info; if (mModule->get_camera_info(cameraId, &info) != OK) { @@ -258,7 +305,7 @@ bool CameraService::isValidCameraId(int cameraId) { return false; } -bool CameraService::validateConnect(int cameraId, +status_t CameraService::validateConnect(int cameraId, /*inout*/ int& clientUid) const { @@ -271,19 +318,19 @@ bool CameraService::validateConnect(int cameraId, if (callingPid != getpid()) { ALOGE("CameraService::connect X (pid %d) rejected (don't trust clientUid)", callingPid); - return false; + return PERMISSION_DENIED; } } if (!mModule) { ALOGE("Camera HAL module not loaded"); - return false; + return -ENODEV; } if (cameraId < 0 || cameraId >= mNumberOfCameras) { ALOGE("CameraService::connect X (pid %d) rejected (invalid cameraId %d).", callingPid, cameraId); - return false; + return -ENODEV; } char value[PROPERTY_VALUE_MAX]; @@ -291,36 +338,36 @@ bool CameraService::validateConnect(int cameraId, if (strcmp(value, "1") == 0) { // Camera is disabled by DevicePolicyManager. ALOGI("Camera is disabled. connect X (pid %d) rejected", callingPid); - return false; + return -EACCES; } ICameraServiceListener::Status currentStatus = getStatus(cameraId); if (currentStatus == ICameraServiceListener::STATUS_NOT_PRESENT) { ALOGI("Camera is not plugged in," " connect X (pid %d) rejected", callingPid); - return false; + return -ENODEV; } else if (currentStatus == ICameraServiceListener::STATUS_ENUMERATING) { ALOGI("Camera is enumerating," " connect X (pid %d) rejected", callingPid); - return false; + return -EBUSY; } // Else don't check for STATUS_NOT_AVAILABLE. // -- It's done implicitly in canConnectUnsafe /w the mBusy array - return true; + return OK; } bool CameraService::canConnectUnsafe(int cameraId, const String16& clientPackageName, const sp<IBinder>& remoteCallback, - sp<Client> &client) { + sp<BasicClient> &client) { String8 clientName8(clientPackageName); int callingPid = getCallingPid(); if (mClient[cameraId] != 0) { client = mClient[cameraId].promote(); if (client != 0) { - if (remoteCallback == client->getRemoteCallback()->asBinder()) { + if (remoteCallback == client->getRemote()) { LOG1("CameraService::connect X (pid %d) (the same client)", callingPid); return true; @@ -354,11 +401,13 @@ bool CameraService::canConnectUnsafe(int cameraId, return true; } -sp<ICamera> CameraService::connect( +status_t CameraService::connect( const sp<ICameraClient>& cameraClient, int cameraId, const String16& clientPackageName, - int clientUid) { + int clientUid, + /*out*/ + sp<ICamera>& device) { String8 clientName8(clientPackageName); int callingPid = getCallingPid(); @@ -366,20 +415,23 @@ sp<ICamera> CameraService::connect( LOG1("CameraService::connect E (pid %d \"%s\", id %d)", callingPid, clientName8.string(), cameraId); - if (!validateConnect(cameraId, /*inout*/clientUid)) { - return NULL; + status_t status = validateConnect(cameraId, /*inout*/clientUid); + if (status != OK) { + return status; } - sp<Client> client; + sp<Client> client; { Mutex::Autolock lock(mServiceLock); + sp<BasicClient> clientTmp; if (!canConnectUnsafe(cameraId, clientPackageName, cameraClient->asBinder(), - /*out*/client)) { - return NULL; + /*out*/clientTmp)) { + return -EBUSY; } else if (client.get() != NULL) { - return client; + device = static_cast<Client*>(clientTmp.get()); + return OK; } int facing = -1; @@ -409,18 +461,18 @@ sp<ICamera> CameraService::connect( break; case -1: ALOGE("Invalid camera id %d", cameraId); - return NULL; + return BAD_VALUE; default: ALOGE("Unknown camera device HAL version: %d", deviceVersion); - return NULL; + return INVALID_OPERATION; } - if (!connectFinishUnsafe(client, client->asBinder())) { + status_t status = connectFinishUnsafe(client, client->getRemote()); + if (status != OK) { // this is probably not recoverable.. maybe the client can try again // OK: we can only get here if we were originally in PRESENT state updateStatus(ICameraServiceListener::STATUS_PRESENT, cameraId); - - return NULL; + return status; } mClient[cameraId] = client; @@ -430,45 +482,49 @@ sp<ICamera> CameraService::connect( // important: release the mutex here so the client can call back // into the service from its destructor (can be at the end of the call) - return client; + device = client; + return OK; } -bool CameraService::connectFinishUnsafe(const sp<BasicClient>& client, - const sp<IBinder>& clientBinder) { - if (client->initialize(mModule) != OK) { - return false; +status_t CameraService::connectFinishUnsafe(const sp<BasicClient>& client, + const sp<IBinder>& remoteCallback) { + status_t status = client->initialize(mModule); + if (status != OK) { + return status; } - clientBinder->linkToDeath(this); + remoteCallback->linkToDeath(this); - return true; + return OK; } -sp<IProCameraUser> CameraService::connect( +status_t CameraService::connectPro( const sp<IProCameraCallbacks>& cameraCb, int cameraId, const String16& clientPackageName, - int clientUid) + int clientUid, + /*out*/ + sp<IProCameraUser>& device) { String8 clientName8(clientPackageName); int callingPid = getCallingPid(); LOG1("CameraService::connectPro E (pid %d \"%s\", id %d)", callingPid, clientName8.string(), cameraId); - - if (!validateConnect(cameraId, /*inout*/clientUid)) { - return NULL; + status_t status = validateConnect(cameraId, /*inout*/clientUid); + if (status != OK) { + return status; } sp<ProClient> client; { Mutex::Autolock lock(mServiceLock); { - sp<Client> client; + sp<BasicClient> client; if (!canConnectUnsafe(cameraId, clientPackageName, cameraCb->asBinder(), /*out*/client)) { - return NULL; + return -EBUSY; } } @@ -479,23 +535,25 @@ sp<IProCameraUser> CameraService::connect( case CAMERA_DEVICE_API_VERSION_1_0: ALOGE("Camera id %d uses HALv1, doesn't support ProCamera", cameraId); - return NULL; + return -EOPNOTSUPP; break; case CAMERA_DEVICE_API_VERSION_2_0: case CAMERA_DEVICE_API_VERSION_2_1: + case CAMERA_DEVICE_API_VERSION_3_0: client = new ProCamera2Client(this, cameraCb, String16(), cameraId, facing, callingPid, USE_CALLING_UID, getpid()); break; case -1: ALOGE("Invalid camera id %d", cameraId); - return NULL; + return BAD_VALUE; default: ALOGE("Unknown camera device HAL version: %d", deviceVersion); - return NULL; + return INVALID_OPERATION; } - if (!connectFinishUnsafe(client, client->asBinder())) { - return NULL; + status_t status = connectFinishUnsafe(client, client->getRemote()); + if (status != OK) { + return status; } mProClientList[cameraId].push(client); @@ -505,10 +563,93 @@ sp<IProCameraUser> CameraService::connect( } // important: release the mutex here so the client can call back // into the service from its destructor (can be at the end of the call) + device = client; + return OK; +} - return client; +status_t CameraService::connectDevice( + const sp<ICameraDeviceCallbacks>& cameraCb, + int cameraId, + const String16& clientPackageName, + int clientUid, + /*out*/ + sp<ICameraDeviceUser>& device) +{ + + String8 clientName8(clientPackageName); + int callingPid = getCallingPid(); + + LOG1("CameraService::connectDevice E (pid %d \"%s\", id %d)", callingPid, + clientName8.string(), cameraId); + + status_t status = validateConnect(cameraId, /*inout*/clientUid); + if (status != OK) { + return status; + } + + sp<CameraDeviceClient> client; + { + Mutex::Autolock lock(mServiceLock); + { + sp<BasicClient> client; + if (!canConnectUnsafe(cameraId, clientPackageName, + cameraCb->asBinder(), + /*out*/client)) { + return -EBUSY; + } + } + + int facing = -1; + int deviceVersion = getDeviceVersion(cameraId, &facing); + + // If there are other non-exclusive users of the camera, + // this will tear them down before we can reuse the camera + if (isValidCameraId(cameraId)) { + // transition from PRESENT -> NOT_AVAILABLE + updateStatus(ICameraServiceListener::STATUS_NOT_AVAILABLE, + cameraId); + } + + switch(deviceVersion) { + case CAMERA_DEVICE_API_VERSION_1_0: + ALOGW("Camera using old HAL version: %d", deviceVersion); + return -EOPNOTSUPP; + // TODO: don't allow 2.0 Only allow 2.1 and higher + case CAMERA_DEVICE_API_VERSION_2_0: + case CAMERA_DEVICE_API_VERSION_2_1: + case CAMERA_DEVICE_API_VERSION_3_0: + client = new CameraDeviceClient(this, cameraCb, String16(), + cameraId, facing, callingPid, USE_CALLING_UID, getpid()); + break; + case -1: + ALOGE("Invalid camera id %d", cameraId); + return BAD_VALUE; + default: + ALOGE("Unknown camera device HAL version: %d", deviceVersion); + return INVALID_OPERATION; + } + + status_t status = connectFinishUnsafe(client, client->getRemote()); + if (status != OK) { + // this is probably not recoverable.. maybe the client can try again + // OK: we can only get here if we were originally in PRESENT state + updateStatus(ICameraServiceListener::STATUS_PRESENT, cameraId); + return status; + } + + LOG1("CameraService::connectDevice X (id %d, this pid is %d)", cameraId, + getpid()); + + mClient[cameraId] = client; + } + // important: release the mutex here so the client can call back + // into the service from its destructor (can be at the end of the call) + + device = client; + return OK; } + status_t CameraService::addListener( const sp<ICameraServiceListener>& listener) { ALOGV("%s: Add listener %p", __FUNCTION__, listener.get()); @@ -566,14 +707,14 @@ void CameraService::removeClientByRemote(const wp<IBinder>& remoteBinder) { Mutex::Autolock lock(mServiceLock); int outIndex; - sp<Client> client = findClientUnsafe(remoteBinder, outIndex); + sp<BasicClient> client = findClientUnsafe(remoteBinder, outIndex); if (client != 0) { // Found our camera, clear and leave. LOG1("removeClient: clear camera %d", outIndex); mClient[outIndex].clear(); - client->unlinkToDeath(this); + client->getRemote()->unlinkToDeath(this); } else { sp<ProClient> clientPro = findProClientUnsafe(remoteBinder); @@ -620,9 +761,9 @@ sp<CameraService::ProClient> CameraService::findProClientUnsafe( return clientPro; } -sp<CameraService::Client> CameraService::findClientUnsafe( +sp<CameraService::BasicClient> CameraService::findClientUnsafe( const wp<IBinder>& cameraClient, int& outIndex) { - sp<Client> client; + sp<BasicClient> client; for (int i = 0; i < mNumberOfCameras; i++) { @@ -640,7 +781,7 @@ sp<CameraService::Client> CameraService::findClientUnsafe( continue; } - if (cameraClient == client->getRemoteCallback()->asBinder()) { + if (cameraClient == client->getRemote()) { // Found our camera outIndex = i; return client; @@ -651,7 +792,7 @@ sp<CameraService::Client> CameraService::findClientUnsafe( return NULL; } -CameraService::Client* CameraService::getClientByIdUnsafe(int cameraId) { +CameraService::BasicClient* CameraService::getClientByIdUnsafe(int cameraId) { if (cameraId < 0 || cameraId >= mNumberOfCameras) return NULL; return mClient[cameraId].unsafe_get(); } @@ -906,7 +1047,9 @@ Mutex* CameraService::Client::getClientLockFromCookie(void* user) { // Provide client pointer for callbacks. Client lock returned from getClientLockFromCookie should // be acquired for this to be safe CameraService::Client* CameraService::Client::getClientFromCookie(void* user) { - Client* client = gCameraService->getClientByIdUnsafe((int) user); + BasicClient *basicClient = gCameraService->getClientByIdUnsafe((int) user); + // OK: only CameraClient calls this, and they already cast anyway. + Client* client = static_cast<Client*>(basicClient); // This could happen if the Client is in the process of shutting down (the // last strong reference is gone, but the destructor hasn't finished @@ -1058,7 +1201,7 @@ status_t CameraService::dump(int fd, const Vector<String16>& args) { } } - sp<Client> client = mClient[i].promote(); + sp<BasicClient> client = mClient[i].promote(); if (client == 0) { result = String8::format(" Device is closed, no client instance\n"); write(fd, result.string(), result.size()); diff --git a/services/camera/libcameraservice/CameraService.h b/services/camera/libcameraservice/CameraService.h index 710f164..ad6a582 100644 --- a/services/camera/libcameraservice/CameraService.h +++ b/services/camera/libcameraservice/CameraService.h @@ -29,6 +29,8 @@ #include <camera/ICameraClient.h> #include <camera/IProCameraUser.h> #include <camera/IProCameraCallbacks.h> +#include <camera/camera2/ICameraDeviceUser.h> +#include <camera/camera2/ICameraDeviceCallbacks.h> #include <camera/ICameraServiceListener.h> @@ -69,11 +71,26 @@ public: virtual int32_t getNumberOfCameras(); virtual status_t getCameraInfo(int cameraId, struct CameraInfo* cameraInfo); - - virtual sp<ICamera> connect(const sp<ICameraClient>& cameraClient, int cameraId, - const String16& clientPackageName, int clientUid); - virtual sp<IProCameraUser> connect(const sp<IProCameraCallbacks>& cameraCb, - int cameraId, const String16& clientPackageName, int clientUid); + virtual status_t getCameraCharacteristics(int cameraId, + CameraMetadata* cameraInfo); + + virtual status_t connect(const sp<ICameraClient>& cameraClient, int cameraId, + const String16& clientPackageName, int clientUid, + /*out*/ + sp<ICamera>& device); + + virtual status_t connectPro(const sp<IProCameraCallbacks>& cameraCb, + int cameraId, const String16& clientPackageName, int clientUid, + /*out*/ + sp<IProCameraUser>& device); + + virtual status_t connectDevice( + const sp<ICameraDeviceCallbacks>& cameraCb, + int cameraId, + const String16& clientPackageName, + int clientUid, + /*out*/ + sp<ICameraDeviceUser>& device); virtual status_t addListener(const sp<ICameraServiceListener>& listener); virtual status_t removeListener( @@ -99,13 +116,17 @@ public: void playSound(sound_kind kind); void releaseSound(); + ///////////////////////////////////////////////////////////////////// + // CameraDeviceFactory functionality + int getDeviceVersion(int cameraId, int* facing = NULL); + ///////////////////////////////////////////////////////////////////// // CameraClient functionality // returns plain pointer of client. Note that mClientLock should be acquired to // prevent the client from destruction. The result can be NULL. - virtual Client* getClientByIdUnsafe(int cameraId); + virtual BasicClient* getClientByIdUnsafe(int cameraId); virtual Mutex* getClientLockById(int cameraId); class BasicClient : public virtual RefBase { @@ -114,11 +135,17 @@ public: virtual void disconnect() = 0; + // because we can't virtually inherit IInterface, which breaks + // virtual inheritance + virtual sp<IBinder> asBinderWrapper() = 0; + // Return the remote callback binder object (e.g. IProCameraCallbacks) - wp<IBinder> getRemote() { + sp<IBinder> getRemote() { return mRemoteBinder; } + virtual status_t dump(int fd, const Vector<String16>& args) = 0; + protected: BasicClient(const sp<CameraService>& cameraService, const sp<IBinder>& remoteCallback, @@ -147,7 +174,7 @@ public: pid_t mServicePid; // immutable after constructor // - The app-side Binder interface to receive callbacks from us - wp<IBinder> mRemoteBinder; // immutable after constructor + sp<IBinder> mRemoteBinder; // immutable after constructor // permissions management status_t startCameraOps(); @@ -187,9 +214,10 @@ public: virtual status_t connect(const sp<ICameraClient>& client) = 0; virtual status_t lock() = 0; virtual status_t unlock() = 0; - virtual status_t setPreviewDisplay(const sp<Surface>& surface) = 0; - virtual status_t setPreviewTexture(const sp<IGraphicBufferProducer>& bufferProducer)=0; + virtual status_t setPreviewTarget(const sp<IGraphicBufferProducer>& bufferProducer)=0; virtual void setPreviewCallbackFlag(int flag) = 0; + virtual status_t setPreviewCallbackTarget( + const sp<IGraphicBufferProducer>& callbackProducer) = 0; virtual status_t startPreview() = 0; virtual void stopPreview() = 0; virtual bool previewEnabled() = 0; @@ -221,6 +249,10 @@ public: return mRemoteCallback; } + virtual sp<IBinder> asBinderWrapper() { + return asBinder(); + } + protected: static Mutex* getClientLockFromCookie(void* user); // convert client from cookie. Client lock should be acquired before getting Client. @@ -285,7 +317,7 @@ private: virtual void onFirstRef(); // Step 1. Check if we can connect, before we acquire the service lock. - bool validateConnect(int cameraId, + status_t validateConnect(int cameraId, /*inout*/ int& clientUid) const; @@ -294,16 +326,17 @@ private: const String16& clientPackageName, const sp<IBinder>& remoteCallback, /*out*/ - sp<Client> &client); + sp<BasicClient> &client); // When connection is successful, initialize client and track its death - bool connectFinishUnsafe(const sp<BasicClient>& client, - const sp<IBinder>& clientBinder); + status_t connectFinishUnsafe(const sp<BasicClient>& client, + const sp<IBinder>& remoteCallback); virtual sp<BasicClient> getClientByRemote(const wp<IBinder>& cameraClient); Mutex mServiceLock; - wp<Client> mClient[MAX_CAMERAS]; // protected by mServiceLock + // either a Client or CameraDeviceClient + wp<BasicClient> mClient[MAX_CAMERAS]; // protected by mServiceLock Mutex mClientLock[MAX_CAMERAS]; // prevent Client destruction inside callbacks int mNumberOfCameras; @@ -311,7 +344,7 @@ private: Vector<weak_pro_client_ptr> mProClientList[MAX_CAMERAS]; // needs to be called with mServiceLock held - sp<Client> findClientUnsafe(const wp<IBinder>& cameraClient, int& outIndex); + sp<BasicClient> findClientUnsafe(const wp<IBinder>& cameraClient, int& outIndex); sp<ProClient> findProClientUnsafe( const wp<IBinder>& cameraCallbacksRemote); @@ -352,7 +385,6 @@ private: virtual void binderDied(const wp<IBinder> &who); // Helpers - int getDeviceVersion(int cameraId, int* facing); bool isValidCameraId(int cameraId); }; diff --git a/services/camera/libcameraservice/Camera2Client.cpp b/services/camera/libcameraservice/api1/Camera2Client.cpp index 6942006..df3b162 100644 --- a/services/camera/libcameraservice/Camera2Client.cpp +++ b/services/camera/libcameraservice/api1/Camera2Client.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#define LOG_TAG "Camera2" +#define LOG_TAG "Camera2Client" #define ATRACE_TAG ATRACE_TAG_CAMERA //#define LOG_NDEBUG 0 @@ -23,13 +23,15 @@ #include <cutils/properties.h> #include <gui/Surface.h> -#include "camera2/Parameters.h" -#include "Camera2Client.h" -#include "Camera2Device.h" -#include "Camera3Device.h" -#include "camera2/ZslProcessor.h" -#include "camera2/ZslProcessor3.h" +#include "api1/Camera2Client.h" + +#include "api1/client2/StreamingProcessor.h" +#include "api1/client2/JpegProcessor.h" +#include "api1/client2/CaptureSequencer.h" +#include "api1/client2/CallbackProcessor.h" +#include "api1/client2/ZslProcessor.h" +#include "api1/client2/ZslProcessor3.h" #define ALOG1(...) ALOGD_IF(gLogLevel >= 1, __VA_ARGS__); #define ALOG2(...) ALOGD_IF(gLogLevel >= 2, __VA_ARGS__); @@ -58,22 +60,6 @@ Camera2Client::Camera2Client(const sp<CameraService>& cameraService, mDeviceVersion(deviceVersion) { ATRACE_CALL(); - ALOGI("Camera %d: Opened", cameraId); - - switch (mDeviceVersion) { - case CAMERA_DEVICE_API_VERSION_2_0: - mDevice = new Camera2Device(cameraId); - break; - case CAMERA_DEVICE_API_VERSION_3_0: - mDevice = new Camera3Device(cameraId); - break; - default: - ALOGE("Camera %d: Unknown HAL device version %d", - cameraId, mDeviceVersion); - mDevice = NULL; - break; - } - SharedParameters::Lock l(mParameters); l.mParameters.state = Parameters::DISCONNECTED; @@ -297,6 +283,7 @@ status_t Camera2Client::dump(int fd, const Vector<String16>& args) { CASE_APPEND_ENUM(ANDROID_CONTROL_AF_STATE_INACTIVE) CASE_APPEND_ENUM(ANDROID_CONTROL_AF_STATE_PASSIVE_SCAN) CASE_APPEND_ENUM(ANDROID_CONTROL_AF_STATE_PASSIVE_FOCUSED) + CASE_APPEND_ENUM(ANDROID_CONTROL_AF_STATE_PASSIVE_UNFOCUSED) CASE_APPEND_ENUM(ANDROID_CONTROL_AF_STATE_ACTIVE_SCAN) CASE_APPEND_ENUM(ANDROID_CONTROL_AF_STATE_FOCUSED_LOCKED) CASE_APPEND_ENUM(ANDROID_CONTROL_AF_STATE_NOT_FOCUSED_LOCKED) @@ -340,6 +327,10 @@ status_t Camera2Client::dump(int fd, const Vector<String16>& args) { result.appendFormat(" Video stabilization is %s\n", p.videoStabilization ? "enabled" : "disabled"); + result.appendFormat(" Selected still capture FPS range: %d - %d\n", + p.fastInfo.bestStillCaptureFpsRange[0], + p.fastInfo.bestStillCaptureFpsRange[1]); + result.append(" Current streams:\n"); result.appendFormat(" Preview stream ID: %d\n", getPreviewStreamId()); @@ -505,25 +496,7 @@ status_t Camera2Client::unlock() { return EBUSY; } -status_t Camera2Client::setPreviewDisplay( - const sp<Surface>& surface) { - ATRACE_CALL(); - ALOGV("%s: E", __FUNCTION__); - Mutex::Autolock icl(mBinderSerializationLock); - status_t res; - if ( (res = checkPid(__FUNCTION__) ) != OK) return res; - - sp<IBinder> binder; - sp<ANativeWindow> window; - if (surface != 0) { - binder = surface->getIGraphicBufferProducer()->asBinder(); - window = surface; - } - - return setPreviewWindowL(binder,window); -} - -status_t Camera2Client::setPreviewTexture( +status_t Camera2Client::setPreviewTarget( const sp<IGraphicBufferProducer>& bufferProducer) { ATRACE_CALL(); ALOGV("%s: E", __FUNCTION__); @@ -535,7 +508,10 @@ status_t Camera2Client::setPreviewTexture( sp<ANativeWindow> window; if (bufferProducer != 0) { binder = bufferProducer->asBinder(); - window = new Surface(bufferProducer); + // Using controlledByApp flag to ensure that the buffer queue remains in + // async mode for the old camera API, where many applications depend + // on that behavior. + window = new Surface(bufferProducer, /*controlledByApp*/ true); } return setPreviewWindowL(binder, window); } @@ -632,6 +608,19 @@ void Camera2Client::setPreviewCallbackFlagL(Parameters ¶ms, int flag) { params.previewCallbackOneShot = true; } if (params.previewCallbackFlags != (uint32_t)flag) { + + if (params.previewCallbackSurface && flag != CAMERA_FRAME_CALLBACK_FLAG_NOOP) { + // Disable any existing preview callback window when enabling + // preview callback flags + res = mCallbackProcessor->setCallbackWindow(NULL); + if (res != OK) { + ALOGE("%s: Camera %d: Unable to clear preview callback surface:" + " %s (%d)", __FUNCTION__, mCameraId, strerror(-res), res); + return; + } + params.previewCallbackSurface = false; + } + params.previewCallbackFlags = flag; if (params.state == Parameters::PREVIEW) { @@ -643,9 +632,61 @@ void Camera2Client::setPreviewCallbackFlagL(Parameters ¶ms, int flag) { } } } +} +status_t Camera2Client::setPreviewCallbackTarget( + const sp<IGraphicBufferProducer>& callbackProducer) { + ATRACE_CALL(); + ALOGV("%s: E", __FUNCTION__); + Mutex::Autolock icl(mBinderSerializationLock); + status_t res; + if ( (res = checkPid(__FUNCTION__) ) != OK) return res; + + sp<ANativeWindow> window; + if (callbackProducer != 0) { + window = new Surface(callbackProducer); + } + + res = mCallbackProcessor->setCallbackWindow(window); + if (res != OK) { + ALOGE("%s: Camera %d: Unable to set preview callback surface: %s (%d)", + __FUNCTION__, mCameraId, strerror(-res), res); + return res; + } + + SharedParameters::Lock l(mParameters); + + if (window != NULL) { + // Disable traditional callbacks when a valid callback target is given + l.mParameters.previewCallbackFlags = CAMERA_FRAME_CALLBACK_FLAG_NOOP; + l.mParameters.previewCallbackOneShot = false; + l.mParameters.previewCallbackSurface = true; + } else { + // Disable callback target if given a NULL interface. + l.mParameters.previewCallbackSurface = false; + } + + switch(l.mParameters.state) { + case Parameters::PREVIEW: + res = startPreviewL(l.mParameters, true); + break; + case Parameters::RECORD: + case Parameters::VIDEO_SNAPSHOT: + res = startRecordingL(l.mParameters, true); + break; + default: + break; + } + if (res != OK) { + ALOGE("%s: Camera %d: Unable to refresh request in state %s", + __FUNCTION__, mCameraId, + Parameters::getStateName(l.mParameters.state)); + } + + return OK; } + status_t Camera2Client::startPreview() { ATRACE_CALL(); ALOGV("%s: E", __FUNCTION__); @@ -707,9 +748,11 @@ status_t Camera2Client::startPreviewL(Parameters ¶ms, bool restart) { return res; } - Vector<uint8_t> outputStreams; - bool callbacksEnabled = params.previewCallbackFlags & - CAMERA_FRAME_CALLBACK_FLAG_ENABLE_MASK; + Vector<int32_t> outputStreams; + bool callbacksEnabled = (params.previewCallbackFlags & + CAMERA_FRAME_CALLBACK_FLAG_ENABLE_MASK) || + params.previewCallbackSurface; + if (callbacksEnabled) { // Can't have recording stream hanging around when enabling callbacks, // since it exceeds the max stream count on some devices. @@ -816,6 +859,7 @@ void Camera2Client::stopPreviewL() { // no break case Parameters::RECORD: case Parameters::PREVIEW: + syncWithDevice(); res = stopStream(); if (res != OK) { ALOGE("%s: Camera %d: Can't stop streaming: %s (%d)", @@ -960,7 +1004,7 @@ status_t Camera2Client::startRecordingL(Parameters ¶ms, bool restart) { return res; } - Vector<uint8_t> outputStreams; + Vector<int32_t> outputStreams; outputStreams.push(getPreviewStreamId()); outputStreams.push(getRecordingStreamId()); @@ -1105,6 +1149,8 @@ status_t Camera2Client::autoFocus() { l.mParameters.currentAfTriggerId = ++l.mParameters.afTriggerCounter; triggerId = l.mParameters.currentAfTriggerId; } + ATRACE_ASYNC_BEGIN(kAutofocusLabel, triggerId); + syncWithDevice(); mDevice->triggerAutofocus(triggerId); @@ -1127,6 +1173,12 @@ status_t Camera2Client::cancelAutoFocus() { l.mParameters.focusMode == Parameters::FOCUS_MODE_INFINITY) { return OK; } + + // An active AF trigger is canceled + if (l.mParameters.afTriggerCounter == l.mParameters.currentAfTriggerId) { + ATRACE_ASYNC_END(kAutofocusLabel, l.mParameters.currentAfTriggerId); + } + triggerId = ++l.mParameters.afTriggerCounter; // When using triggerAfWithAuto quirk, may need to reset focus mode to @@ -1155,6 +1207,7 @@ status_t Camera2Client::takePicture(int msgType) { status_t res; if ( (res = checkPid(__FUNCTION__) ) != OK) return res; + int takePictureCounter; { SharedParameters::Lock l(mParameters); switch (l.mParameters.state) { @@ -1193,8 +1246,11 @@ status_t Camera2Client::takePicture(int msgType) { __FUNCTION__, mCameraId, strerror(-res), res); return res; } + takePictureCounter = ++l.mParameters.takePictureCounter; } + ATRACE_ASYNC_BEGIN(kTakepictureLabel, takePictureCounter); + // Need HAL to have correct settings before (possibly) triggering precapture syncWithDevice(); @@ -1422,7 +1478,24 @@ void Camera2Client::notifyAutoFocus(uint8_t newState, int triggerId) { bool afInMotion = false; { SharedParameters::Lock l(mParameters); + // Trace end of AF state + char tmp[32]; + if (l.mParameters.afStateCounter > 0) { + camera_metadata_enum_snprint( + ANDROID_CONTROL_AF_STATE, l.mParameters.focusState, tmp, sizeof(tmp)); + ATRACE_ASYNC_END(tmp, l.mParameters.afStateCounter); + } + + // Update state l.mParameters.focusState = newState; + l.mParameters.afStateCounter++; + + // Trace start of AF state + + camera_metadata_enum_snprint( + ANDROID_CONTROL_AF_STATE, l.mParameters.focusState, tmp, sizeof(tmp)); + ATRACE_ASYNC_BEGIN(tmp, l.mParameters.afStateCounter); + switch (l.mParameters.focusMode) { case Parameters::FOCUS_MODE_AUTO: case Parameters::FOCUS_MODE_MACRO: @@ -1444,6 +1517,7 @@ void Camera2Client::notifyAutoFocus(uint8_t newState, int triggerId) { case ANDROID_CONTROL_AF_STATE_INACTIVE: case ANDROID_CONTROL_AF_STATE_PASSIVE_SCAN: case ANDROID_CONTROL_AF_STATE_PASSIVE_FOCUSED: + case ANDROID_CONTROL_AF_STATE_PASSIVE_UNFOCUSED: default: // Unexpected in AUTO/MACRO mode ALOGE("%s: Unexpected AF state transition in AUTO/MACRO mode: %d", @@ -1486,6 +1560,7 @@ void Camera2Client::notifyAutoFocus(uint8_t newState, int triggerId) { afInMotion = true; // no break case ANDROID_CONTROL_AF_STATE_PASSIVE_FOCUSED: + case ANDROID_CONTROL_AF_STATE_PASSIVE_UNFOCUSED: // Stop passive scan, inform upstream if (l.mParameters.enableFocusMoveMessages) { sendMovingMessage = true; @@ -1514,6 +1589,7 @@ void Camera2Client::notifyAutoFocus(uint8_t newState, int triggerId) { } } if (sendCompletedMessage) { + ATRACE_ASYNC_END(kAutofocusLabel, triggerId); SharedCameraCallbacks::Lock l(mSharedCameraCallbacks); if (l.mRemoteCallback != 0) { l.mRemoteCallback->notifyCallback(CAMERA_MSG_FOCUS, @@ -1723,4 +1799,7 @@ status_t Camera2Client::updateProcessorStream(sp<ProcessorT> processor, return res; } +const char* Camera2Client::kAutofocusLabel = "autofocus"; +const char* Camera2Client::kTakepictureLabel = "take_picture"; + } // namespace android diff --git a/services/camera/libcameraservice/Camera2Client.h b/services/camera/libcameraservice/api1/Camera2Client.h index 8ab46b1..fe0bf74 100644 --- a/services/camera/libcameraservice/Camera2Client.h +++ b/services/camera/libcameraservice/api1/Camera2Client.h @@ -17,19 +17,29 @@ #ifndef ANDROID_SERVERS_CAMERA_CAMERA2CLIENT_H #define ANDROID_SERVERS_CAMERA_CAMERA2CLIENT_H -#include "CameraDeviceBase.h" #include "CameraService.h" -#include "camera2/Parameters.h" -#include "camera2/FrameProcessor.h" -#include "camera2/StreamingProcessor.h" -#include "camera2/JpegProcessor.h" -#include "camera2/ZslProcessorInterface.h" -#include "camera2/CaptureSequencer.h" -#include "camera2/CallbackProcessor.h" -#include "Camera2ClientBase.h" +#include "common/CameraDeviceBase.h" +#include "common/Camera2ClientBase.h" +#include "api1/client2/Parameters.h" +#include "api1/client2/FrameProcessor.h" +//#include "api1/client2/StreamingProcessor.h" +//#include "api1/client2/JpegProcessor.h" +//#include "api1/client2/ZslProcessorInterface.h" +//#include "api1/client2/CaptureSequencer.h" +//#include "api1/client2/CallbackProcessor.h" namespace android { +namespace camera2 { + +class StreamingProcessor; +class JpegProcessor; +class ZslProcessorInterface; +class CaptureSequencer; +class CallbackProcessor; + +} + class IMemory; /** * Interface between android.hardware.Camera API and Camera HAL device for versions @@ -47,10 +57,12 @@ public: virtual status_t connect(const sp<ICameraClient>& client); virtual status_t lock(); virtual status_t unlock(); - virtual status_t setPreviewDisplay(const sp<Surface>& surface); - virtual status_t setPreviewTexture( + virtual status_t setPreviewTarget( const sp<IGraphicBufferProducer>& bufferProducer); virtual void setPreviewCallbackFlag(int flag); + virtual status_t setPreviewCallbackTarget( + const sp<IGraphicBufferProducer>& callbackProducer); + virtual status_t startPreview(); virtual void stopPreview(); virtual bool previewEnabled(); @@ -124,6 +136,10 @@ public: static const int32_t kCaptureRequestIdStart = 30000000; static const int32_t kCaptureRequestIdEnd = 40000000; + // Constant strings for ATRACE logging + static const char* kAutofocusLabel; + static const char* kTakepictureLabel; + private: /** ICamera interface-related private members */ typedef camera2::Parameters Parameters; diff --git a/services/camera/libcameraservice/CameraClient.cpp b/services/camera/libcameraservice/api1/CameraClient.cpp index e577fa3..bd6805d 100644 --- a/services/camera/libcameraservice/CameraClient.cpp +++ b/services/camera/libcameraservice/api1/CameraClient.cpp @@ -20,8 +20,8 @@ #include <cutils/properties.h> #include <gui/Surface.h> -#include "CameraClient.h" -#include "CameraHardwareInterface.h" +#include "api1/CameraClient.h" +#include "device1/CameraHardwareInterface.h" #include "CameraService.h" namespace android { @@ -308,26 +308,20 @@ status_t CameraClient::setPreviewWindow(const sp<IBinder>& binder, return result; } -// set the Surface that the preview will use -status_t CameraClient::setPreviewDisplay(const sp<Surface>& surface) { - LOG1("setPreviewDisplay(%p) (pid %d)", surface.get(), getCallingPid()); - - sp<IBinder> binder(surface != 0 ? surface->getIGraphicBufferProducer()->asBinder() : 0); - sp<ANativeWindow> window(surface); - return setPreviewWindow(binder, window); -} - -// set the SurfaceTextureClient that the preview will use -status_t CameraClient::setPreviewTexture( +// set the buffer consumer that the preview will use +status_t CameraClient::setPreviewTarget( const sp<IGraphicBufferProducer>& bufferProducer) { - LOG1("setPreviewTexture(%p) (pid %d)", bufferProducer.get(), + LOG1("setPreviewTarget(%p) (pid %d)", bufferProducer.get(), getCallingPid()); sp<IBinder> binder; sp<ANativeWindow> window; if (bufferProducer != 0) { binder = bufferProducer->asBinder(); - window = new Surface(bufferProducer); + // Using controlledByApp flag to ensure that the buffer queue remains in + // async mode for the old camera API, where many applications depend + // on that behavior. + window = new Surface(bufferProducer, /*controlledByApp*/ true); } return setPreviewWindow(binder, window); } @@ -347,6 +341,13 @@ void CameraClient::setPreviewCallbackFlag(int callback_flag) { } } +status_t CameraClient::setPreviewCallbackTarget( + const sp<IGraphicBufferProducer>& callbackProducer) { + (void)callbackProducer; + ALOGE("%s: Unimplemented!", __FUNCTION__); + return INVALID_OPERATION; +} + // start preview mode status_t CameraClient::startPreview() { LOG1("startPreview (pid %d)", getCallingPid()); diff --git a/services/camera/libcameraservice/CameraClient.h b/services/camera/libcameraservice/api1/CameraClient.h index 7f0cb29..4b89564 100644 --- a/services/camera/libcameraservice/CameraClient.h +++ b/services/camera/libcameraservice/api1/CameraClient.h @@ -37,9 +37,10 @@ public: virtual status_t connect(const sp<ICameraClient>& client); virtual status_t lock(); virtual status_t unlock(); - virtual status_t setPreviewDisplay(const sp<Surface>& surface); - virtual status_t setPreviewTexture(const sp<IGraphicBufferProducer>& bufferProducer); + virtual status_t setPreviewTarget(const sp<IGraphicBufferProducer>& bufferProducer); virtual void setPreviewCallbackFlag(int flag); + virtual status_t setPreviewCallbackTarget( + const sp<IGraphicBufferProducer>& callbackProducer); virtual status_t startPreview(); virtual void stopPreview(); virtual bool previewEnabled(); diff --git a/services/camera/libcameraservice/camera2/BurstCapture.cpp b/services/camera/libcameraservice/api1/client2/BurstCapture.cpp index 192d419..0bfdfd4 100644 --- a/services/camera/libcameraservice/camera2/BurstCapture.cpp +++ b/services/camera/libcameraservice/api1/client2/BurstCapture.cpp @@ -22,8 +22,8 @@ #include "BurstCapture.h" -#include "../Camera2Client.h" -#include "JpegCompressor.h" +#include "api1/Camera2Client.h" +#include "api1/client2/JpegCompressor.h" namespace android { namespace camera2 { diff --git a/services/camera/libcameraservice/camera2/BurstCapture.h b/services/camera/libcameraservice/api1/client2/BurstCapture.h index a2cc893..ea321fd 100644 --- a/services/camera/libcameraservice/camera2/BurstCapture.h +++ b/services/camera/libcameraservice/api1/client2/BurstCapture.h @@ -17,11 +17,12 @@ #ifndef ANDROID_SERVERS_CAMERA_BURST_CAPTURE_H #define ANDROID_SERVERS_CAMERA_BURST_CAPTURE_H -#include "camera/CameraMetadata.h" +#include <camera/CameraMetadata.h> #include <binder/MemoryBase.h> #include <binder/MemoryHeapBase.h> #include <gui/CpuConsumer.h> -#include "Camera2Device.h" + +#include "device2/Camera2Device.h" namespace android { diff --git a/services/camera/libcameraservice/camera2/CallbackProcessor.cpp b/services/camera/libcameraservice/api1/client2/CallbackProcessor.cpp index 522f49a..d2ac79c 100644 --- a/services/camera/libcameraservice/camera2/CallbackProcessor.cpp +++ b/services/camera/libcameraservice/api1/client2/CallbackProcessor.cpp @@ -20,11 +20,11 @@ #include <utils/Log.h> #include <utils/Trace.h> - -#include "CallbackProcessor.h" #include <gui/Surface.h> -#include "../CameraDeviceBase.h" -#include "../Camera2Client.h" + +#include "common/CameraDeviceBase.h" +#include "api1/Camera2Client.h" +#include "api1/client2/CallbackProcessor.h" #define ALIGN(x, mask) ( ((x) + (mask) - 1) & ~((mask) - 1) ) @@ -37,6 +37,7 @@ CallbackProcessor::CallbackProcessor(sp<Camera2Client> client): mDevice(client->getCameraDevice()), mId(client->getCameraId()), mCallbackAvailable(false), + mCallbackToApp(false), mCallbackStreamId(NO_STREAM) { } @@ -53,6 +54,35 @@ void CallbackProcessor::onFrameAvailable() { } } +status_t CallbackProcessor::setCallbackWindow( + sp<ANativeWindow> callbackWindow) { + ATRACE_CALL(); + status_t res; + + Mutex::Autolock l(mInputMutex); + + sp<Camera2Client> client = mClient.promote(); + if (client == 0) return OK; + sp<CameraDeviceBase> device = client->getCameraDevice(); + + // If the window is changing, clear out stream if it already exists + if (mCallbackWindow != callbackWindow && mCallbackStreamId != NO_STREAM) { + res = device->deleteStream(mCallbackStreamId); + if (res != OK) { + ALOGE("%s: Camera %d: Unable to delete old stream " + "for callbacks: %s (%d)", __FUNCTION__, + client->getCameraId(), strerror(-res), res); + return res; + } + mCallbackStreamId = NO_STREAM; + mCallbackConsumer.clear(); + } + mCallbackWindow = callbackWindow; + mCallbackToApp = (mCallbackWindow != NULL); + + return OK; +} + status_t CallbackProcessor::updateStream(const Parameters ¶ms) { ATRACE_CALL(); status_t res; @@ -67,21 +97,24 @@ status_t CallbackProcessor::updateStream(const Parameters ¶ms) { // If possible, use the flexible YUV format int32_t callbackFormat = params.previewFormat; - if (params.fastInfo.useFlexibleYuv && + if (mCallbackToApp) { + // TODO: etalvala: This should use the flexible YUV format as well, but + // need to reconcile HAL2/HAL3 requirements. + callbackFormat = HAL_PIXEL_FORMAT_YV12; + } else if(params.fastInfo.useFlexibleYuv && (params.previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP || params.previewFormat == HAL_PIXEL_FORMAT_YV12) ) { callbackFormat = HAL_PIXEL_FORMAT_YCbCr_420_888; } - if (mCallbackConsumer == 0) { - // Create CPU buffer queue endpoint. Make it async to avoid disconnect - // deadlocks. - mCallbackConsumer = new CpuConsumer(kCallbackHeapCount, - /*synchronized*/ false); + if (!mCallbackToApp && mCallbackConsumer == 0) { + // Create CPU buffer queue endpoint, since app hasn't given us one + // Make it async to avoid disconnect deadlocks + sp<BufferQueue> bq = new BufferQueue(); + mCallbackConsumer = new CpuConsumer(bq, kCallbackHeapCount); mCallbackConsumer->setFrameAvailableListener(this); mCallbackConsumer->setName(String8("Camera2Client::CallbackConsumer")); - mCallbackWindow = new Surface( - mCallbackConsumer->getProducerInterface()); + mCallbackWindow = new Surface(bq); } if (mCallbackStreamId != NO_STREAM) { @@ -106,8 +139,8 @@ status_t CallbackProcessor::updateStream(const Parameters ¶ms) { res = device->deleteStream(mCallbackStreamId); if (res != OK) { ALOGE("%s: Camera %d: Unable to delete old output stream " - "for callbacks: %s (%d)", __FUNCTION__, mId, - strerror(-res), res); + "for callbacks: %s (%d)", __FUNCTION__, + mId, strerror(-res), res); return res; } mCallbackStreamId = NO_STREAM; @@ -279,6 +312,16 @@ status_t CallbackProcessor::processNewCallback(sp<Camera2Client> &client) { return OK; } + if (imgBuffer.width != static_cast<uint32_t>(l.mParameters.previewWidth) || + imgBuffer.height != static_cast<uint32_t>(l.mParameters.previewHeight)) { + ALOGW("%s: The preview size has changed to %d x %d from %d x %d, this buffer is" + " no longer valid, dropping",__FUNCTION__, + l.mParameters.previewWidth, l.mParameters.previewHeight, + imgBuffer.width, imgBuffer.height); + mCallbackConsumer->unlockBuffer(imgBuffer); + return OK; + } + previewFormat = l.mParameters.previewFormat; useFlexibleYuv = l.mParameters.fastInfo.useFlexibleYuv && (previewFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP || diff --git a/services/camera/libcameraservice/camera2/CallbackProcessor.h b/services/camera/libcameraservice/api1/client2/CallbackProcessor.h index d851a84..613f5be 100644 --- a/services/camera/libcameraservice/camera2/CallbackProcessor.h +++ b/services/camera/libcameraservice/api1/client2/CallbackProcessor.h @@ -23,9 +23,8 @@ #include <utils/Mutex.h> #include <utils/Condition.h> #include <gui/CpuConsumer.h> -#include "Parameters.h" -#include "camera/CameraMetadata.h" -#include "Camera2Heap.h" + +#include "api1/client2/Camera2Heap.h" namespace android { @@ -34,6 +33,8 @@ class CameraDeviceBase; namespace camera2 { +class Parameters; + /*** * Still image capture output image processing */ @@ -45,6 +46,8 @@ class CallbackProcessor: void onFrameAvailable(); + // Set to NULL to disable the direct-to-app callback window + status_t setCallbackWindow(sp<ANativeWindow> callbackWindow); status_t updateStream(const Parameters ¶ms); status_t deleteStream(); int getStreamId() const; @@ -64,6 +67,9 @@ class CallbackProcessor: NO_STREAM = -1 }; + // True if mCallbackWindow is a remote consumer, false if just the local + // mCallbackConsumer + bool mCallbackToApp; int mCallbackStreamId; static const size_t kCallbackHeapCount = 6; sp<CpuConsumer> mCallbackConsumer; diff --git a/services/camera/libcameraservice/camera2/Camera2Heap.h b/services/camera/libcameraservice/api1/client2/Camera2Heap.h index 9c72d76..9c72d76 100644 --- a/services/camera/libcameraservice/camera2/Camera2Heap.h +++ b/services/camera/libcameraservice/api1/client2/Camera2Heap.h diff --git a/services/camera/libcameraservice/camera2/CaptureSequencer.cpp b/services/camera/libcameraservice/api1/client2/CaptureSequencer.cpp index e5a011c..8a4ce4e 100644 --- a/services/camera/libcameraservice/camera2/CaptureSequencer.cpp +++ b/services/camera/libcameraservice/api1/client2/CaptureSequencer.cpp @@ -22,12 +22,11 @@ #include <utils/Trace.h> #include <utils/Vector.h> -#include "CaptureSequencer.h" -#include "BurstCapture.h" -#include "../Camera2Device.h" -#include "../Camera2Client.h" -#include "Parameters.h" -#include "ZslProcessorInterface.h" +#include "api1/Camera2Client.h" +#include "api1/client2/CaptureSequencer.h" +#include "api1/client2/BurstCapture.h" +#include "api1/client2/Parameters.h" +#include "api1/client2/ZslProcessorInterface.h" namespace android { namespace camera2 { @@ -44,6 +43,7 @@ CaptureSequencer::CaptureSequencer(wp<Camera2Client> client): mShutterNotified(false), mClient(client), mCaptureState(IDLE), + mStateTransitionCount(0), mTriggerId(0), mTimeoutCount(0), mCaptureId(Camera2Client::kCaptureRequestIdStart), @@ -104,12 +104,12 @@ void CaptureSequencer::notifyAutoExposure(uint8_t newState, int triggerId) { } } -void CaptureSequencer::onFrameAvailable(int32_t frameId, +void CaptureSequencer::onFrameAvailable(int32_t requestId, const CameraMetadata &frame) { ALOGV("%s: Listener found new frame", __FUNCTION__); ATRACE_CALL(); Mutex::Autolock l(mInputMutex); - mNewFrameId = frameId; + mNewFrameId = requestId; mNewFrame = frame; if (!mNewFrameReceived) { mNewFrameReceived = true; @@ -199,8 +199,14 @@ bool CaptureSequencer::threadLoop() { Mutex::Autolock l(mStateMutex); if (currentState != mCaptureState) { + if (mCaptureState != IDLE) { + ATRACE_ASYNC_END(kStateNames[mCaptureState], mStateTransitionCount); + } mCaptureState = currentState; - ATRACE_INT("cam2_capt_state", mCaptureState); + mStateTransitionCount++; + if (mCaptureState != IDLE) { + ATRACE_ASYNC_BEGIN(kStateNames[mCaptureState], mStateTransitionCount); + } ALOGV("Camera %d: New capture state %s", client->getCameraId(), kStateNames[mCaptureState]); mStateChanged.signal(); @@ -244,6 +250,7 @@ CaptureSequencer::CaptureState CaptureSequencer::manageDone(sp<Camera2Client> &c mBusy = false; } + int takePictureCounter = 0; { SharedParameters::Lock l(client->getParameters()); switch (l.mParameters.state) { @@ -271,6 +278,7 @@ CaptureSequencer::CaptureState CaptureSequencer::manageDone(sp<Camera2Client> &c Parameters::getStateName(l.mParameters.state)); res = INVALID_OPERATION; } + takePictureCounter = l.mParameters.takePictureCounter; } sp<ZslProcessorInterface> processor = mZslProcessor.promote(); if (processor != 0) { @@ -283,6 +291,8 @@ CaptureSequencer::CaptureState CaptureSequencer::manageDone(sp<Camera2Client> &c * Fire the jpegCallback in Camera#takePicture(..., jpegCallback) */ if (mCaptureBuffer != 0 && res == OK) { + ATRACE_ASYNC_END(Camera2Client::kTakepictureLabel, takePictureCounter); + Camera2Client::SharedCameraCallbacks::Lock l(client->mSharedCameraCallbacks); ALOGV("%s: Sending still image to client", __FUNCTION__); @@ -380,11 +390,23 @@ CaptureSequencer::CaptureState CaptureSequencer::manageStandardStart( sp<Camera2Client> &client) { ATRACE_CALL(); + bool isAeConverged = false; // Get the onFrameAvailable callback when the requestID == mCaptureId client->registerFrameListener(mCaptureId, mCaptureId + 1, this); + + { + Mutex::Autolock l(mInputMutex); + isAeConverged = (mAEState == ANDROID_CONTROL_AE_STATE_CONVERGED); + } + { SharedParameters::Lock l(client->getParameters()); + // Skip AE precapture when it is already converged and not in force flash mode. + if (l.mParameters.flashMode != Parameters::FLASH_MODE_ON && isAeConverged) { + return STANDARD_CAPTURE; + } + mTriggerId = l.mParameters.precaptureTriggerCounter++; } client->getCameraDevice()->triggerPrecaptureMetering(mTriggerId); @@ -438,7 +460,8 @@ CaptureSequencer::CaptureState CaptureSequencer::manageStandardCapture( status_t res; ATRACE_CALL(); SharedParameters::Lock l(client->getParameters()); - Vector<uint8_t> outputStreams; + Vector<int32_t> outputStreams; + uint8_t captureIntent = static_cast<uint8_t>(ANDROID_CONTROL_CAPTURE_INTENT_STILL_CAPTURE); /** * Set up output streams in the request @@ -457,6 +480,7 @@ CaptureSequencer::CaptureState CaptureSequencer::manageStandardCapture( if (l.mParameters.state == Parameters::VIDEO_SNAPSHOT) { outputStreams.push(client->getRecordingStreamId()); + captureIntent = static_cast<uint8_t>(ANDROID_CONTROL_CAPTURE_INTENT_VIDEO_SNAPSHOT); } res = mCaptureRequest.update(ANDROID_REQUEST_OUTPUT_STREAMS, @@ -466,6 +490,10 @@ CaptureSequencer::CaptureState CaptureSequencer::manageStandardCapture( &mCaptureId, 1); } if (res == OK) { + res = mCaptureRequest.update(ANDROID_CONTROL_CAPTURE_INTENT, + &captureIntent, 1); + } + if (res == OK) { res = mCaptureRequest.sort(); } diff --git a/services/camera/libcameraservice/camera2/CaptureSequencer.h b/services/camera/libcameraservice/api1/client2/CaptureSequencer.h index 76750aa..9fb4ee7 100644 --- a/services/camera/libcameraservice/camera2/CaptureSequencer.h +++ b/services/camera/libcameraservice/api1/client2/CaptureSequencer.h @@ -62,7 +62,7 @@ class CaptureSequencer: void notifyAutoExposure(uint8_t newState, int triggerId); // Notifications from the frame processor - virtual void onFrameAvailable(int32_t frameId, const CameraMetadata &frame); + virtual void onFrameAvailable(int32_t requestId, const CameraMetadata &frame); // Notifications from the JPEG processor void onCaptureAvailable(nsecs_t timestamp, sp<MemoryBase> captureBuffer); @@ -100,7 +100,7 @@ class CaptureSequencer: * Internal to CaptureSequencer */ static const nsecs_t kWaitDuration = 100000000; // 100 ms - static const int kMaxTimeoutsForPrecaptureStart = 2; // 200 ms + static const int kMaxTimeoutsForPrecaptureStart = 10; // 1 sec static const int kMaxTimeoutsForPrecaptureEnd = 20; // 2 sec static const int kMaxTimeoutsForCaptureEnd = 40; // 4 sec @@ -125,6 +125,7 @@ class CaptureSequencer: NUM_CAPTURE_STATES } mCaptureState; static const char* kStateNames[]; + int mStateTransitionCount; Mutex mStateMutex; // Guards mCaptureState Condition mStateChanged; diff --git a/services/camera/libcameraservice/camera2/FrameProcessor.cpp b/services/camera/libcameraservice/api1/client2/FrameProcessor.cpp index 114a7a8..c34cb12 100644 --- a/services/camera/libcameraservice/camera2/FrameProcessor.cpp +++ b/services/camera/libcameraservice/api1/client2/FrameProcessor.cpp @@ -21,16 +21,16 @@ #include <utils/Log.h> #include <utils/Trace.h> -#include "FrameProcessor.h" -#include "../CameraDeviceBase.h" -#include "../Camera2Client.h" +#include "common/CameraDeviceBase.h" +#include "api1/Camera2Client.h" +#include "api1/client2/FrameProcessor.h" namespace android { namespace camera2 { FrameProcessor::FrameProcessor(wp<CameraDeviceBase> device, wp<Camera2Client> client) : - ProFrameProcessor(device), + FrameProcessorBase(device), mClient(client), mLastFrameNumberOfFaces(0) { @@ -58,7 +58,7 @@ bool FrameProcessor::processSingleFrame(CameraMetadata &frame, process3aState(frame, client); } - if (!ProFrameProcessor::processSingleFrame(frame, device)) { + if (!FrameProcessorBase::processSingleFrame(frame, device)) { return false; } diff --git a/services/camera/libcameraservice/camera2/FrameProcessor.h b/services/camera/libcameraservice/api1/client2/FrameProcessor.h index f480c55..2a17d45 100644 --- a/services/camera/libcameraservice/camera2/FrameProcessor.h +++ b/services/camera/libcameraservice/api1/client2/FrameProcessor.h @@ -24,7 +24,7 @@ #include <utils/List.h> #include <camera/CameraMetadata.h> -#include "ProFrameProcessor.h" +#include "common/FrameProcessorBase.h" struct camera_frame_metadata; @@ -37,7 +37,7 @@ namespace camera2 { /* Output frame metadata processing thread. This thread waits for new * frames from the device, and analyzes them as necessary. */ -class FrameProcessor : public ProFrameProcessor { +class FrameProcessor : public FrameProcessorBase { public: FrameProcessor(wp<CameraDeviceBase> device, wp<Camera2Client> client); ~FrameProcessor(); diff --git a/services/camera/libcameraservice/camera2/JpegCompressor.cpp b/services/camera/libcameraservice/api1/client2/JpegCompressor.cpp index c9af71e..2f0c67d 100644 --- a/services/camera/libcameraservice/camera2/JpegCompressor.cpp +++ b/services/camera/libcameraservice/api1/client2/JpegCompressor.cpp @@ -210,7 +210,8 @@ boolean JpegCompressor::jpegEmptyOutputBuffer(j_compress_ptr /*cinfo*/) { return true; } -void JpegCompressor::jpegTermDestination(j_compress_ptr /*cinfo*/) { +void JpegCompressor::jpegTermDestination(j_compress_ptr cinfo) { + (void) cinfo; // TODO: clean up ALOGV("%s", __FUNCTION__); ALOGV("%s: Done writing JPEG data. %d bytes left in buffer", __FUNCTION__, cinfo->dest->free_in_buffer); diff --git a/services/camera/libcameraservice/camera2/JpegCompressor.h b/services/camera/libcameraservice/api1/client2/JpegCompressor.h index 945b1de..945b1de 100644 --- a/services/camera/libcameraservice/camera2/JpegCompressor.h +++ b/services/camera/libcameraservice/api1/client2/JpegCompressor.h diff --git a/services/camera/libcameraservice/camera2/JpegProcessor.cpp b/services/camera/libcameraservice/api1/client2/JpegProcessor.cpp index f0a13ca..77d5c8a 100644 --- a/services/camera/libcameraservice/camera2/JpegProcessor.cpp +++ b/services/camera/libcameraservice/api1/client2/JpegProcessor.cpp @@ -24,12 +24,13 @@ #include <binder/MemoryHeapBase.h> #include <utils/Log.h> #include <utils/Trace.h> - -#include "JpegProcessor.h" #include <gui/Surface.h> -#include "../CameraDeviceBase.h" -#include "../Camera2Client.h" +#include "common/CameraDeviceBase.h" +#include "api1/Camera2Client.h" +#include "api1/client2/Camera2Heap.h" +#include "api1/client2/CaptureSequencer.h" +#include "api1/client2/JpegProcessor.h" namespace android { namespace camera2 { @@ -82,11 +83,11 @@ status_t JpegProcessor::updateStream(const Parameters ¶ms) { if (mCaptureConsumer == 0) { // Create CPU buffer queue endpoint - mCaptureConsumer = new CpuConsumer(1); + sp<BufferQueue> bq = new BufferQueue(); + mCaptureConsumer = new CpuConsumer(bq, 1); mCaptureConsumer->setFrameAvailableListener(this); mCaptureConsumer->setName(String8("Camera2Client::CaptureConsumer")); - mCaptureWindow = new Surface( - mCaptureConsumer->getProducerInterface()); + mCaptureWindow = new Surface(bq); // Create memory for API consumption mCaptureHeap = new MemoryHeapBase(maxJpegSize.data.i32[0], 0, "Camera2Client::CaptureHeap"); diff --git a/services/camera/libcameraservice/camera2/JpegProcessor.h b/services/camera/libcameraservice/api1/client2/JpegProcessor.h index a38611c..b2c05df 100644 --- a/services/camera/libcameraservice/camera2/JpegProcessor.h +++ b/services/camera/libcameraservice/api1/client2/JpegProcessor.h @@ -23,7 +23,7 @@ #include <utils/Mutex.h> #include <utils/Condition.h> #include <gui/CpuConsumer.h> -#include "Parameters.h" + #include "camera/CameraMetadata.h" namespace android { @@ -35,6 +35,7 @@ class MemoryHeapBase; namespace camera2 { class CaptureSequencer; +class Parameters; /*** * Still image capture output image processing diff --git a/services/camera/libcameraservice/camera2/Parameters.cpp b/services/camera/libcameraservice/api1/client2/Parameters.cpp index a248b76..8a4e75c 100644 --- a/services/camera/libcameraservice/camera2/Parameters.cpp +++ b/services/camera/libcameraservice/api1/client2/Parameters.cpp @@ -58,13 +58,13 @@ status_t Parameters::initialize(const CameraMetadata *info) { res = buildQuirks(); if (res != OK) return res; - camera_metadata_ro_entry_t availableProcessedSizes = - staticInfo(ANDROID_SCALER_AVAILABLE_PROCESSED_SIZES, 2); - if (!availableProcessedSizes.count) return NO_INIT; + const Size MAX_PREVIEW_SIZE = { MAX_PREVIEW_WIDTH, MAX_PREVIEW_HEIGHT }; + res = getFilteredPreviewSizes(MAX_PREVIEW_SIZE, &availablePreviewSizes); + if (res != OK) return res; // TODO: Pick more intelligently - previewWidth = availableProcessedSizes.data.i32[0]; - previewHeight = availableProcessedSizes.data.i32[1]; + previewWidth = availablePreviewSizes[0].width; + previewHeight = availablePreviewSizes[0].height; videoWidth = previewWidth; videoHeight = previewHeight; @@ -75,12 +75,13 @@ status_t Parameters::initialize(const CameraMetadata *info) { previewWidth, previewHeight)); { String8 supportedPreviewSizes; - for (size_t i=0; i < availableProcessedSizes.count; i += 2) { + for (size_t i = 0; i < availablePreviewSizes.size(); i++) { if (i != 0) supportedPreviewSizes += ","; supportedPreviewSizes += String8::format("%dx%d", - availableProcessedSizes.data.i32[i], - availableProcessedSizes.data.i32[i+1]); + availablePreviewSizes[i].width, + availablePreviewSizes[i].height); } + ALOGV("Supported preview sizes are: %s", supportedPreviewSizes.string()); params.set(CameraParameters::KEY_SUPPORTED_PREVIEW_SIZES, supportedPreviewSizes); params.set(CameraParameters::KEY_SUPPORTED_VIDEO_SIZES, @@ -183,6 +184,7 @@ status_t Parameters::initialize(const CameraMetadata *info) { // NOTE: Not scaled like FPS range values are. previewFps = fpsFromRange(previewFpsRange[0], previewFpsRange[1]); + lastSetPreviewFps = previewFps; params.set(CameraParameters::KEY_PREVIEW_FRAME_RATE, previewFps); @@ -248,9 +250,17 @@ status_t Parameters::initialize(const CameraMetadata *info) { staticInfo(ANDROID_JPEG_AVAILABLE_THUMBNAIL_SIZES, 4); if (!availableJpegThumbnailSizes.count) return NO_INIT; - // TODO: Pick default thumbnail size sensibly - jpegThumbSize[0] = availableJpegThumbnailSizes.data.i32[0]; - jpegThumbSize[1] = availableJpegThumbnailSizes.data.i32[1]; + // Pick the largest thumbnail size that matches still image aspect ratio. + ALOG_ASSERT(pictureWidth > 0 && pictureHeight > 0, + "Invalid picture size, %d x %d", pictureWidth, pictureHeight); + float picAspectRatio = static_cast<float>(pictureWidth) / pictureHeight; + Size thumbnailSize = + getMaxSizeForRatio( + picAspectRatio, + &availableJpegThumbnailSizes.data.i32[0], + availableJpegThumbnailSizes.count); + jpegThumbSize[0] = thumbnailSize.width; + jpegThumbSize[1] = thumbnailSize.height; params.set(CameraParameters::KEY_JPEG_THUMBNAIL_WIDTH, jpegThumbSize[0]); @@ -292,8 +302,11 @@ status_t Parameters::initialize(const CameraMetadata *info) { CameraParameters::WHITE_BALANCE_AUTO); camera_metadata_ro_entry_t availableWhiteBalanceModes = - staticInfo(ANDROID_CONTROL_AWB_AVAILABLE_MODES); - { + staticInfo(ANDROID_CONTROL_AWB_AVAILABLE_MODES, 0, 0, false); + if (!availableWhiteBalanceModes.count) { + params.set(CameraParameters::KEY_SUPPORTED_WHITE_BALANCE, + CameraParameters::WHITE_BALANCE_AUTO); + } else { String8 supportedWhiteBalance; bool addComma = false; for (size_t i=0; i < availableWhiteBalanceModes.count; i++) { @@ -353,9 +366,11 @@ status_t Parameters::initialize(const CameraMetadata *info) { CameraParameters::EFFECT_NONE); camera_metadata_ro_entry_t availableEffects = - staticInfo(ANDROID_CONTROL_AVAILABLE_EFFECTS); - if (!availableEffects.count) return NO_INIT; - { + staticInfo(ANDROID_CONTROL_AVAILABLE_EFFECTS, 0, 0, false); + if (!availableEffects.count) { + params.set(CameraParameters::KEY_SUPPORTED_EFFECTS, + CameraParameters::EFFECT_NONE); + } else { String8 supportedEffects; bool addComma = false; for (size_t i=0; i < availableEffects.count; i++) { @@ -413,9 +428,11 @@ status_t Parameters::initialize(const CameraMetadata *info) { CameraParameters::ANTIBANDING_AUTO); camera_metadata_ro_entry_t availableAntibandingModes = - staticInfo(ANDROID_CONTROL_AE_AVAILABLE_ANTIBANDING_MODES); - if (!availableAntibandingModes.count) return NO_INIT; - { + staticInfo(ANDROID_CONTROL_AE_AVAILABLE_ANTIBANDING_MODES, 0, 0, false); + if (!availableAntibandingModes.count) { + params.set(CameraParameters::KEY_SUPPORTED_ANTIBANDING, + CameraParameters::ANTIBANDING_OFF); + } else { String8 supportedAntibanding; bool addComma = false; for (size_t i=0; i < availableAntibandingModes.count; i++) { @@ -455,9 +472,10 @@ status_t Parameters::initialize(const CameraMetadata *info) { CameraParameters::SCENE_MODE_AUTO); camera_metadata_ro_entry_t availableSceneModes = - staticInfo(ANDROID_CONTROL_AVAILABLE_SCENE_MODES); - if (!availableSceneModes.count) return NO_INIT; - { + staticInfo(ANDROID_CONTROL_AVAILABLE_SCENE_MODES, 0, 0, false); + if (!availableSceneModes.count) { + params.remove(CameraParameters::KEY_SCENE_MODE); + } else { String8 supportedSceneModes(CameraParameters::SCENE_MODE_AUTO); bool addComma = true; bool noSceneModes = false; @@ -548,15 +566,17 @@ status_t Parameters::initialize(const CameraMetadata *info) { } } + bool isFlashAvailable = false; camera_metadata_ro_entry_t flashAvailable = - staticInfo(ANDROID_FLASH_INFO_AVAILABLE, 1, 1); - if (!flashAvailable.count) return NO_INIT; + staticInfo(ANDROID_FLASH_INFO_AVAILABLE, 0, 1, false); + if (flashAvailable.count) { + isFlashAvailable = flashAvailable.data.u8[0]; + } camera_metadata_ro_entry_t availableAeModes = - staticInfo(ANDROID_CONTROL_AE_AVAILABLE_MODES); - if (!availableAeModes.count) return NO_INIT; + staticInfo(ANDROID_CONTROL_AE_AVAILABLE_MODES, 0, 0, false); - if (flashAvailable.data.u8[0]) { + if (isFlashAvailable) { flashMode = Parameters::FLASH_MODE_OFF; params.set(CameraParameters::KEY_FLASH_MODE, CameraParameters::FLASH_MODE_OFF); @@ -585,14 +605,12 @@ status_t Parameters::initialize(const CameraMetadata *info) { } camera_metadata_ro_entry_t minFocusDistance = - staticInfo(ANDROID_LENS_INFO_MINIMUM_FOCUS_DISTANCE, 1, 1); - if (!minFocusDistance.count) return NO_INIT; + staticInfo(ANDROID_LENS_INFO_MINIMUM_FOCUS_DISTANCE, 0, 1, false); camera_metadata_ro_entry_t availableAfModes = - staticInfo(ANDROID_CONTROL_AF_AVAILABLE_MODES); - if (!availableAfModes.count) return NO_INIT; + staticInfo(ANDROID_CONTROL_AF_AVAILABLE_MODES, 0, 0, false); - if (minFocusDistance.data.f[0] == 0) { + if (!minFocusDistance.count || minFocusDistance.data.f[0] == 0) { // Fixed-focus lens focusMode = Parameters::FOCUS_MODE_FIXED; params.set(CameraParameters::KEY_FOCUS_MODE, @@ -662,7 +680,7 @@ status_t Parameters::initialize(const CameraMetadata *info) { focusingAreas.add(Parameters::Area(0,0,0,0,0)); camera_metadata_ro_entry_t availableFocalLengths = - staticInfo(ANDROID_LENS_INFO_AVAILABLE_FOCAL_LENGTHS); + staticInfo(ANDROID_LENS_INFO_AVAILABLE_FOCAL_LENGTHS, 0, 0, false); if (!availableFocalLengths.count) return NO_INIT; float minFocalLength = availableFocalLengths.data.f[0]; @@ -768,8 +786,8 @@ status_t Parameters::initialize(const CameraMetadata *info) { CameraParameters::FALSE); camera_metadata_ro_entry_t availableVideoStabilizationModes = - staticInfo(ANDROID_CONTROL_AVAILABLE_VIDEO_STABILIZATION_MODES); - if (!availableVideoStabilizationModes.count) return NO_INIT; + staticInfo(ANDROID_CONTROL_AVAILABLE_VIDEO_STABILIZATION_MODES, 0, 0, + false); if (availableVideoStabilizationModes.count > 1) { params.set(CameraParameters::KEY_VIDEO_STABILIZATION_SUPPORTED, @@ -787,29 +805,25 @@ status_t Parameters::initialize(const CameraMetadata *info) { enableFocusMoveMessages = false; afTriggerCounter = 1; + afStateCounter = 0; currentAfTriggerId = -1; afInMotion = false; precaptureTriggerCounter = 1; + takePictureCounter = 0; + previewCallbackFlags = 0; previewCallbackOneShot = false; + previewCallbackSurface = false; - camera_metadata_ro_entry_t supportedHardwareLevel = - staticInfo(ANDROID_INFO_SUPPORTED_HARDWARE_LEVEL); - if (!supportedHardwareLevel.count || (supportedHardwareLevel.data.u8[0] == - ANDROID_INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED)) { - ALOGI("Camera %d: ZSL mode disabled for limited mode HALs", cameraId); + char value[PROPERTY_VALUE_MAX]; + property_get("camera.disable_zsl_mode", value, "0"); + if (!strcmp(value,"1")) { + ALOGI("Camera %d: Disabling ZSL mode", cameraId); zslMode = false; } else { - char value[PROPERTY_VALUE_MAX]; - property_get("camera.disable_zsl_mode", value, "0"); - if (!strcmp(value,"1")) { - ALOGI("Camera %d: Disabling ZSL mode", cameraId); - zslMode = false; - } else { - zslMode = true; - } + zslMode = true; } lightFx = LIGHTFX_NONE; @@ -828,14 +842,50 @@ String8 Parameters::get() const { status_t Parameters::buildFastInfo() { camera_metadata_ro_entry_t activeArraySize = - staticInfo(ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE, 2, 2); + staticInfo(ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE, 2, 4); if (!activeArraySize.count) return NO_INIT; - int32_t arrayWidth = activeArraySize.data.i32[0]; - int32_t arrayHeight = activeArraySize.data.i32[1]; + int32_t arrayWidth; + int32_t arrayHeight; + if (activeArraySize.count == 2) { + ALOGW("%s: Camera %d: activeArraySize is missing xmin/ymin!", + __FUNCTION__, cameraId); + arrayWidth = activeArraySize.data.i32[0]; + arrayHeight = activeArraySize.data.i32[1]; + } else if (activeArraySize.count == 4) { + arrayWidth = activeArraySize.data.i32[2]; + arrayHeight = activeArraySize.data.i32[3]; + } else return NO_INIT; + + // We'll set the target FPS range for still captures to be as wide + // as possible to give the HAL maximum latitude for exposure selection + camera_metadata_ro_entry_t availableFpsRanges = + staticInfo(ANDROID_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES, 2); + if (availableFpsRanges.count < 2 || availableFpsRanges.count % 2 != 0) { + return NO_INIT; + } + + int32_t bestStillCaptureFpsRange[2] = { + availableFpsRanges.data.i32[0], availableFpsRanges.data.i32[1] + }; + int32_t curRange = + bestStillCaptureFpsRange[1] - bestStillCaptureFpsRange[0]; + for (size_t i = 2; i < availableFpsRanges.count; i += 2) { + int32_t nextRange = + availableFpsRanges.data.i32[i + 1] - + availableFpsRanges.data.i32[i]; + if ( (nextRange > curRange) || // Maximize size of FPS range first + (nextRange == curRange && // Then minimize low-end FPS + bestStillCaptureFpsRange[0] > availableFpsRanges.data.i32[i])) { + + bestStillCaptureFpsRange[0] = availableFpsRanges.data.i32[i]; + bestStillCaptureFpsRange[1] = availableFpsRanges.data.i32[i + 1]; + curRange = nextRange; + } + } camera_metadata_ro_entry_t availableFaceDetectModes = - staticInfo(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES); - if (!availableFaceDetectModes.count) return NO_INIT; + staticInfo(ANDROID_STATISTICS_INFO_AVAILABLE_FACE_DETECT_MODES, 0, 0, + false); uint8_t bestFaceDetectMode = ANDROID_STATISTICS_FACE_DETECT_MODE_OFF; @@ -862,19 +912,21 @@ status_t Parameters::buildFastInfo() { } } + int32_t maxFaces = 0; camera_metadata_ro_entry_t maxFacesDetected = - staticInfo(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT, 1, 1); - if (!maxFacesDetected.count) return NO_INIT; - - int32_t maxFaces = maxFacesDetected.data.i32[0]; + staticInfo(ANDROID_STATISTICS_INFO_MAX_FACE_COUNT, 0, 1, false); + if (maxFacesDetected.count) { + maxFaces = maxFacesDetected.data.i32[0]; + } camera_metadata_ro_entry_t availableSceneModes = - staticInfo(ANDROID_CONTROL_AVAILABLE_SCENE_MODES); + staticInfo(ANDROID_CONTROL_AVAILABLE_SCENE_MODES, 0, 0, false); camera_metadata_ro_entry_t sceneModeOverrides = - staticInfo(ANDROID_CONTROL_SCENE_MODE_OVERRIDES); + staticInfo(ANDROID_CONTROL_SCENE_MODE_OVERRIDES, 0, 0, false); camera_metadata_ro_entry_t minFocusDistance = - staticInfo(ANDROID_LENS_INFO_MINIMUM_FOCUS_DISTANCE); - bool fixedLens = (minFocusDistance.data.f[0] == 0); + staticInfo(ANDROID_LENS_INFO_MINIMUM_FOCUS_DISTANCE, 0, 0, false); + bool fixedLens = minFocusDistance.count == 0 || + minFocusDistance.data.f[0] == 0; camera_metadata_ro_entry_t availableFocalLengths = staticInfo(ANDROID_LENS_INFO_AVAILABLE_FOCAL_LENGTHS); @@ -950,6 +1002,8 @@ status_t Parameters::buildFastInfo() { fastInfo.arrayWidth = arrayWidth; fastInfo.arrayHeight = arrayHeight; + fastInfo.bestStillCaptureFpsRange[0] = bestStillCaptureFpsRange[0]; + fastInfo.bestStillCaptureFpsRange[1] = bestStillCaptureFpsRange[1]; fastInfo.bestFaceDetectMode = bestFaceDetectMode; fastInfo.maxFaces = maxFaces; @@ -1052,15 +1106,13 @@ status_t Parameters::set(const String8& paramString) { validatedParams.previewWidth, validatedParams.previewHeight); return BAD_VALUE; } - camera_metadata_ro_entry_t availablePreviewSizes = - staticInfo(ANDROID_SCALER_AVAILABLE_PROCESSED_SIZES); - for (i = 0; i < availablePreviewSizes.count; i += 2 ) { - if ((availablePreviewSizes.data.i32[i] == + for (i = 0; i < availablePreviewSizes.size(); i++) { + if ((availablePreviewSizes[i].width == validatedParams.previewWidth) && - (availablePreviewSizes.data.i32[i+1] == + (availablePreviewSizes[i].height == validatedParams.previewHeight)) break; } - if (i == availablePreviewSizes.count) { + if (i == availablePreviewSizes.size()) { ALOGE("%s: Requested preview size %d x %d is not supported", __FUNCTION__, validatedParams.previewWidth, validatedParams.previewHeight); @@ -1104,6 +1156,12 @@ status_t Parameters::set(const String8& paramString) { validatedParams.previewFps = fpsFromRange(validatedParams.previewFpsRange[0], validatedParams.previewFpsRange[1]); + + // Update our last-seen single preview FPS, needed for disambiguating + // when the application is intending to use the deprecated single-FPS + // setting vs. the range FPS setting + validatedParams.lastSetPreviewFps = newParams.getPreviewFrameRate(); + newParams.setPreviewFrameRate(validatedParams.previewFps); } @@ -1139,12 +1197,15 @@ status_t Parameters::set(const String8& paramString) { } } - // PREVIEW_FRAME_RATE - // Deprecated, only use if the preview fps range is unchanged this time. - // The single-value FPS is the same as the minimum of the range. + // PREVIEW_FRAME_RATE Deprecated, only use if the preview fps range is + // unchanged this time. The single-value FPS is the same as the minimum of + // the range. To detect whether the application has changed the value of + // previewFps, compare against their last-set preview FPS instead of the + // single FPS we may have synthesized from a range FPS set. if (!fpsRangeChanged) { validatedParams.previewFps = newParams.getPreviewFrameRate(); - if (validatedParams.previewFps != previewFps || recordingHintChanged) { + if (validatedParams.previewFps != lastSetPreviewFps || + recordingHintChanged) { camera_metadata_ro_entry_t availableFrameRates = staticInfo(ANDROID_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES); /** @@ -1215,7 +1276,10 @@ status_t Parameters::set(const String8& paramString) { String8::format("%d,%d", validatedParams.previewFpsRange[0] * kFpsToApiScale, validatedParams.previewFpsRange[1] * kFpsToApiScale)); - + // Update our last-seen single preview FPS, needed for disambiguating + // when the application is intending to use the deprecated single-FPS + // setting vs. the range FPS setting + validatedParams.lastSetPreviewFps = validatedParams.previewFps; } // PICTURE_SIZE @@ -1465,7 +1529,7 @@ status_t Parameters::set(const String8& paramString) { } if (validatedParams.wbMode != wbMode) { camera_metadata_ro_entry_t availableWbModes = - staticInfo(ANDROID_CONTROL_AWB_AVAILABLE_MODES); + staticInfo(ANDROID_CONTROL_AWB_AVAILABLE_MODES, 0, 0, false); for (i = 0; i < availableWbModes.count; i++) { if (validatedParams.wbMode == availableWbModes.data.u8[i]) break; } @@ -1496,8 +1560,9 @@ status_t Parameters::set(const String8& paramString) { validatedParams.currentAfTriggerId = -1; if (validatedParams.focusMode != Parameters::FOCUS_MODE_FIXED) { camera_metadata_ro_entry_t minFocusDistance = - staticInfo(ANDROID_LENS_INFO_MINIMUM_FOCUS_DISTANCE); - if (minFocusDistance.data.f[0] == 0) { + staticInfo(ANDROID_LENS_INFO_MINIMUM_FOCUS_DISTANCE, 0, 0, + false); + if (minFocusDistance.count && minFocusDistance.data.f[0] == 0) { ALOGE("%s: Requested focus mode \"%s\" is not available: " "fixed focus lens", __FUNCTION__, @@ -1597,15 +1662,13 @@ status_t Parameters::set(const String8& paramString) { __FUNCTION__); return BAD_VALUE; } - camera_metadata_ro_entry_t availableVideoSizes = - staticInfo(ANDROID_SCALER_AVAILABLE_PROCESSED_SIZES); - for (i = 0; i < availableVideoSizes.count; i += 2 ) { - if ((availableVideoSizes.data.i32[i] == + for (i = 0; i < availablePreviewSizes.size(); i++) { + if ((availablePreviewSizes[i].width == validatedParams.videoWidth) && - (availableVideoSizes.data.i32[i+1] == + (availablePreviewSizes[i].height == validatedParams.videoHeight)) break; } - if (i == availableVideoSizes.count) { + if (i == availablePreviewSizes.size()) { ALOGE("%s: Requested video size %d x %d is not supported", __FUNCTION__, validatedParams.videoWidth, validatedParams.videoHeight); @@ -1617,7 +1680,8 @@ status_t Parameters::set(const String8& paramString) { validatedParams.videoStabilization = boolFromString( newParams.get(CameraParameters::KEY_VIDEO_STABILIZATION) ); camera_metadata_ro_entry_t availableVideoStabilizationModes = - staticInfo(ANDROID_CONTROL_AVAILABLE_VIDEO_STABILIZATION_MODES); + staticInfo(ANDROID_CONTROL_AVAILABLE_VIDEO_STABILIZATION_MODES, 0, 0, + false); if (validatedParams.videoStabilization && availableVideoStabilizationModes.count == 1) { ALOGE("%s: Video stabilization not supported", __FUNCTION__); @@ -1690,8 +1754,15 @@ status_t Parameters::updateRequest(CameraMetadata *request) const { &metadataMode, 1); if (res != OK) return res; - res = request->update(ANDROID_CONTROL_AE_TARGET_FPS_RANGE, - previewFpsRange, 2); + camera_metadata_entry_t intent = + request->find(ANDROID_CONTROL_CAPTURE_INTENT); + if (intent.data.u8[0] == ANDROID_CONTROL_CAPTURE_INTENT_STILL_CAPTURE) { + res = request->update(ANDROID_CONTROL_AE_TARGET_FPS_RANGE, + fastInfo.bestStillCaptureFpsRange, 2); + } else { + res = request->update(ANDROID_CONTROL_AE_TARGET_FPS_RANGE, + previewFpsRange, 2); + } if (res != OK) return res; uint8_t reqWbLock = autoWhiteBalanceLock ? @@ -2425,6 +2496,64 @@ int Parameters::normalizedYToArray(int y) const { return cropYToArray(normalizedYToCrop(y)); } +status_t Parameters::getFilteredPreviewSizes(Size limit, Vector<Size> *sizes) { + if (info == NULL) { + ALOGE("%s: Static metadata is not initialized", __FUNCTION__); + return NO_INIT; + } + if (sizes == NULL) { + ALOGE("%s: Input size is null", __FUNCTION__); + return BAD_VALUE; + } + + const size_t SIZE_COUNT = sizeof(Size) / sizeof(int); + camera_metadata_ro_entry_t availableProcessedSizes = + staticInfo(ANDROID_SCALER_AVAILABLE_PROCESSED_SIZES, SIZE_COUNT); + if (availableProcessedSizes.count < SIZE_COUNT) return BAD_VALUE; + + Size previewSize; + for (size_t i = 0; i < availableProcessedSizes.count; i += SIZE_COUNT) { + previewSize.width = availableProcessedSizes.data.i32[i]; + previewSize.height = availableProcessedSizes.data.i32[i+1]; + // Need skip the preview sizes that are too large. + if (previewSize.width <= limit.width && + previewSize.height <= limit.height) { + sizes->push(previewSize); + } + } + if (sizes->isEmpty()) { + ALOGE("generated preview size list is empty!!"); + return BAD_VALUE; + } + return OK; +} + +Parameters::Size Parameters::getMaxSizeForRatio( + float ratio, const int32_t* sizeArray, size_t count) { + ALOG_ASSERT(sizeArray != NULL, "size array shouldn't be NULL"); + ALOG_ASSERT(count >= 2 && count % 2 == 0, "count must be a positive even number"); + + Size maxSize = {0, 0}; + for (size_t i = 0; i < count; i += 2) { + if (sizeArray[i] > 0 && sizeArray[i+1] > 0) { + float curRatio = static_cast<float>(sizeArray[i]) / sizeArray[i+1]; + if (fabs(curRatio - ratio) < ASPECT_RATIO_TOLERANCE && maxSize.width < sizeArray[i]) { + maxSize.width = sizeArray[i]; + maxSize.height = sizeArray[i+1]; + } + } + } + + if (maxSize.width == 0 || maxSize.height == 0) { + maxSize.width = sizeArray[0]; + maxSize.height = sizeArray[1]; + ALOGW("Unable to find the size to match the given aspect ratio %f." + "Fall back to %d x %d", ratio, maxSize.width, maxSize.height); + } + + return maxSize; +} + Parameters::CropRegion Parameters::calculateCropRegion( Parameters::CropRegion::Outputs outputs) const { @@ -2544,10 +2673,6 @@ status_t Parameters::calculatePictureFovs(float *horizFov, float *vertFov) staticInfo(ANDROID_SENSOR_INFO_PHYSICAL_SIZE, 2, 2); if (!sensorSize.count) return NO_INIT; - camera_metadata_ro_entry_t availableFocalLengths = - staticInfo(ANDROID_LENS_INFO_AVAILABLE_FOCAL_LENGTHS); - if (!availableFocalLengths.count) return NO_INIT; - float arrayAspect = static_cast<float>(fastInfo.arrayWidth) / fastInfo.arrayHeight; float stillAspect = static_cast<float>(pictureWidth) / pictureHeight; diff --git a/services/camera/libcameraservice/camera2/Parameters.h b/services/camera/libcameraservice/api1/client2/Parameters.h index be05b54..bcbdb99 100644 --- a/services/camera/libcameraservice/camera2/Parameters.h +++ b/services/camera/libcameraservice/api1/client2/Parameters.h @@ -46,6 +46,7 @@ struct Parameters { int previewWidth, previewHeight; int32_t previewFpsRange[2]; + int lastSetPreviewFps; // the last single FPS value seen in a set call int previewFps; // deprecated, here only for tracking changes int previewFormat; @@ -105,6 +106,11 @@ struct Parameters { }; Vector<Area> focusingAreas; + struct Size { + int32_t width; + int32_t height; + }; + int32_t exposureCompensation; bool autoExposureLock; bool autoWhiteBalanceLock; @@ -135,13 +141,17 @@ struct Parameters { bool enableFocusMoveMessages; int afTriggerCounter; + int afStateCounter; int currentAfTriggerId; bool afInMotion; int precaptureTriggerCounter; + int takePictureCounter; + uint32_t previewCallbackFlags; bool previewCallbackOneShot; + bool previewCallbackSurface; bool zslMode; @@ -158,6 +168,11 @@ struct Parameters { // Number of zoom steps to simulate static const unsigned int NUM_ZOOM_STEPS = 100; + // Max preview size allowed + static const unsigned int MAX_PREVIEW_WIDTH = 1920; + static const unsigned int MAX_PREVIEW_HEIGHT = 1080; + // Aspect ratio tolerance + static const float ASPECT_RATIO_TOLERANCE = 0.001; // Full static camera info, object owned by someone else, such as // Camera2Device. @@ -170,6 +185,7 @@ struct Parameters { struct DeviceInfo { int32_t arrayWidth; int32_t arrayHeight; + int32_t bestStillCaptureFpsRange[2]; uint8_t bestFaceDetectMode; int32_t maxFaces; struct OverrideModes { @@ -316,6 +332,12 @@ private: int cropYToNormalized(int y) const; int normalizedXToCrop(int x) const; int normalizedYToCrop(int y) const; + + Vector<Size> availablePreviewSizes; + // Get size list (that are no larger than limit) from static metadata. + status_t getFilteredPreviewSizes(Size limit, Vector<Size> *sizes); + // Get max size (from the size array) that matches the given aspect ratio. + Size getMaxSizeForRatio(float ratio, const int32_t* sizeArray, size_t count); }; // This class encapsulates the Parameters class so that it can only be accessed diff --git a/services/camera/libcameraservice/camera2/StreamingProcessor.cpp b/services/camera/libcameraservice/api1/client2/StreamingProcessor.cpp index f7a6be7..6076dae 100644 --- a/services/camera/libcameraservice/camera2/StreamingProcessor.cpp +++ b/services/camera/libcameraservice/api1/client2/StreamingProcessor.cpp @@ -30,10 +30,10 @@ #include <gui/Surface.h> #include <media/hardware/MetadataBufferType.h> -#include "StreamingProcessor.h" -#include "Camera2Heap.h" -#include "../Camera2Client.h" -#include "../CameraDeviceBase.h" +#include "common/CameraDeviceBase.h" +#include "api1/Camera2Client.h" +#include "api1/client2/StreamingProcessor.h" +#include "api1/client2/Camera2Heap.h" namespace android { namespace camera2 { @@ -319,14 +319,13 @@ status_t StreamingProcessor::updateRecordingStream(const Parameters ¶ms) { // Create CPU buffer queue endpoint. We need one more buffer here so that we can // always acquire and free a buffer when the heap is full; otherwise the consumer // will have buffers in flight we'll never clear out. - mRecordingConsumer = new BufferItemConsumer( + sp<BufferQueue> bq = new BufferQueue(); + mRecordingConsumer = new BufferItemConsumer(bq, GRALLOC_USAGE_HW_VIDEO_ENCODER, - mRecordingHeapCount + 1, - true); + mRecordingHeapCount + 1); mRecordingConsumer->setFrameAvailableListener(this); mRecordingConsumer->setName(String8("Camera2-RecordingConsumer")); - mRecordingWindow = new Surface( - mRecordingConsumer->getProducerInterface()); + mRecordingWindow = new Surface(bq); newConsumer = true; // Allocate memory later, since we don't know buffer size until receipt } @@ -413,7 +412,7 @@ int StreamingProcessor::getRecordingStreamId() const { } status_t StreamingProcessor::startStream(StreamType type, - const Vector<uint8_t> &outputStreams) { + const Vector<int32_t> &outputStreams) { ATRACE_CALL(); status_t res; @@ -617,7 +616,7 @@ status_t StreamingProcessor::processRecordingFrame() { if (client == 0) { // Discard frames during shutdown BufferItemConsumer::BufferItem imgBuffer; - res = mRecordingConsumer->acquireBuffer(&imgBuffer); + res = mRecordingConsumer->acquireBuffer(&imgBuffer, 0); if (res != OK) { if (res != BufferItemConsumer::NO_BUFFER_AVAILABLE) { ALOGE("%s: Camera %d: Can't acquire recording buffer: %s (%d)", @@ -635,7 +634,7 @@ status_t StreamingProcessor::processRecordingFrame() { SharedParameters::Lock l(client->getParameters()); Mutex::Autolock m(mMutex); BufferItemConsumer::BufferItem imgBuffer; - res = mRecordingConsumer->acquireBuffer(&imgBuffer); + res = mRecordingConsumer->acquireBuffer(&imgBuffer, 0); if (res != OK) { if (res != BufferItemConsumer::NO_BUFFER_AVAILABLE) { ALOGE("%s: Camera %d: Can't acquire recording buffer: %s (%d)", @@ -831,8 +830,8 @@ void StreamingProcessor::releaseAllRecordingFramesLocked() { mRecordingHeapFree = mRecordingHeapCount; } -bool StreamingProcessor::isStreamActive(const Vector<uint8_t> &streams, - uint8_t recordingStreamId) { +bool StreamingProcessor::isStreamActive(const Vector<int32_t> &streams, + int32_t recordingStreamId) { for (size_t i = 0; i < streams.size(); i++) { if (streams[i] == recordingStreamId) { return true; diff --git a/services/camera/libcameraservice/camera2/StreamingProcessor.h b/services/camera/libcameraservice/api1/client2/StreamingProcessor.h index 3ec2df7..833bb8f 100644 --- a/services/camera/libcameraservice/camera2/StreamingProcessor.h +++ b/services/camera/libcameraservice/api1/client2/StreamingProcessor.h @@ -21,7 +21,6 @@ #include <utils/String16.h> #include <gui/BufferItemConsumer.h> -#include "Parameters.h" #include "camera/CameraMetadata.h" namespace android { @@ -32,6 +31,7 @@ class IMemory; namespace camera2 { +class Parameters; class Camera2Heap; /** @@ -64,7 +64,7 @@ class StreamingProcessor: RECORD }; status_t startStream(StreamType type, - const Vector<uint8_t> &outputStreams); + const Vector<int32_t> &outputStreams); // Toggle between paused and unpaused. Stream must be started first. status_t togglePauseStream(bool pause); @@ -97,7 +97,7 @@ class StreamingProcessor: StreamType mActiveRequest; bool mPaused; - Vector<uint8_t> mActiveStreamIds; + Vector<int32_t> mActiveStreamIds; // Preview-related members int32_t mPreviewRequestId; @@ -132,8 +132,8 @@ class StreamingProcessor: void releaseAllRecordingFramesLocked(); // Determine if the specified stream is currently in use - static bool isStreamActive(const Vector<uint8_t> &streams, - uint8_t recordingStreamId); + static bool isStreamActive(const Vector<int32_t> &streams, + int32_t recordingStreamId); }; diff --git a/services/camera/libcameraservice/camera2/ZslProcessor.cpp b/services/camera/libcameraservice/api1/client2/ZslProcessor.cpp index 94059cd..4207ba9 100644 --- a/services/camera/libcameraservice/camera2/ZslProcessor.cpp +++ b/services/camera/libcameraservice/api1/client2/ZslProcessor.cpp @@ -27,12 +27,12 @@ #include <utils/Log.h> #include <utils/Trace.h> - -#include "ZslProcessor.h" #include <gui/Surface.h> -#include "../CameraDeviceBase.h" -#include "../Camera2Client.h" +#include "common/CameraDeviceBase.h" +#include "api1/Camera2Client.h" +#include "api1/client2/CaptureSequencer.h" +#include "api1/client2/ZslProcessor.h" namespace android { namespace camera2 { @@ -71,7 +71,7 @@ void ZslProcessor::onFrameAvailable() { } } -void ZslProcessor::onFrameAvailable(int32_t /*frameId*/, +void ZslProcessor::onFrameAvailable(int32_t /*requestId*/, const CameraMetadata &frame) { Mutex::Autolock l(mInputMutex); camera_metadata_ro_entry_t entry; @@ -128,14 +128,13 @@ status_t ZslProcessor::updateStream(const Parameters ¶ms) { if (mZslConsumer == 0) { // Create CPU buffer queue endpoint - mZslConsumer = new BufferItemConsumer( + sp<BufferQueue> bq = new BufferQueue(); + mZslConsumer = new BufferItemConsumer(bq, GRALLOC_USAGE_HW_CAMERA_ZSL, - kZslBufferDepth, - true); + kZslBufferDepth); mZslConsumer->setFrameAvailableListener(this); mZslConsumer->setName(String8("Camera2Client::ZslConsumer")); - mZslWindow = new Surface( - mZslConsumer->getProducerInterface()); + mZslWindow = new Surface(bq); } if (mZslStreamId != NO_STREAM) { @@ -301,12 +300,12 @@ status_t ZslProcessor::pushToReprocess(int32_t requestId) { uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS; res = request.update(ANDROID_REQUEST_TYPE, &requestType, 1); - uint8_t inputStreams[1] = - { static_cast<uint8_t>(mZslReprocessStreamId) }; + int32_t inputStreams[1] = + { mZslReprocessStreamId }; if (res == OK) request.update(ANDROID_REQUEST_INPUT_STREAMS, inputStreams, 1); - uint8_t outputStreams[1] = - { static_cast<uint8_t>(client->getCaptureStreamId()) }; + int32_t outputStreams[1] = + { client->getCaptureStreamId() }; if (res == OK) request.update(ANDROID_REQUEST_OUTPUT_STREAMS, outputStreams, 1); res = request.update(ANDROID_REQUEST_ID, @@ -426,7 +425,7 @@ status_t ZslProcessor::processNewZslBuffer() { } ALOGVV("Trying to get next buffer"); BufferItemConsumer::BufferItem item; - res = zslConsumer->acquireBuffer(&item); + res = zslConsumer->acquireBuffer(&item, 0); if (res != OK) { if (res != BufferItemConsumer::NO_BUFFER_AVAILABLE) { ALOGE("%s: Camera %d: Error receiving ZSL image buffer: " diff --git a/services/camera/libcameraservice/camera2/ZslProcessor.h b/services/camera/libcameraservice/api1/client2/ZslProcessor.h index 27b597e..6d3cb85 100644 --- a/services/camera/libcameraservice/camera2/ZslProcessor.h +++ b/services/camera/libcameraservice/api1/client2/ZslProcessor.h @@ -23,12 +23,11 @@ #include <utils/Mutex.h> #include <utils/Condition.h> #include <gui/BufferItemConsumer.h> -#include "Parameters.h" -#include "FrameProcessor.h" -#include "camera/CameraMetadata.h" -#include "Camera2Heap.h" -#include "../CameraDeviceBase.h" -#include "ZslProcessorInterface.h" +#include <camera/CameraMetadata.h> + +#include "common/CameraDeviceBase.h" +#include "api1/client2/ZslProcessorInterface.h" +#include "api1/client2/FrameProcessor.h" namespace android { @@ -37,6 +36,7 @@ class Camera2Client; namespace camera2 { class CaptureSequencer; +class Parameters; /*** * ZSL queue processing @@ -54,7 +54,7 @@ class ZslProcessor: // From mZslConsumer virtual void onFrameAvailable(); // From FrameProcessor - virtual void onFrameAvailable(int32_t frameId, const CameraMetadata &frame); + virtual void onFrameAvailable(int32_t requestId, const CameraMetadata &frame); virtual void onBufferReleased(buffer_handle_t *handle); diff --git a/services/camera/libcameraservice/camera2/ZslProcessor3.cpp b/services/camera/libcameraservice/api1/client2/ZslProcessor3.cpp index 40c77df..776ebe2 100644 --- a/services/camera/libcameraservice/camera2/ZslProcessor3.cpp +++ b/services/camera/libcameraservice/api1/client2/ZslProcessor3.cpp @@ -27,13 +27,13 @@ #include <utils/Log.h> #include <utils/Trace.h> - -#include "ZslProcessor3.h" #include <gui/Surface.h> -#include "../CameraDeviceBase.h" -#include "../Camera3Device.h" -#include "../Camera2Client.h" +#include "common/CameraDeviceBase.h" +#include "api1/Camera2Client.h" +#include "api1/client2/CaptureSequencer.h" +#include "api1/client2/ZslProcessor3.h" +#include "device3/Camera3Device.h" namespace android { namespace camera2 { @@ -61,7 +61,7 @@ ZslProcessor3::~ZslProcessor3() { deleteStream(); } -void ZslProcessor3::onFrameAvailable(int32_t /*frameId*/, +void ZslProcessor3::onFrameAvailable(int32_t /*requestId*/, const CameraMetadata &frame) { Mutex::Autolock l(mInputMutex); camera_metadata_ro_entry_t entry; @@ -247,13 +247,13 @@ status_t ZslProcessor3::pushToReprocess(int32_t requestId) { uint8_t requestType = ANDROID_REQUEST_TYPE_REPROCESS; res = request.update(ANDROID_REQUEST_TYPE, &requestType, 1); - uint8_t inputStreams[1] = - { static_cast<uint8_t>(mZslStreamId) }; + int32_t inputStreams[1] = + { mZslStreamId }; if (res == OK) request.update(ANDROID_REQUEST_INPUT_STREAMS, inputStreams, 1); // TODO: Shouldn't we also update the latest preview frame? - uint8_t outputStreams[1] = - { static_cast<uint8_t>(client->getCaptureStreamId()) }; + int32_t outputStreams[1] = + { client->getCaptureStreamId() }; if (res == OK) request.update(ANDROID_REQUEST_OUTPUT_STREAMS, outputStreams, 1); res = request.update(ANDROID_REQUEST_ID, diff --git a/services/camera/libcameraservice/camera2/ZslProcessor3.h b/services/camera/libcameraservice/api1/client2/ZslProcessor3.h index cb98b99..d2f8322 100644 --- a/services/camera/libcameraservice/camera2/ZslProcessor3.h +++ b/services/camera/libcameraservice/api1/client2/ZslProcessor3.h @@ -23,13 +23,11 @@ #include <utils/Mutex.h> #include <utils/Condition.h> #include <gui/BufferItemConsumer.h> -#include "Parameters.h" -#include "FrameProcessor.h" -#include "camera/CameraMetadata.h" -#include "Camera2Heap.h" -#include "../CameraDeviceBase.h" -#include "ZslProcessorInterface.h" -#include "../camera3/Camera3ZslStream.h" +#include <camera/CameraMetadata.h> + +#include "api1/client2/FrameProcessor.h" +#include "api1/client2/ZslProcessorInterface.h" +#include "device3/Camera3ZslStream.h" namespace android { @@ -38,6 +36,7 @@ class Camera2Client; namespace camera2 { class CaptureSequencer; +class Parameters; /*** * ZSL queue processing @@ -52,7 +51,7 @@ class ZslProcessor3 : ~ZslProcessor3(); // From FrameProcessor - virtual void onFrameAvailable(int32_t frameId, const CameraMetadata &frame); + virtual void onFrameAvailable(int32_t requestId, const CameraMetadata &frame); /** **************************************** diff --git a/services/camera/libcameraservice/camera2/ZslProcessorInterface.h b/services/camera/libcameraservice/api1/client2/ZslProcessorInterface.h index 183c0c2..183c0c2 100644 --- a/services/camera/libcameraservice/camera2/ZslProcessorInterface.h +++ b/services/camera/libcameraservice/api1/client2/ZslProcessorInterface.h diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp new file mode 100644 index 0000000..72126c1 --- /dev/null +++ b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp @@ -0,0 +1,679 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "CameraDeviceClient" +#define ATRACE_TAG ATRACE_TAG_CAMERA +// #define LOG_NDEBUG 0 + +#include <cutils/properties.h> +#include <utils/Log.h> +#include <utils/Trace.h> +#include <gui/Surface.h> +#include <camera/camera2/CaptureRequest.h> + +#include "common/CameraDeviceBase.h" +#include "api2/CameraDeviceClient.h" + + + +namespace android { +using namespace camera2; + +CameraDeviceClientBase::CameraDeviceClientBase( + const sp<CameraService>& cameraService, + const sp<ICameraDeviceCallbacks>& remoteCallback, + const String16& clientPackageName, + int cameraId, + int cameraFacing, + int clientPid, + uid_t clientUid, + int servicePid) : + BasicClient(cameraService, remoteCallback->asBinder(), clientPackageName, + cameraId, cameraFacing, clientPid, clientUid, servicePid), + mRemoteCallback(remoteCallback) { +} + +// Interface used by CameraService + +CameraDeviceClient::CameraDeviceClient(const sp<CameraService>& cameraService, + const sp<ICameraDeviceCallbacks>& remoteCallback, + const String16& clientPackageName, + int cameraId, + int cameraFacing, + int clientPid, + uid_t clientUid, + int servicePid) : + Camera2ClientBase(cameraService, remoteCallback, clientPackageName, + cameraId, cameraFacing, clientPid, clientUid, servicePid), + mRequestIdCounter(0) { + + ATRACE_CALL(); + ALOGI("CameraDeviceClient %d: Opened", cameraId); +} + +status_t CameraDeviceClient::initialize(camera_module_t *module) +{ + ATRACE_CALL(); + status_t res; + + res = Camera2ClientBase::initialize(module); + if (res != OK) { + return res; + } + + String8 threadName; + mFrameProcessor = new FrameProcessorBase(mDevice); + threadName = String8::format("CDU-%d-FrameProc", mCameraId); + mFrameProcessor->run(threadName.string()); + + mFrameProcessor->registerListener(FRAME_PROCESSOR_LISTENER_MIN_ID, + FRAME_PROCESSOR_LISTENER_MAX_ID, + /*listener*/this); + + return OK; +} + +CameraDeviceClient::~CameraDeviceClient() { +} + +status_t CameraDeviceClient::submitRequest(sp<CaptureRequest> request, + bool streaming) { + ATRACE_CALL(); + ALOGV("%s", __FUNCTION__); + + status_t res; + + if ( (res = checkPid(__FUNCTION__) ) != OK) return res; + + Mutex::Autolock icl(mBinderSerializationLock); + + if (!mDevice.get()) return DEAD_OBJECT; + + if (request == 0) { + ALOGE("%s: Camera %d: Sent null request. Rejecting request.", + __FUNCTION__, mCameraId); + return BAD_VALUE; + } + + CameraMetadata metadata(request->mMetadata); + + if (metadata.isEmpty()) { + ALOGE("%s: Camera %d: Sent empty metadata packet. Rejecting request.", + __FUNCTION__, mCameraId); + return BAD_VALUE; + } else if (request->mSurfaceList.size() == 0) { + ALOGE("%s: Camera %d: Requests must have at least one surface target. " + "Rejecting request.", __FUNCTION__, mCameraId); + return BAD_VALUE; + } + + if (!enforceRequestPermissions(metadata)) { + // Callee logs + return PERMISSION_DENIED; + } + + /** + * Write in the output stream IDs which we calculate from + * the capture request's list of surface targets + */ + Vector<int32_t> outputStreamIds; + outputStreamIds.setCapacity(request->mSurfaceList.size()); + for (size_t i = 0; i < request->mSurfaceList.size(); ++i) { + sp<Surface> surface = request->mSurfaceList[i]; + + if (surface == 0) continue; + + sp<IGraphicBufferProducer> gbp = surface->getIGraphicBufferProducer(); + int idx = mStreamMap.indexOfKey(gbp->asBinder()); + + // Trying to submit request with surface that wasn't created + if (idx == NAME_NOT_FOUND) { + ALOGE("%s: Camera %d: Tried to submit a request with a surface that" + " we have not called createStream on", + __FUNCTION__, mCameraId); + return BAD_VALUE; + } + + int streamId = mStreamMap.valueAt(idx); + outputStreamIds.push_back(streamId); + ALOGV("%s: Camera %d: Appending output stream %d to request", + __FUNCTION__, mCameraId, streamId); + } + + metadata.update(ANDROID_REQUEST_OUTPUT_STREAMS, &outputStreamIds[0], + outputStreamIds.size()); + + int32_t requestId = mRequestIdCounter++; + metadata.update(ANDROID_REQUEST_ID, &requestId, /*size*/1); + ALOGV("%s: Camera %d: Submitting request with ID %d", + __FUNCTION__, mCameraId, requestId); + + if (streaming) { + res = mDevice->setStreamingRequest(metadata); + if (res != OK) { + ALOGE("%s: Camera %d: Got error %d after trying to set streaming " + "request", __FUNCTION__, mCameraId, res); + } else { + mStreamingRequestList.push_back(requestId); + } + } else { + res = mDevice->capture(metadata); + if (res != OK) { + ALOGE("%s: Camera %d: Got error %d after trying to set capture", + __FUNCTION__, mCameraId, res); + } + } + + ALOGV("%s: Camera %d: End of function", __FUNCTION__, mCameraId); + if (res == OK) { + return requestId; + } + + return res; +} + +status_t CameraDeviceClient::cancelRequest(int requestId) { + ATRACE_CALL(); + ALOGV("%s, requestId = %d", __FUNCTION__, requestId); + + status_t res; + + if ( (res = checkPid(__FUNCTION__) ) != OK) return res; + + Mutex::Autolock icl(mBinderSerializationLock); + + if (!mDevice.get()) return DEAD_OBJECT; + + Vector<int>::iterator it, end; + for (it = mStreamingRequestList.begin(), end = mStreamingRequestList.end(); + it != end; ++it) { + if (*it == requestId) { + break; + } + } + + if (it == end) { + ALOGE("%s: Camera%d: Did not find request id %d in list of streaming " + "requests", __FUNCTION__, mCameraId, requestId); + return BAD_VALUE; + } + + res = mDevice->clearStreamingRequest(); + + if (res == OK) { + ALOGV("%s: Camera %d: Successfully cleared streaming request", + __FUNCTION__, mCameraId); + mStreamingRequestList.erase(it); + } + + return res; +} + +status_t CameraDeviceClient::deleteStream(int streamId) { + ATRACE_CALL(); + ALOGV("%s (streamId = 0x%x)", __FUNCTION__, streamId); + + status_t res; + if ( (res = checkPid(__FUNCTION__) ) != OK) return res; + + Mutex::Autolock icl(mBinderSerializationLock); + + if (!mDevice.get()) return DEAD_OBJECT; + + // Guard against trying to delete non-created streams + ssize_t index = NAME_NOT_FOUND; + for (size_t i = 0; i < mStreamMap.size(); ++i) { + if (streamId == mStreamMap.valueAt(i)) { + index = i; + break; + } + } + + if (index == NAME_NOT_FOUND) { + ALOGW("%s: Camera %d: Invalid stream ID (%d) specified, no stream " + "created yet", __FUNCTION__, mCameraId, streamId); + return BAD_VALUE; + } + + // Also returns BAD_VALUE if stream ID was not valid + res = mDevice->deleteStream(streamId); + + if (res == BAD_VALUE) { + ALOGE("%s: Camera %d: Unexpected BAD_VALUE when deleting stream, but we" + " already checked and the stream ID (%d) should be valid.", + __FUNCTION__, mCameraId, streamId); + } else if (res == OK) { + mStreamMap.removeItemsAt(index); + + ALOGV("%s: Camera %d: Successfully deleted stream ID (%d)", + __FUNCTION__, mCameraId, streamId); + } + + return res; +} + +status_t CameraDeviceClient::createStream(int width, int height, int format, + const sp<IGraphicBufferProducer>& bufferProducer) +{ + ATRACE_CALL(); + ALOGV("%s (w = %d, h = %d, f = 0x%x)", __FUNCTION__, width, height, format); + + status_t res; + if ( (res = checkPid(__FUNCTION__) ) != OK) return res; + + Mutex::Autolock icl(mBinderSerializationLock); + + if (!mDevice.get()) return DEAD_OBJECT; + + // Don't create multiple streams for the same target surface + { + ssize_t index = mStreamMap.indexOfKey(bufferProducer->asBinder()); + if (index != NAME_NOT_FOUND) { + ALOGW("%s: Camera %d: Buffer producer already has a stream for it " + "(ID %d)", + __FUNCTION__, mCameraId, index); + return ALREADY_EXISTS; + } + } + + // HACK b/10949105 + // Query consumer usage bits to set async operation mode for + // GLConsumer using controlledByApp parameter. + bool useAsync = false; + int32_t consumerUsage; + if ((res = bufferProducer->query(NATIVE_WINDOW_CONSUMER_USAGE_BITS, + &consumerUsage)) != OK) { + ALOGE("%s: Camera %d: Failed to query consumer usage", __FUNCTION__, + mCameraId); + return res; + } + if (consumerUsage & GraphicBuffer::USAGE_HW_TEXTURE) { + ALOGW("%s: Camera %d: Forcing asynchronous mode for stream", + __FUNCTION__, mCameraId); + useAsync = true; + } + + sp<IBinder> binder; + sp<ANativeWindow> anw; + if (bufferProducer != 0) { + binder = bufferProducer->asBinder(); + anw = new Surface(bufferProducer, useAsync); + } + + // TODO: remove w,h,f since we are ignoring them + + if ((res = anw->query(anw.get(), NATIVE_WINDOW_WIDTH, &width)) != OK) { + ALOGE("%s: Camera %d: Failed to query Surface width", __FUNCTION__, + mCameraId); + return res; + } + if ((res = anw->query(anw.get(), NATIVE_WINDOW_HEIGHT, &height)) != OK) { + ALOGE("%s: Camera %d: Failed to query Surface height", __FUNCTION__, + mCameraId); + return res; + } + if ((res = anw->query(anw.get(), NATIVE_WINDOW_FORMAT, &format)) != OK) { + ALOGE("%s: Camera %d: Failed to query Surface format", __FUNCTION__, + mCameraId); + return res; + } + + // FIXME: remove this override since the default format should be + // IMPLEMENTATION_DEFINED. b/9487482 + if (format >= HAL_PIXEL_FORMAT_RGBA_8888 && + format <= HAL_PIXEL_FORMAT_BGRA_8888) { + ALOGW("%s: Camera %d: Overriding format 0x%x to IMPLEMENTATION_DEFINED", + __FUNCTION__, mCameraId, format); + format = HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED; + } + + // TODO: add startConfigure/stopConfigure call to CameraDeviceBase + // this will make it so Camera3Device doesn't call configure_streams + // after each call, but only once we are done with all. + + int streamId = -1; + if (format == HAL_PIXEL_FORMAT_BLOB) { + // JPEG buffers need to be sized for maximum possible compressed size + CameraMetadata staticInfo = mDevice->info(); + camera_metadata_entry_t entry = staticInfo.find(ANDROID_JPEG_MAX_SIZE); + if (entry.count == 0) { + ALOGE("%s: Camera %d: Can't find maximum JPEG size in " + "static metadata!", __FUNCTION__, mCameraId); + return INVALID_OPERATION; + } + int32_t maxJpegSize = entry.data.i32[0]; + res = mDevice->createStream(anw, width, height, format, maxJpegSize, + &streamId); + } else { + // All other streams are a known size + res = mDevice->createStream(anw, width, height, format, /*size*/0, + &streamId); + } + + if (res == OK) { + mStreamMap.add(bufferProducer->asBinder(), streamId); + + ALOGV("%s: Camera %d: Successfully created a new stream ID %d", + __FUNCTION__, mCameraId, streamId); + + /** + * Set the stream transform flags to automatically + * rotate the camera stream for preview use cases. + */ + int32_t transform = 0; + res = getRotationTransformLocked(&transform); + + if (res != OK) { + // Error logged by getRotationTransformLocked. + return res; + } + + res = mDevice->setStreamTransform(streamId, transform); + if (res != OK) { + ALOGE("%s: Failed to set stream transform (stream id %d)", + __FUNCTION__, streamId); + return res; + } + + return streamId; + } + + return res; +} + +// Create a request object from a template. +status_t CameraDeviceClient::createDefaultRequest(int templateId, + /*out*/ + CameraMetadata* request) +{ + ATRACE_CALL(); + ALOGV("%s (templateId = 0x%x)", __FUNCTION__, templateId); + + status_t res; + if ( (res = checkPid(__FUNCTION__) ) != OK) return res; + + Mutex::Autolock icl(mBinderSerializationLock); + + if (!mDevice.get()) return DEAD_OBJECT; + + CameraMetadata metadata; + if ( (res = mDevice->createDefaultRequest(templateId, &metadata) ) == OK && + request != NULL) { + + request->swap(metadata); + } + + return res; +} + +status_t CameraDeviceClient::getCameraInfo(/*out*/CameraMetadata* info) +{ + ATRACE_CALL(); + ALOGV("%s", __FUNCTION__); + + status_t res = OK; + + if ( (res = checkPid(__FUNCTION__) ) != OK) return res; + + Mutex::Autolock icl(mBinderSerializationLock); + + if (!mDevice.get()) return DEAD_OBJECT; + + if (info != NULL) { + *info = mDevice->info(); // static camera metadata + // TODO: merge with device-specific camera metadata + } + + return res; +} + +status_t CameraDeviceClient::waitUntilIdle() +{ + ATRACE_CALL(); + ALOGV("%s", __FUNCTION__); + + status_t res = OK; + if ( (res = checkPid(__FUNCTION__) ) != OK) return res; + + Mutex::Autolock icl(mBinderSerializationLock); + + if (!mDevice.get()) return DEAD_OBJECT; + + // FIXME: Also need check repeating burst. + if (!mStreamingRequestList.isEmpty()) { + ALOGE("%s: Camera %d: Try to waitUntilIdle when there are active streaming requests", + __FUNCTION__, mCameraId); + return INVALID_OPERATION; + } + res = mDevice->waitUntilDrained(); + ALOGV("%s Done", __FUNCTION__); + + return res; +} + +status_t CameraDeviceClient::flush() { + ATRACE_CALL(); + ALOGV("%s", __FUNCTION__); + + status_t res = OK; + if ( (res = checkPid(__FUNCTION__) ) != OK) return res; + + Mutex::Autolock icl(mBinderSerializationLock); + + if (!mDevice.get()) return DEAD_OBJECT; + + return mDevice->flush(); +} + +status_t CameraDeviceClient::dump(int fd, const Vector<String16>& args) { + String8 result; + result.appendFormat("CameraDeviceClient[%d] (%p) PID: %d, dump:\n", + mCameraId, + getRemoteCallback()->asBinder().get(), + mClientPid); + result.append(" State: "); + + // TODO: print dynamic/request section from most recent requests + mFrameProcessor->dump(fd, args); + + return dumpDevice(fd, args); +} + + +void CameraDeviceClient::notifyError() { + // Thread safe. Don't bother locking. + sp<ICameraDeviceCallbacks> remoteCb = getRemoteCallback(); + + if (remoteCb != 0) { + remoteCb->onDeviceError(ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE); + } +} + +void CameraDeviceClient::notifyIdle() { + // Thread safe. Don't bother locking. + sp<ICameraDeviceCallbacks> remoteCb = getRemoteCallback(); + + if (remoteCb != 0) { + remoteCb->onDeviceIdle(); + } +} + +void CameraDeviceClient::notifyShutter(int requestId, + nsecs_t timestamp) { + // Thread safe. Don't bother locking. + sp<ICameraDeviceCallbacks> remoteCb = getRemoteCallback(); + if (remoteCb != 0) { + remoteCb->onCaptureStarted(requestId, timestamp); + } +} + +// TODO: refactor the code below this with IProCameraUser. +// it's 100% copy-pasted, so lets not change it right now to make it easier. + +void CameraDeviceClient::detachDevice() { + if (mDevice == 0) return; + + ALOGV("Camera %d: Stopping processors", mCameraId); + + mFrameProcessor->removeListener(FRAME_PROCESSOR_LISTENER_MIN_ID, + FRAME_PROCESSOR_LISTENER_MAX_ID, + /*listener*/this); + mFrameProcessor->requestExit(); + ALOGV("Camera %d: Waiting for threads", mCameraId); + mFrameProcessor->join(); + ALOGV("Camera %d: Disconnecting device", mCameraId); + + // WORKAROUND: HAL refuses to disconnect while there's streams in flight + { + mDevice->clearStreamingRequest(); + + status_t code; + if ((code = mDevice->waitUntilDrained()) != OK) { + ALOGE("%s: waitUntilDrained failed with code 0x%x", __FUNCTION__, + code); + } + } + + Camera2ClientBase::detachDevice(); +} + +/** Device-related methods */ +void CameraDeviceClient::onFrameAvailable(int32_t requestId, + const CameraMetadata& frame) { + ATRACE_CALL(); + ALOGV("%s", __FUNCTION__); + + // Thread-safe. No lock necessary. + sp<ICameraDeviceCallbacks> remoteCb = mRemoteCallback; + if (remoteCb != NULL) { + ALOGV("%s: frame = %p ", __FUNCTION__, &frame); + remoteCb->onResultReceived(requestId, frame); + } +} + +// TODO: move to Camera2ClientBase +bool CameraDeviceClient::enforceRequestPermissions(CameraMetadata& metadata) { + + const int pid = IPCThreadState::self()->getCallingPid(); + const int selfPid = getpid(); + camera_metadata_entry_t entry; + + /** + * Mixin default important security values + * - android.led.transmit = defaulted ON + */ + CameraMetadata staticInfo = mDevice->info(); + entry = staticInfo.find(ANDROID_LED_AVAILABLE_LEDS); + for(size_t i = 0; i < entry.count; ++i) { + uint8_t led = entry.data.u8[i]; + + switch(led) { + case ANDROID_LED_AVAILABLE_LEDS_TRANSMIT: { + uint8_t transmitDefault = ANDROID_LED_TRANSMIT_ON; + if (!metadata.exists(ANDROID_LED_TRANSMIT)) { + metadata.update(ANDROID_LED_TRANSMIT, + &transmitDefault, 1); + } + break; + } + } + } + + // We can do anything! + if (pid == selfPid) { + return true; + } + + /** + * Permission check special fields in the request + * - android.led.transmit = android.permission.CAMERA_DISABLE_TRANSMIT + */ + entry = metadata.find(ANDROID_LED_TRANSMIT); + if (entry.count > 0 && entry.data.u8[0] != ANDROID_LED_TRANSMIT_ON) { + String16 permissionString = + String16("android.permission.CAMERA_DISABLE_TRANSMIT_LED"); + if (!checkCallingPermission(permissionString)) { + const int uid = IPCThreadState::self()->getCallingUid(); + ALOGE("Permission Denial: " + "can't disable transmit LED pid=%d, uid=%d", pid, uid); + return false; + } + } + + return true; +} + +status_t CameraDeviceClient::getRotationTransformLocked(int32_t* transform) { + ALOGV("%s: begin", __FUNCTION__); + + if (transform == NULL) { + ALOGW("%s: null transform", __FUNCTION__); + return BAD_VALUE; + } + + *transform = 0; + + const CameraMetadata& staticInfo = mDevice->info(); + camera_metadata_ro_entry_t entry = staticInfo.find(ANDROID_SENSOR_ORIENTATION); + if (entry.count == 0) { + ALOGE("%s: Camera %d: Can't find android.sensor.orientation in " + "static metadata!", __FUNCTION__, mCameraId); + return INVALID_OPERATION; + } + + int32_t& flags = *transform; + + int orientation = entry.data.i32[0]; + switch (orientation) { + case 0: + flags = 0; + break; + case 90: + flags = NATIVE_WINDOW_TRANSFORM_ROT_90; + break; + case 180: + flags = NATIVE_WINDOW_TRANSFORM_ROT_180; + break; + case 270: + flags = NATIVE_WINDOW_TRANSFORM_ROT_270; + break; + default: + ALOGE("%s: Invalid HAL android.sensor.orientation value: %d", + __FUNCTION__, orientation); + return INVALID_OPERATION; + } + + /** + * This magic flag makes surfaceflinger un-rotate the buffers + * to counter the extra global device UI rotation whenever the user + * physically rotates the device. + * + * By doing this, the camera buffer always ends up aligned + * with the physical camera for a "see through" effect. + * + * In essence, the buffer only gets rotated during preview use-cases. + * The user is still responsible to re-create streams of the proper + * aspect ratio, or the preview will end up looking non-uniformly + * stretched. + */ + flags |= NATIVE_WINDOW_TRANSFORM_INVERSE_DISPLAY; + + ALOGV("%s: final transform = 0x%x", __FUNCTION__, flags); + + return OK; +} + +} // namespace android diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.h b/services/camera/libcameraservice/api2/CameraDeviceClient.h new file mode 100644 index 0000000..b9c16aa --- /dev/null +++ b/services/camera/libcameraservice/api2/CameraDeviceClient.h @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_SERVERS_CAMERA_PHOTOGRAPHY_CAMERADEVICECLIENT_H +#define ANDROID_SERVERS_CAMERA_PHOTOGRAPHY_CAMERADEVICECLIENT_H + +#include <camera/camera2/ICameraDeviceUser.h> +#include <camera/camera2/ICameraDeviceCallbacks.h> + +#include "CameraService.h" +#include "common/FrameProcessorBase.h" +#include "common/Camera2ClientBase.h" + +namespace android { + +struct CameraDeviceClientBase : + public CameraService::BasicClient, public BnCameraDeviceUser +{ + typedef ICameraDeviceCallbacks TCamCallbacks; + + const sp<ICameraDeviceCallbacks>& getRemoteCallback() { + return mRemoteCallback; + } + +protected: + CameraDeviceClientBase(const sp<CameraService>& cameraService, + const sp<ICameraDeviceCallbacks>& remoteCallback, + const String16& clientPackageName, + int cameraId, + int cameraFacing, + int clientPid, + uid_t clientUid, + int servicePid); + + sp<ICameraDeviceCallbacks> mRemoteCallback; +}; + +/** + * Implements the binder ICameraDeviceUser API, + * meant for HAL3-public implementation of + * android.hardware.photography.CameraDevice + */ +class CameraDeviceClient : + public Camera2ClientBase<CameraDeviceClientBase>, + public camera2::FrameProcessorBase::FilteredListener +{ +public: + /** + * ICameraDeviceUser interface (see ICameraDeviceUser for details) + */ + + // Note that the callee gets a copy of the metadata. + virtual int submitRequest(sp<CaptureRequest> request, + bool streaming = false); + virtual status_t cancelRequest(int requestId); + + // Returns -EBUSY if device is not idle + virtual status_t deleteStream(int streamId); + + virtual status_t createStream( + int width, + int height, + int format, + const sp<IGraphicBufferProducer>& bufferProducer); + + // Create a request object from a template. + virtual status_t createDefaultRequest(int templateId, + /*out*/ + CameraMetadata* request); + + // Get the static metadata for the camera + // -- Caller owns the newly allocated metadata + virtual status_t getCameraInfo(/*out*/CameraMetadata* info); + + // Wait until all the submitted requests have finished processing + virtual status_t waitUntilIdle(); + + // Flush all active and pending requests as fast as possible + virtual status_t flush(); + + /** + * Interface used by CameraService + */ + + CameraDeviceClient(const sp<CameraService>& cameraService, + const sp<ICameraDeviceCallbacks>& remoteCallback, + const String16& clientPackageName, + int cameraId, + int cameraFacing, + int clientPid, + uid_t clientUid, + int servicePid); + virtual ~CameraDeviceClient(); + + virtual status_t initialize(camera_module_t *module); + + virtual status_t dump(int fd, const Vector<String16>& args); + + /** + * Device listener interface + */ + + virtual void notifyIdle(); + virtual void notifyError(); + virtual void notifyShutter(int requestId, nsecs_t timestamp); + + /** + * Interface used by independent components of CameraDeviceClient. + */ +protected: + /** FilteredListener implementation **/ + virtual void onFrameAvailable(int32_t requestId, + const CameraMetadata& frame); + virtual void detachDevice(); + + // Calculate the ANativeWindow transform from android.sensor.orientation + status_t getRotationTransformLocked(/*out*/int32_t* transform); + +private: + /** ICameraDeviceUser interface-related private members */ + + /** Preview callback related members */ + sp<camera2::FrameProcessorBase> mFrameProcessor; + static const int32_t FRAME_PROCESSOR_LISTENER_MIN_ID = 0; + static const int32_t FRAME_PROCESSOR_LISTENER_MAX_ID = 0x7fffffffL; + + /** Utility members */ + bool enforceRequestPermissions(CameraMetadata& metadata); + + // IGraphicsBufferProducer binder -> Stream ID + KeyedVector<sp<IBinder>, int> mStreamMap; + + // Stream ID + Vector<int> mStreamingRequestList; + + int32_t mRequestIdCounter; +}; + +}; // namespace android + +#endif diff --git a/services/camera/libcameraservice/ProCamera2Client.cpp b/services/camera/libcameraservice/api_pro/ProCamera2Client.cpp index 251fdab..1a7a7a7 100644 --- a/services/camera/libcameraservice/ProCamera2Client.cpp +++ b/services/camera/libcameraservice/api_pro/ProCamera2Client.cpp @@ -24,10 +24,9 @@ #include <cutils/properties.h> #include <gui/Surface.h> #include <gui/Surface.h> -#include "camera2/Parameters.h" -#include "ProCamera2Client.h" -#include "camera2/ProFrameProcessor.h" -#include "CameraDeviceBase.h" + +#include "api_pro/ProCamera2Client.h" +#include "common/CameraDeviceBase.h" namespace android { using namespace camera2; @@ -62,7 +61,7 @@ status_t ProCamera2Client::initialize(camera_module_t *module) } String8 threadName; - mFrameProcessor = new ProFrameProcessor(mDevice); + mFrameProcessor = new FrameProcessorBase(mDevice); threadName = String8::format("PC2-%d-FrameProc", mCameraId); mFrameProcessor->run(threadName.string()); @@ -218,6 +217,7 @@ status_t ProCamera2Client::submitRequest(camera_metadata_t* request, } status_t ProCamera2Client::cancelRequest(int requestId) { + (void)requestId; ATRACE_CALL(); ALOGV("%s", __FUNCTION__); @@ -374,7 +374,7 @@ void ProCamera2Client::detachDevice() { } /** Device-related methods */ -void ProCamera2Client::onFrameAvailable(int32_t frameId, +void ProCamera2Client::onFrameAvailable(int32_t requestId, const CameraMetadata& frame) { ATRACE_CALL(); ALOGV("%s", __FUNCTION__); @@ -386,7 +386,7 @@ void ProCamera2Client::onFrameAvailable(int32_t frameId, CameraMetadata tmp(frame); camera_metadata_t* meta = tmp.release(); ALOGV("%s: meta = %p ", __FUNCTION__, meta); - mRemoteCallback->onResultReceived(frameId, meta); + mRemoteCallback->onResultReceived(requestId, meta); tmp.acquire(meta); } diff --git a/services/camera/libcameraservice/ProCamera2Client.h b/services/camera/libcameraservice/api_pro/ProCamera2Client.h index faee9f9..8a0f547 100644 --- a/services/camera/libcameraservice/ProCamera2Client.h +++ b/services/camera/libcameraservice/api_pro/ProCamera2Client.h @@ -17,10 +17,10 @@ #ifndef ANDROID_SERVERS_CAMERA_PROCAMERA2CLIENT_H #define ANDROID_SERVERS_CAMERA_PROCAMERA2CLIENT_H -#include "Camera2Device.h" #include "CameraService.h" -#include "camera2/ProFrameProcessor.h" -#include "Camera2ClientBase.h" +#include "common/FrameProcessorBase.h" +#include "common/Camera2ClientBase.h" +#include "device2/Camera2Device.h" namespace android { @@ -31,7 +31,7 @@ class IMemory; */ class ProCamera2Client : public Camera2ClientBase<CameraService::ProClient>, - public camera2::ProFrameProcessor::FilteredListener + public camera2::FrameProcessorBase::FilteredListener { public: /** @@ -97,7 +97,7 @@ public: protected: /** FilteredListener implementation **/ - virtual void onFrameAvailable(int32_t frameId, + virtual void onFrameAvailable(int32_t requestId, const CameraMetadata& frame); virtual void detachDevice(); @@ -105,7 +105,7 @@ private: /** IProCameraUser interface-related private members */ /** Preview callback related members */ - sp<camera2::ProFrameProcessor> mFrameProcessor; + sp<camera2::FrameProcessorBase> mFrameProcessor; static const int32_t FRAME_PROCESSOR_LISTENER_MIN_ID = 0; static const int32_t FRAME_PROCESSOR_LISTENER_MAX_ID = 0x7fffffffL; diff --git a/services/camera/libcameraservice/Camera2ClientBase.cpp b/services/camera/libcameraservice/common/Camera2ClientBase.cpp index 0623b89..2d1253f 100644 --- a/services/camera/libcameraservice/Camera2ClientBase.cpp +++ b/services/camera/libcameraservice/common/Camera2ClientBase.cpp @@ -24,11 +24,12 @@ #include <cutils/properties.h> #include <gui/Surface.h> #include <gui/Surface.h> -#include "camera2/Parameters.h" -#include "Camera2ClientBase.h" -#include "camera2/ProFrameProcessor.h" -#include "Camera2Device.h" +#include "common/Camera2ClientBase.h" + +#include "api2/CameraDeviceClient.h" + +#include "CameraDeviceFactory.h" namespace android { using namespace camera2; @@ -54,7 +55,9 @@ Camera2ClientBase<TClientBase>::Camera2ClientBase( mSharedCameraCallbacks(remoteCallback) { ALOGI("Camera %d: Opened", cameraId); - mDevice = new Camera2Device(cameraId); + + mDevice = CameraDeviceFactory::createDevice(cameraId); + LOG_ALWAYS_FATAL_IF(mDevice == 0, "Device should never be NULL here."); } template <typename TClientBase> @@ -92,7 +95,7 @@ status_t Camera2ClientBase<TClientBase>::initialize(camera_module_t *module) { if (res != OK) { ALOGE("%s: Camera %d: unable to initialize device: %s (%d)", __FUNCTION__, TClientBase::mCameraId, strerror(-res), res); - return NO_INIT; + return res; } res = mDevice->setNotifyCallback(this); @@ -223,13 +226,18 @@ void Camera2ClientBase<TClientBase>::notifyError(int errorCode, int arg1, } template <typename TClientBase> -void Camera2ClientBase<TClientBase>::notifyShutter(int frameNumber, +void Camera2ClientBase<TClientBase>::notifyIdle() { + ALOGV("Camera device is now idle"); +} + +template <typename TClientBase> +void Camera2ClientBase<TClientBase>::notifyShutter(int requestId, nsecs_t timestamp) { - (void)frameNumber; + (void)requestId; (void)timestamp; - ALOGV("%s: Shutter notification for frame %d at time %lld", __FUNCTION__, - frameNumber, timestamp); + ALOGV("%s: Shutter notification for request id %d at time %lld", + __FUNCTION__, requestId, timestamp); } template <typename TClientBase> @@ -241,13 +249,6 @@ void Camera2ClientBase<TClientBase>::notifyAutoFocus(uint8_t newState, ALOGV("%s: Autofocus state now %d, last trigger %d", __FUNCTION__, newState, triggerId); - typename SharedCameraCallbacks::Lock l(mSharedCameraCallbacks); - if (l.mRemoteCallback != 0) { - l.mRemoteCallback->notifyCallback(CAMERA_MSG_FOCUS_MOVE, 1, 0); - } - if (l.mRemoteCallback != 0) { - l.mRemoteCallback->notifyCallback(CAMERA_MSG_FOCUS, 1, 0); - } } template <typename TClientBase> @@ -325,5 +326,6 @@ void Camera2ClientBase<TClientBase>::SharedCameraCallbacks::clear() { template class Camera2ClientBase<CameraService::ProClient>; template class Camera2ClientBase<CameraService::Client>; +template class Camera2ClientBase<CameraDeviceClientBase>; } // namespace android diff --git a/services/camera/libcameraservice/Camera2ClientBase.h b/services/camera/libcameraservice/common/Camera2ClientBase.h index 9001efb..61e44f0 100644 --- a/services/camera/libcameraservice/Camera2ClientBase.h +++ b/services/camera/libcameraservice/common/Camera2ClientBase.h @@ -17,13 +17,14 @@ #ifndef ANDROID_SERVERS_CAMERA_CAMERA2CLIENT_BASE_H #define ANDROID_SERVERS_CAMERA_CAMERA2CLIENT_BASE_H -#include "CameraDeviceBase.h" -#include "CameraService.h" +#include "common/CameraDeviceBase.h" namespace android { class IMemory; +class CameraService; + template <typename TClientBase> class Camera2ClientBase : public TClientBase, @@ -61,7 +62,8 @@ public: */ virtual void notifyError(int errorCode, int arg1, int arg2); - virtual void notifyShutter(int frameNumber, nsecs_t timestamp); + virtual void notifyIdle(); + virtual void notifyShutter(int requestId, nsecs_t timestamp); virtual void notifyAutoFocus(uint8_t newState, int triggerId); virtual void notifyAutoExposure(uint8_t newState, int triggerId); virtual void notifyAutoWhitebalance(uint8_t newState, @@ -101,6 +103,10 @@ public: protected: + virtual sp<IBinder> asBinderWrapper() { + return IInterface::asBinder(); + } + virtual status_t dumpDevice(int fd, const Vector<String16>& args); /** Binder client interface-related private members */ diff --git a/services/camera/libcameraservice/CameraDeviceBase.cpp b/services/camera/libcameraservice/common/CameraDeviceBase.cpp index 6c4e87f..6c4e87f 100644 --- a/services/camera/libcameraservice/CameraDeviceBase.cpp +++ b/services/camera/libcameraservice/common/CameraDeviceBase.cpp diff --git a/services/camera/libcameraservice/CameraDeviceBase.h b/services/camera/libcameraservice/common/CameraDeviceBase.h index aa92bec..e80abf1 100644 --- a/services/camera/libcameraservice/CameraDeviceBase.h +++ b/services/camera/libcameraservice/common/CameraDeviceBase.h @@ -138,9 +138,18 @@ class CameraDeviceBase : public virtual RefBase { */ class NotificationListener { public: - // Refer to the Camera2 HAL definition for notification definitions + // The set of notifications is a merge of the notifications required for + // API1 and API2. + + // Required for API 1 and 2 virtual void notifyError(int errorCode, int arg1, int arg2) = 0; - virtual void notifyShutter(int frameNumber, nsecs_t timestamp) = 0; + + // Required only for API2 + virtual void notifyIdle() = 0; + virtual void notifyShutter(int requestId, + nsecs_t timestamp) = 0; + + // Required only for API1 virtual void notifyAutoFocus(uint8_t newState, int triggerId) = 0; virtual void notifyAutoExposure(uint8_t newState, int triggerId) = 0; virtual void notifyAutoWhitebalance(uint8_t newState, @@ -165,12 +174,14 @@ class CameraDeviceBase : public virtual RefBase { /** * Wait for a new frame to be produced, with timeout in nanoseconds. * Returns TIMED_OUT when no frame produced within the specified duration + * May be called concurrently to most methods, except for getNextFrame */ virtual status_t waitForNextFrame(nsecs_t timeout) = 0; /** * Get next metadata frame from the frame queue. Returns NULL if the queue * is empty; caller takes ownership of the metadata buffer. + * May be called concurrently to most methods, except for waitForNextFrame */ virtual status_t getNextFrame(CameraMetadata *frame) = 0; @@ -209,6 +220,13 @@ class CameraDeviceBase : public virtual RefBase { */ virtual status_t pushReprocessBuffer(int reprocessStreamId, buffer_handle_t *buffer, wp<BufferReleasedListener> listener) = 0; + + /** + * Flush all pending and in-flight requests. Blocks until flush is + * complete. + */ + virtual status_t flush() = 0; + }; }; // namespace android diff --git a/services/camera/libcameraservice/camera2/ProFrameProcessor.cpp b/services/camera/libcameraservice/common/FrameProcessorBase.cpp index 4012fc5..52906ee 100644 --- a/services/camera/libcameraservice/camera2/ProFrameProcessor.cpp +++ b/services/camera/libcameraservice/common/FrameProcessorBase.cpp @@ -14,29 +14,29 @@ * limitations under the License. */ -#define LOG_TAG "Camera2-ProFrameProcessor" +#define LOG_TAG "Camera2-FrameProcessorBase" #define ATRACE_TAG ATRACE_TAG_CAMERA //#define LOG_NDEBUG 0 #include <utils/Log.h> #include <utils/Trace.h> -#include "ProFrameProcessor.h" -#include "../CameraDeviceBase.h" +#include "common/FrameProcessorBase.h" +#include "common/CameraDeviceBase.h" namespace android { namespace camera2 { -ProFrameProcessor::ProFrameProcessor(wp<CameraDeviceBase> device) : +FrameProcessorBase::FrameProcessorBase(wp<CameraDeviceBase> device) : Thread(/*canCallJava*/false), mDevice(device) { } -ProFrameProcessor::~ProFrameProcessor() { +FrameProcessorBase::~FrameProcessorBase() { ALOGV("%s: Exit", __FUNCTION__); } -status_t ProFrameProcessor::registerListener(int32_t minId, +status_t FrameProcessorBase::registerListener(int32_t minId, int32_t maxId, wp<FilteredListener> listener) { Mutex::Autolock l(mInputMutex); ALOGV("%s: Registering listener for frame id range %d - %d", @@ -46,7 +46,7 @@ status_t ProFrameProcessor::registerListener(int32_t minId, return OK; } -status_t ProFrameProcessor::removeListener(int32_t minId, +status_t FrameProcessorBase::removeListener(int32_t minId, int32_t maxId, wp<FilteredListener> listener) { Mutex::Autolock l(mInputMutex); @@ -63,13 +63,20 @@ status_t ProFrameProcessor::removeListener(int32_t minId, return OK; } -void ProFrameProcessor::dump(int fd, const Vector<String16>& /*args*/) { +void FrameProcessorBase::dump(int fd, const Vector<String16>& /*args*/) { String8 result(" Latest received frame:\n"); write(fd, result.string(), result.size()); - mLastFrame.dump(fd, 2, 6); + + CameraMetadata lastFrame; + { + // Don't race while dumping metadata + Mutex::Autolock al(mLastFrameMutex); + lastFrame = CameraMetadata(mLastFrame); + } + lastFrame.dump(fd, 2, 6); } -bool ProFrameProcessor::threadLoop() { +bool FrameProcessorBase::threadLoop() { status_t res; sp<CameraDeviceBase> device; @@ -82,14 +89,14 @@ bool ProFrameProcessor::threadLoop() { if (res == OK) { processNewFrames(device); } else if (res != TIMED_OUT) { - ALOGE("ProFrameProcessor: Error waiting for new " + ALOGE("FrameProcessorBase: Error waiting for new " "frames: %s (%d)", strerror(-res), res); } return true; } -void ProFrameProcessor::processNewFrames(const sp<CameraDeviceBase> &device) { +void FrameProcessorBase::processNewFrames(const sp<CameraDeviceBase> &device) { status_t res; ATRACE_CALL(); CameraMetadata frame; @@ -113,6 +120,7 @@ void ProFrameProcessor::processNewFrames(const sp<CameraDeviceBase> &device) { } if (!frame.isEmpty()) { + Mutex::Autolock al(mLastFrameMutex); mLastFrame.acquire(frame); } } @@ -125,14 +133,14 @@ void ProFrameProcessor::processNewFrames(const sp<CameraDeviceBase> &device) { return; } -bool ProFrameProcessor::processSingleFrame(CameraMetadata &frame, +bool FrameProcessorBase::processSingleFrame(CameraMetadata &frame, const sp<CameraDeviceBase> &device) { ALOGV("%s: Camera %d: Process single frame (is empty? %d)", __FUNCTION__, device->getId(), frame.isEmpty()); return processListeners(frame, device) == OK; } -status_t ProFrameProcessor::processListeners(const CameraMetadata &frame, +status_t FrameProcessorBase::processListeners(const CameraMetadata &frame, const sp<CameraDeviceBase> &device) { ATRACE_CALL(); camera_metadata_ro_entry_t entry; @@ -143,7 +151,7 @@ status_t ProFrameProcessor::processListeners(const CameraMetadata &frame, __FUNCTION__, device->getId()); return BAD_VALUE; } - int32_t frameId = entry.data.i32[0]; + int32_t requestId = entry.data.i32[0]; List<sp<FilteredListener> > listeners; { @@ -151,8 +159,8 @@ status_t ProFrameProcessor::processListeners(const CameraMetadata &frame, List<RangeListener>::iterator item = mRangeListeners.begin(); while (item != mRangeListeners.end()) { - if (frameId >= item->minId && - frameId < item->maxId) { + if (requestId >= item->minId && + requestId < item->maxId) { sp<FilteredListener> listener = item->listener.promote(); if (listener == 0) { item = mRangeListeners.erase(item); @@ -167,7 +175,7 @@ status_t ProFrameProcessor::processListeners(const CameraMetadata &frame, ALOGV("Got %d range listeners out of %d", listeners.size(), mRangeListeners.size()); List<sp<FilteredListener> >::iterator item = listeners.begin(); for (; item != listeners.end(); item++) { - (*item)->onFrameAvailable(frameId, frame); + (*item)->onFrameAvailable(requestId, frame); } return OK; } diff --git a/services/camera/libcameraservice/camera2/ProFrameProcessor.h b/services/camera/libcameraservice/common/FrameProcessorBase.h index b82942c..4d80ebf 100644 --- a/services/camera/libcameraservice/camera2/ProFrameProcessor.h +++ b/services/camera/libcameraservice/common/FrameProcessorBase.h @@ -33,13 +33,13 @@ namespace camera2 { /* Output frame metadata processing thread. This thread waits for new * frames from the device, and analyzes them as necessary. */ -class ProFrameProcessor: public Thread { +class FrameProcessorBase: public Thread { public: - ProFrameProcessor(wp<CameraDeviceBase> device); - virtual ~ProFrameProcessor(); + FrameProcessorBase(wp<CameraDeviceBase> device); + virtual ~FrameProcessorBase(); struct FilteredListener: virtual public RefBase { - virtual void onFrameAvailable(int32_t frameId, + virtual void onFrameAvailable(int32_t requestId, const CameraMetadata &frame) = 0; }; @@ -58,6 +58,7 @@ class ProFrameProcessor: public Thread { virtual bool threadLoop(); Mutex mInputMutex; + Mutex mLastFrameMutex; struct RangeListener { int32_t minId; diff --git a/services/camera/libcameraservice/CameraHardwareInterface.h b/services/camera/libcameraservice/device1/CameraHardwareInterface.h index 87b2807..87b2807 100644 --- a/services/camera/libcameraservice/CameraHardwareInterface.h +++ b/services/camera/libcameraservice/device1/CameraHardwareInterface.h diff --git a/services/camera/libcameraservice/Camera2Device.cpp b/services/camera/libcameraservice/device2/Camera2Device.cpp index 710d0e9..2bc1a8a 100644 --- a/services/camera/libcameraservice/Camera2Device.cpp +++ b/services/camera/libcameraservice/device2/Camera2Device.cpp @@ -464,8 +464,10 @@ void Camera2Device::notificationCallback(int32_t msg_type, listener->notifyError(ext1, ext2, ext3); break; case CAMERA2_MSG_SHUTTER: { - nsecs_t timestamp = (nsecs_t)ext2 | ((nsecs_t)(ext3) << 32 ); - listener->notifyShutter(ext1, timestamp); + // TODO: Only needed for camera2 API, which is unsupported + // by HAL2 directly. + // nsecs_t timestamp = (nsecs_t)ext2 | ((nsecs_t)(ext3) << 32 ); + // listener->notifyShutter(requestId, timestamp); break; } case CAMERA2_MSG_AUTOFOCUS: @@ -567,6 +569,13 @@ status_t Camera2Device::pushReprocessBuffer(int reprocessStreamId, return res; } +status_t Camera2Device::flush() { + ATRACE_CALL(); + + mRequestQueue.clear(); + return waitUntilDrained(); +} + /** * Camera2Device::MetadataQueue */ @@ -591,9 +600,7 @@ Camera2Device::MetadataQueue::MetadataQueue(): Camera2Device::MetadataQueue::~MetadataQueue() { ATRACE_CALL(); - Mutex::Autolock l(mMutex); - freeBuffers(mEntries.begin(), mEntries.end()); - freeBuffers(mStreamSlot.begin(), mStreamSlot.end()); + clear(); } // Connect to camera2 HAL as consumer (input requests/reprocessing) @@ -784,6 +791,23 @@ status_t Camera2Device::MetadataQueue::setStreamSlot( return signalConsumerLocked(); } +status_t Camera2Device::MetadataQueue::clear() +{ + ATRACE_CALL(); + ALOGV("%s: E", __FUNCTION__); + + Mutex::Autolock l(mMutex); + + // Clear streaming slot + freeBuffers(mStreamSlot.begin(), mStreamSlot.end()); + mStreamSlotCount = 0; + + // Clear request queue + freeBuffers(mEntries.begin(), mEntries.end()); + mCount = 0; + return OK; +} + status_t Camera2Device::MetadataQueue::dump(int fd, const Vector<String16>& /*args*/) { ATRACE_CALL(); diff --git a/services/camera/libcameraservice/Camera2Device.h b/services/camera/libcameraservice/device2/Camera2Device.h index 372ce9f..1f53c56 100644 --- a/services/camera/libcameraservice/Camera2Device.h +++ b/services/camera/libcameraservice/device2/Camera2Device.h @@ -22,12 +22,16 @@ #include <utils/List.h> #include <utils/Mutex.h> -#include "CameraDeviceBase.h" +#include "common/CameraDeviceBase.h" namespace android { /** * CameraDevice for HAL devices with version CAMERA_DEVICE_API_VERSION_2_0 + * + * TODO for camera2 API implementation: + * Does not produce notifyShutter / notifyIdle callbacks to NotificationListener + * Use waitUntilDrained for idle. */ class Camera2Device: public CameraDeviceBase { public: @@ -67,6 +71,8 @@ class Camera2Device: public CameraDeviceBase { virtual status_t triggerPrecaptureMetering(uint32_t id); virtual status_t pushReprocessBuffer(int reprocessStreamId, buffer_handle_t *buffer, wp<BufferReleasedListener> listener); + // Flush implemented as just a wait + virtual status_t flush(); private: const int mId; camera2_device_t *mHal2Device; @@ -113,6 +119,9 @@ class Camera2Device: public CameraDeviceBase { status_t setStreamSlot(camera_metadata_t *buf); status_t setStreamSlot(const List<camera_metadata_t*> &bufs); + // Clear the request queue and the streaming slot + status_t clear(); + status_t dump(int fd, const Vector<String16>& args); private: diff --git a/services/camera/libcameraservice/Camera3Device.cpp b/services/camera/libcameraservice/device3/Camera3Device.cpp index 353fe74..6f2dc85 100644 --- a/services/camera/libcameraservice/Camera3Device.cpp +++ b/services/camera/libcameraservice/device3/Camera3Device.cpp @@ -40,9 +40,11 @@ #include <utils/Log.h> #include <utils/Trace.h> #include <utils/Timers.h> -#include "Camera3Device.h" -#include "camera3/Camera3OutputStream.h" -#include "camera3/Camera3InputStream.h" + +#include "device3/Camera3Device.h" +#include "device3/Camera3OutputStream.h" +#include "device3/Camera3InputStream.h" +#include "device3/Camera3ZslStream.h" using namespace android::camera3; @@ -80,6 +82,7 @@ int Camera3Device::getId() const { status_t Camera3Device::initialize(camera_module_t *module) { ATRACE_CALL(); + Mutex::Autolock il(mInterfaceLock); Mutex::Autolock l(mLock); ALOGV("%s: Initializing device for camera %d", __FUNCTION__, mId); @@ -128,7 +131,10 @@ status_t Camera3Device::initialize(camera_module_t *module) /** Initialize device with callback functions */ + ATRACE_BEGIN("camera3->initialize"); res = device->ops->initialize(device, this); + ATRACE_END(); + if (res != OK) { SET_ERR_L("Unable to initialize HAL device: %s (%d)", strerror(-res), res); @@ -140,7 +146,9 @@ status_t Camera3Device::initialize(camera_module_t *module) mVendorTagOps.get_camera_vendor_section_name = NULL; + ATRACE_BEGIN("camera3->get_metadata_vendor_tag_ops"); device->ops->get_metadata_vendor_tag_ops(device, &mVendorTagOps); + ATRACE_END(); if (mVendorTagOps.get_camera_vendor_section_name != NULL) { res = set_camera_metadata_vendor_tag_ops(&mVendorTagOps); @@ -152,9 +160,20 @@ status_t Camera3Device::initialize(camera_module_t *module) } } + /** Start up status tracker thread */ + mStatusTracker = new StatusTracker(this); + res = mStatusTracker->run(String8::format("C3Dev-%d-Status", mId).string()); + if (res != OK) { + SET_ERR_L("Unable to start status tracking thread: %s (%d)", + strerror(-res), res); + device->common.close(&device->common); + mStatusTracker.clear(); + return res; + } + /** Start up request queue thread */ - mRequestThread = new RequestThread(this, device); + mRequestThread = new RequestThread(this, mStatusTracker, device); res = mRequestThread->run(String8::format("C3Dev-%d-ReqQueue", mId).string()); if (res != OK) { SET_ERR_L("Unable to start request queue thread: %s (%d)", @@ -168,81 +187,130 @@ status_t Camera3Device::initialize(camera_module_t *module) mDeviceInfo = info.static_camera_characteristics; mHal3Device = device; - mStatus = STATUS_IDLE; + mStatus = STATUS_UNCONFIGURED; mNextStreamId = 0; mNeedConfig = true; + mPauseStateNotify = false; return OK; } status_t Camera3Device::disconnect() { ATRACE_CALL(); - Mutex::Autolock l(mLock); + Mutex::Autolock il(mInterfaceLock); ALOGV("%s: E", __FUNCTION__); status_t res = OK; - if (mStatus == STATUS_UNINITIALIZED) return res; - if (mStatus == STATUS_ACTIVE || - (mStatus == STATUS_ERROR && mRequestThread != NULL)) { - res = mRequestThread->clearRepeatingRequests(); - if (res != OK) { - SET_ERR_L("Can't stop streaming"); - // Continue to close device even in case of error - } else { - res = waitUntilDrainedLocked(); + { + Mutex::Autolock l(mLock); + if (mStatus == STATUS_UNINITIALIZED) return res; + + if (mStatus == STATUS_ACTIVE || + (mStatus == STATUS_ERROR && mRequestThread != NULL)) { + res = mRequestThread->clearRepeatingRequests(); if (res != OK) { - SET_ERR_L("Timeout waiting for HAL to drain"); + SET_ERR_L("Can't stop streaming"); // Continue to close device even in case of error + } else { + res = waitUntilStateThenRelock(/*active*/ false, kShutdownTimeout); + if (res != OK) { + SET_ERR_L("Timeout waiting for HAL to drain"); + // Continue to close device even in case of error + } } } + + if (mStatus == STATUS_ERROR) { + CLOGE("Shutting down in an error state"); + } + + if (mStatusTracker != NULL) { + mStatusTracker->requestExit(); + } + + if (mRequestThread != NULL) { + mRequestThread->requestExit(); + } + + mOutputStreams.clear(); + mInputStream.clear(); } - assert(mStatus == STATUS_IDLE || mStatus == STATUS_ERROR); - if (mStatus == STATUS_ERROR) { - CLOGE("Shutting down in an error state"); + // Joining done without holding mLock, otherwise deadlocks may ensue + // as the threads try to access parent state + if (mRequestThread != NULL && mStatus != STATUS_ERROR) { + // HAL may be in a bad state, so waiting for request thread + // (which may be stuck in the HAL processCaptureRequest call) + // could be dangerous. + mRequestThread->join(); } - if (mRequestThread != NULL) { - mRequestThread->requestExit(); + if (mStatusTracker != NULL) { + mStatusTracker->join(); } - mOutputStreams.clear(); - mInputStream.clear(); + { + Mutex::Autolock l(mLock); - if (mRequestThread != NULL) { - if (mStatus != STATUS_ERROR) { - // HAL may be in a bad state, so waiting for request thread - // (which may be stuck in the HAL processCaptureRequest call) - // could be dangerous. - mRequestThread->join(); - } mRequestThread.clear(); - } + mStatusTracker.clear(); - if (mHal3Device != NULL) { - mHal3Device->common.close(&mHal3Device->common); - mHal3Device = NULL; - } + if (mHal3Device != NULL) { + mHal3Device->common.close(&mHal3Device->common); + mHal3Device = NULL; + } - mStatus = STATUS_UNINITIALIZED; + mStatus = STATUS_UNINITIALIZED; + } ALOGV("%s: X", __FUNCTION__); return res; } +// For dumping/debugging only - +// try to acquire a lock a few times, eventually give up to proceed with +// debug/dump operations +bool Camera3Device::tryLockSpinRightRound(Mutex& lock) { + bool gotLock = false; + for (size_t i = 0; i < kDumpLockAttempts; ++i) { + if (lock.tryLock() == NO_ERROR) { + gotLock = true; + break; + } else { + usleep(kDumpSleepDuration); + } + } + return gotLock; +} + status_t Camera3Device::dump(int fd, const Vector<String16> &args) { ATRACE_CALL(); (void)args; + + // Try to lock, but continue in case of failure (to avoid blocking in + // deadlocks) + bool gotInterfaceLock = tryLockSpinRightRound(mInterfaceLock); + bool gotLock = tryLockSpinRightRound(mLock); + + ALOGW_IF(!gotInterfaceLock, + "Camera %d: %s: Unable to lock interface lock, proceeding anyway", + mId, __FUNCTION__); + ALOGW_IF(!gotLock, + "Camera %d: %s: Unable to lock main lock, proceeding anyway", + mId, __FUNCTION__); + String8 lines; const char *status = mStatus == STATUS_ERROR ? "ERROR" : mStatus == STATUS_UNINITIALIZED ? "UNINITIALIZED" : - mStatus == STATUS_IDLE ? "IDLE" : + mStatus == STATUS_UNCONFIGURED ? "UNCONFIGURED" : + mStatus == STATUS_CONFIGURED ? "CONFIGURED" : mStatus == STATUS_ACTIVE ? "ACTIVE" : "Unknown"; + lines.appendFormat(" Device status: %s\n", status); if (mStatus == STATUS_ERROR) { lines.appendFormat(" Error cause: %s\n", mErrorCause.string()); @@ -274,12 +342,23 @@ status_t Camera3Device::dump(int fd, const Vector<String16> &args) { } write(fd, lines.string(), lines.size()); + { + lines = String8(" Last request sent:\n"); + write(fd, lines.string(), lines.size()); + + CameraMetadata lastRequest = getLatestRequestLocked(); + lastRequest.dump(fd, /*verbosity*/2, /*indentation*/6); + } + if (mHal3Device != NULL) { lines = String8(" HAL device dump:\n"); write(fd, lines.string(), lines.size()); mHal3Device->ops->dump(mHal3Device, fd); } + if (gotLock) mLock.unlock(); + if (gotInterfaceLock) mInterfaceLock.unlock(); + return OK; } @@ -296,6 +375,8 @@ const CameraMetadata& Camera3Device::info() const { status_t Camera3Device::capture(CameraMetadata &request) { ATRACE_CALL(); + status_t res; + Mutex::Autolock il(mInterfaceLock); Mutex::Autolock l(mLock); // TODO: take ownership of the request @@ -307,7 +388,9 @@ status_t Camera3Device::capture(CameraMetadata &request) { case STATUS_UNINITIALIZED: CLOGE("Device not initialized"); return INVALID_OPERATION; - case STATUS_IDLE: + case STATUS_UNCONFIGURED: + // May be lazily configuring streams, will check during setup + case STATUS_CONFIGURED: case STATUS_ACTIVE: // OK break; @@ -322,12 +405,23 @@ status_t Camera3Device::capture(CameraMetadata &request) { return BAD_VALUE; } - return mRequestThread->queueRequest(newRequest); + res = mRequestThread->queueRequest(newRequest); + if (res == OK) { + waitUntilStateThenRelock(/*active*/ true, kActiveTimeout); + if (res != OK) { + SET_ERR_L("Can't transition to active in %f seconds!", + kActiveTimeout/1e9); + } + ALOGV("Camera %d: Capture request enqueued", mId); + } + return res; } status_t Camera3Device::setStreamingRequest(const CameraMetadata &request) { ATRACE_CALL(); + status_t res; + Mutex::Autolock il(mInterfaceLock); Mutex::Autolock l(mLock); switch (mStatus) { @@ -337,7 +431,9 @@ status_t Camera3Device::setStreamingRequest(const CameraMetadata &request) { case STATUS_UNINITIALIZED: CLOGE("Device not initialized"); return INVALID_OPERATION; - case STATUS_IDLE: + case STATUS_UNCONFIGURED: + // May be lazily configuring streams, will check during setup + case STATUS_CONFIGURED: case STATUS_ACTIVE: // OK break; @@ -355,7 +451,16 @@ status_t Camera3Device::setStreamingRequest(const CameraMetadata &request) { RequestList newRepeatingRequests; newRepeatingRequests.push_back(newRepeatingRequest); - return mRequestThread->setRepeatingRequests(newRepeatingRequests); + res = mRequestThread->setRepeatingRequests(newRepeatingRequests); + if (res == OK) { + waitUntilStateThenRelock(/*active*/ true, kActiveTimeout); + if (res != OK) { + SET_ERR_L("Can't transition to active in %f seconds!", + kActiveTimeout/1e9); + } + ALOGV("Camera %d: Repeating request set", mId); + } + return res; } @@ -363,12 +468,16 @@ sp<Camera3Device::CaptureRequest> Camera3Device::setUpRequestLocked( const CameraMetadata &request) { status_t res; - if (mStatus == STATUS_IDLE) { + if (mStatus == STATUS_UNCONFIGURED || mNeedConfig) { res = configureStreamsLocked(); if (res != OK) { SET_ERR_L("Can't set up streams: %s (%d)", strerror(-res), res); return NULL; } + if (mStatus == STATUS_UNCONFIGURED) { + CLOGE("No streams configured"); + return NULL; + } } sp<CaptureRequest> newRequest = createCaptureRequest(request); @@ -377,6 +486,7 @@ sp<Camera3Device::CaptureRequest> Camera3Device::setUpRequestLocked( status_t Camera3Device::clearStreamingRequest() { ATRACE_CALL(); + Mutex::Autolock il(mInterfaceLock); Mutex::Autolock l(mLock); switch (mStatus) { @@ -386,7 +496,8 @@ status_t Camera3Device::clearStreamingRequest() { case STATUS_UNINITIALIZED: CLOGE("Device not initialized"); return INVALID_OPERATION; - case STATUS_IDLE: + case STATUS_UNCONFIGURED: + case STATUS_CONFIGURED: case STATUS_ACTIVE: // OK break; @@ -394,12 +505,13 @@ status_t Camera3Device::clearStreamingRequest() { SET_ERR_L("Unexpected status: %d", mStatus); return INVALID_OPERATION; } - + ALOGV("Camera %d: Clearing repeating request", mId); return mRequestThread->clearRepeatingRequests(); } status_t Camera3Device::waitUntilRequestReceived(int32_t requestId, nsecs_t timeout) { ATRACE_CALL(); + Mutex::Autolock il(mInterfaceLock); return mRequestThread->waitUntilRequestProcessed(requestId, timeout); } @@ -407,7 +519,10 @@ status_t Camera3Device::waitUntilRequestReceived(int32_t requestId, nsecs_t time status_t Camera3Device::createInputStream( uint32_t width, uint32_t height, int format, int *id) { ATRACE_CALL(); + Mutex::Autolock il(mInterfaceLock); Mutex::Autolock l(mLock); + ALOGV("Camera %d: Creating new input stream %d: %d x %d, format %d", + mId, mNextStreamId, width, height, format); status_t res; bool wasActive = false; @@ -419,26 +534,24 @@ status_t Camera3Device::createInputStream( case STATUS_UNINITIALIZED: ALOGE("%s: Device not initialized", __FUNCTION__); return INVALID_OPERATION; - case STATUS_IDLE: + case STATUS_UNCONFIGURED: + case STATUS_CONFIGURED: // OK break; case STATUS_ACTIVE: ALOGV("%s: Stopping activity to reconfigure streams", __FUNCTION__); - mRequestThread->setPaused(true); - res = waitUntilDrainedLocked(); + res = internalPauseAndWaitLocked(); if (res != OK) { - ALOGE("%s: Can't pause captures to reconfigure streams!", - __FUNCTION__); - mStatus = STATUS_ERROR; + SET_ERR_L("Can't pause captures to reconfigure streams!"); return res; } wasActive = true; break; default: - ALOGE("%s: Unexpected status: %d", __FUNCTION__, mStatus); + SET_ERR_L("%s: Unexpected status: %d", mStatus); return INVALID_OPERATION; } - assert(mStatus == STATUS_IDLE); + assert(mStatus != STATUS_ACTIVE); if (mInputStream != 0) { ALOGE("%s: Cannot create more than 1 input stream", __FUNCTION__); @@ -447,6 +560,7 @@ status_t Camera3Device::createInputStream( sp<Camera3InputStream> newStream = new Camera3InputStream(mNextStreamId, width, height, format); + newStream->setStatusTracker(mStatusTracker); mInputStream = newStream; @@ -461,9 +575,10 @@ status_t Camera3Device::createInputStream( __FUNCTION__, mNextStreamId, strerror(-res), res); return res; } - mRequestThread->setPaused(false); + internalResumeLocked(); } + ALOGV("Camera %d: Created input stream", mId); return OK; } @@ -475,7 +590,10 @@ status_t Camera3Device::createZslStream( int *id, sp<Camera3ZslStream>* zslStream) { ATRACE_CALL(); + Mutex::Autolock il(mInterfaceLock); Mutex::Autolock l(mLock); + ALOGV("Camera %d: Creating ZSL stream %d: %d x %d, depth %d", + mId, mNextStreamId, width, height, depth); status_t res; bool wasActive = false; @@ -487,26 +605,24 @@ status_t Camera3Device::createZslStream( case STATUS_UNINITIALIZED: ALOGE("%s: Device not initialized", __FUNCTION__); return INVALID_OPERATION; - case STATUS_IDLE: + case STATUS_UNCONFIGURED: + case STATUS_CONFIGURED: // OK break; case STATUS_ACTIVE: ALOGV("%s: Stopping activity to reconfigure streams", __FUNCTION__); - mRequestThread->setPaused(true); - res = waitUntilDrainedLocked(); + res = internalPauseAndWaitLocked(); if (res != OK) { - ALOGE("%s: Can't pause captures to reconfigure streams!", - __FUNCTION__); - mStatus = STATUS_ERROR; + SET_ERR_L("Can't pause captures to reconfigure streams!"); return res; } wasActive = true; break; default: - ALOGE("%s: Unexpected status: %d", __FUNCTION__, mStatus); + SET_ERR_L("Unexpected status: %d", mStatus); return INVALID_OPERATION; } - assert(mStatus == STATUS_IDLE); + assert(mStatus != STATUS_ACTIVE); if (mInputStream != 0) { ALOGE("%s: Cannot create more than 1 input stream", __FUNCTION__); @@ -515,6 +631,7 @@ status_t Camera3Device::createZslStream( sp<Camera3ZslStream> newStream = new Camera3ZslStream(mNextStreamId, width, height, depth); + newStream->setStatusTracker(mStatusTracker); res = mOutputStreams.add(mNextStreamId, newStream); if (res < 0) { @@ -536,16 +653,20 @@ status_t Camera3Device::createZslStream( __FUNCTION__, mNextStreamId, strerror(-res), res); return res; } - mRequestThread->setPaused(false); + internalResumeLocked(); } + ALOGV("Camera %d: Created ZSL stream", mId); return OK; } status_t Camera3Device::createStream(sp<ANativeWindow> consumer, uint32_t width, uint32_t height, int format, size_t size, int *id) { ATRACE_CALL(); + Mutex::Autolock il(mInterfaceLock); Mutex::Autolock l(mLock); + ALOGV("Camera %d: Creating new stream %d: %d x %d, format %d, size %d", + mId, mNextStreamId, width, height, format, size); status_t res; bool wasActive = false; @@ -557,16 +678,15 @@ status_t Camera3Device::createStream(sp<ANativeWindow> consumer, case STATUS_UNINITIALIZED: CLOGE("Device not initialized"); return INVALID_OPERATION; - case STATUS_IDLE: + case STATUS_UNCONFIGURED: + case STATUS_CONFIGURED: // OK break; case STATUS_ACTIVE: ALOGV("%s: Stopping activity to reconfigure streams", __FUNCTION__); - mRequestThread->setPaused(true); - res = waitUntilDrainedLocked(); + res = internalPauseAndWaitLocked(); if (res != OK) { - ALOGE("%s: Can't pause captures to reconfigure streams!", - __FUNCTION__); + SET_ERR_L("Can't pause captures to reconfigure streams!"); return res; } wasActive = true; @@ -575,7 +695,7 @@ status_t Camera3Device::createStream(sp<ANativeWindow> consumer, SET_ERR_L("Unexpected status: %d", mStatus); return INVALID_OPERATION; } - assert(mStatus == STATUS_IDLE); + assert(mStatus != STATUS_ACTIVE); sp<Camera3OutputStream> newStream; if (format == HAL_PIXEL_FORMAT_BLOB) { @@ -585,6 +705,7 @@ status_t Camera3Device::createStream(sp<ANativeWindow> consumer, newStream = new Camera3OutputStream(mNextStreamId, consumer, width, height, format); } + newStream->setStatusTracker(mStatusTracker); res = mOutputStreams.add(mNextStreamId, newStream); if (res < 0) { @@ -604,9 +725,9 @@ status_t Camera3Device::createStream(sp<ANativeWindow> consumer, mNextStreamId, strerror(-res), res); return res; } - mRequestThread->setPaused(false); + internalResumeLocked(); } - + ALOGV("Camera %d: Created new stream", mId); return OK; } @@ -622,6 +743,7 @@ status_t Camera3Device::createReprocessStreamFromStream(int outputId, int *id) { status_t Camera3Device::getStreamInfo(int id, uint32_t *width, uint32_t *height, uint32_t *format) { ATRACE_CALL(); + Mutex::Autolock il(mInterfaceLock); Mutex::Autolock l(mLock); switch (mStatus) { @@ -631,7 +753,8 @@ status_t Camera3Device::getStreamInfo(int id, case STATUS_UNINITIALIZED: CLOGE("Device not initialized!"); return INVALID_OPERATION; - case STATUS_IDLE: + case STATUS_UNCONFIGURED: + case STATUS_CONFIGURED: case STATUS_ACTIVE: // OK break; @@ -656,6 +779,7 @@ status_t Camera3Device::getStreamInfo(int id, status_t Camera3Device::setStreamTransform(int id, int transform) { ATRACE_CALL(); + Mutex::Autolock il(mInterfaceLock); Mutex::Autolock l(mLock); switch (mStatus) { @@ -665,7 +789,8 @@ status_t Camera3Device::setStreamTransform(int id, case STATUS_UNINITIALIZED: CLOGE("Device not initialized"); return INVALID_OPERATION; - case STATUS_IDLE: + case STATUS_UNCONFIGURED: + case STATUS_CONFIGURED: case STATUS_ACTIVE: // OK break; @@ -686,6 +811,7 @@ status_t Camera3Device::setStreamTransform(int id, status_t Camera3Device::deleteStream(int id) { ATRACE_CALL(); + Mutex::Autolock il(mInterfaceLock); Mutex::Autolock l(mLock); status_t res; @@ -693,7 +819,7 @@ status_t Camera3Device::deleteStream(int id) { // CameraDevice semantics require device to already be idle before // deleteStream is called, unlike for createStream. - if (mStatus != STATUS_IDLE) { + if (mStatus == STATUS_ACTIVE) { ALOGV("%s: Camera %d: Device not idle", __FUNCTION__, mId); return -EBUSY; } @@ -736,7 +862,8 @@ status_t Camera3Device::deleteReprocessStream(int id) { status_t Camera3Device::createDefaultRequest(int templateId, CameraMetadata *request) { ATRACE_CALL(); - ALOGV("%s: E", __FUNCTION__); + ALOGV("%s: for template %d", __FUNCTION__, templateId); + Mutex::Autolock il(mInterfaceLock); Mutex::Autolock l(mLock); switch (mStatus) { @@ -746,7 +873,8 @@ status_t Camera3Device::createDefaultRequest(int templateId, case STATUS_UNINITIALIZED: CLOGE("Device is not initialized!"); return INVALID_OPERATION; - case STATUS_IDLE: + case STATUS_UNCONFIGURED: + case STATUS_CONFIGURED: case STATUS_ACTIVE: // OK break; @@ -756,8 +884,10 @@ status_t Camera3Device::createDefaultRequest(int templateId, } const camera_metadata_t *rawRequest; + ATRACE_BEGIN("camera3->construct_default_request_settings"); rawRequest = mHal3Device->ops->construct_default_request_settings( mHal3Device, templateId); + ATRACE_END(); if (rawRequest == NULL) { SET_ERR_L("HAL is unable to construct default settings for template %d", templateId); @@ -770,61 +900,88 @@ status_t Camera3Device::createDefaultRequest(int templateId, status_t Camera3Device::waitUntilDrained() { ATRACE_CALL(); + Mutex::Autolock il(mInterfaceLock); Mutex::Autolock l(mLock); - return waitUntilDrainedLocked(); -} - -status_t Camera3Device::waitUntilDrainedLocked() { - ATRACE_CALL(); - status_t res; - switch (mStatus) { case STATUS_UNINITIALIZED: - case STATUS_IDLE: + case STATUS_UNCONFIGURED: ALOGV("%s: Already idle", __FUNCTION__); return OK; + case STATUS_CONFIGURED: + // To avoid race conditions, check with tracker to be sure case STATUS_ERROR: case STATUS_ACTIVE: - // Need to shut down + // Need to verify shut down break; default: SET_ERR_L("Unexpected status: %d",mStatus); return INVALID_OPERATION; } - if (mRequestThread != NULL) { - res = mRequestThread->waitUntilPaused(kShutdownTimeout); - if (res != OK) { - SET_ERR_L("Can't stop request thread in %f seconds!", - kShutdownTimeout/1e9); - return res; - } - } - if (mInputStream != NULL) { - res = mInputStream->waitUntilIdle(kShutdownTimeout); - if (res != OK) { - SET_ERR_L("Can't idle input stream %d in %f seconds!", - mInputStream->getId(), kShutdownTimeout/1e9); - return res; - } + ALOGV("%s: Camera %d: Waiting until idle", __FUNCTION__, mId); + status_t res = waitUntilStateThenRelock(/*active*/ false, kShutdownTimeout); + return res; +} + +// Pause to reconfigure +status_t Camera3Device::internalPauseAndWaitLocked() { + mRequestThread->setPaused(true); + mPauseStateNotify = true; + + ALOGV("%s: Camera %d: Internal wait until idle", __FUNCTION__, mId); + status_t res = waitUntilStateThenRelock(/*active*/ false, kShutdownTimeout); + if (res != OK) { + SET_ERR_L("Can't idle device in %f seconds!", + kShutdownTimeout/1e9); } - for (size_t i = 0; i < mOutputStreams.size(); i++) { - res = mOutputStreams.editValueAt(i)->waitUntilIdle(kShutdownTimeout); - if (res != OK) { - SET_ERR_L("Can't idle output stream %d in %f seconds!", - mOutputStreams.keyAt(i), kShutdownTimeout/1e9); - return res; - } + + return res; +} + +// Resume after internalPauseAndWaitLocked +status_t Camera3Device::internalResumeLocked() { + status_t res; + + mRequestThread->setPaused(false); + + res = waitUntilStateThenRelock(/*active*/ true, kActiveTimeout); + if (res != OK) { + SET_ERR_L("Can't transition to active in %f seconds!", + kActiveTimeout/1e9); } + mPauseStateNotify = false; + return OK; +} - if (mStatus != STATUS_ERROR) { - mStatus = STATUS_IDLE; +status_t Camera3Device::waitUntilStateThenRelock(bool active, + nsecs_t timeout) { + status_t res = OK; + if (active == (mStatus == STATUS_ACTIVE)) { + // Desired state already reached + return res; } - return OK; + bool stateSeen = false; + do { + mRecentStatusUpdates.clear(); + + res = mStatusChanged.waitRelative(mLock, timeout); + if (res != OK) break; + + // Check state change history during wait + for (size_t i = 0; i < mRecentStatusUpdates.size(); i++) { + if (active == (mRecentStatusUpdates[i] == STATUS_ACTIVE) ) { + stateSeen = true; + break; + } + } + } while (!stateSeen); + + return res; } + status_t Camera3Device::setNotifyCallback(NotificationListener *listener) { ATRACE_CALL(); Mutex::Autolock l(mOutputLock); @@ -842,7 +999,6 @@ bool Camera3Device::willNotify3A() { } status_t Camera3Device::waitForNextFrame(nsecs_t timeout) { - ATRACE_CALL(); status_t res; Mutex::Autolock l(mOutputLock); @@ -876,6 +1032,7 @@ status_t Camera3Device::getNextFrame(CameraMetadata *frame) { status_t Camera3Device::triggerAutofocus(uint32_t id) { ATRACE_CALL(); + Mutex::Autolock il(mInterfaceLock); ALOGV("%s: Triggering autofocus, id %d", __FUNCTION__, id); // Mix-in this trigger into the next request and only the next request. @@ -896,6 +1053,7 @@ status_t Camera3Device::triggerAutofocus(uint32_t id) { status_t Camera3Device::triggerCancelAutofocus(uint32_t id) { ATRACE_CALL(); + Mutex::Autolock il(mInterfaceLock); ALOGV("%s: Triggering cancel autofocus, id %d", __FUNCTION__, id); // Mix-in this trigger into the next request and only the next request. @@ -916,6 +1074,7 @@ status_t Camera3Device::triggerCancelAutofocus(uint32_t id) { status_t Camera3Device::triggerPrecaptureMetering(uint32_t id) { ATRACE_CALL(); + Mutex::Autolock il(mInterfaceLock); ALOGV("%s: Triggering precapture metering, id %d", __FUNCTION__, id); // Mix-in this trigger into the next request and only the next request. @@ -943,6 +1102,51 @@ status_t Camera3Device::pushReprocessBuffer(int reprocessStreamId, return INVALID_OPERATION; } +status_t Camera3Device::flush() { + ATRACE_CALL(); + ALOGV("%s: Camera %d: Flushing all requests", __FUNCTION__, mId); + Mutex::Autolock il(mInterfaceLock); + Mutex::Autolock l(mLock); + + mRequestThread->clear(); + return mHal3Device->ops->flush(mHal3Device); +} + +/** + * Methods called by subclasses + */ + +void Camera3Device::notifyStatus(bool idle) { + { + // Need mLock to safely update state and synchronize to current + // state of methods in flight. + Mutex::Autolock l(mLock); + // We can get various system-idle notices from the status tracker + // while starting up. Only care about them if we've actually sent + // in some requests recently. + if (mStatus != STATUS_ACTIVE && mStatus != STATUS_CONFIGURED) { + return; + } + ALOGV("%s: Camera %d: Now %s", __FUNCTION__, mId, + idle ? "idle" : "active"); + mStatus = idle ? STATUS_CONFIGURED : STATUS_ACTIVE; + mRecentStatusUpdates.add(mStatus); + mStatusChanged.signal(); + + // Skip notifying listener if we're doing some user-transparent + // state changes + if (mPauseStateNotify) return; + } + NotificationListener *listener; + { + Mutex::Autolock l(mOutputLock); + listener = mListener; + } + if (idle && listener != NULL) { + listener->notifyIdle(); + } +} + /** * Camera3Device private methods */ @@ -959,7 +1163,7 @@ sp<Camera3Device::CaptureRequest> Camera3Device::createCaptureRequest( newRequest->mSettings.find(ANDROID_REQUEST_INPUT_STREAMS); if (inputStreams.count > 0) { if (mInputStream == NULL || - mInputStream->getId() != inputStreams.data.u8[0]) { + mInputStream->getId() != inputStreams.data.i32[0]) { CLOGE("Request references unknown input stream %d", inputStreams.data.u8[0]); return NULL; @@ -988,7 +1192,7 @@ sp<Camera3Device::CaptureRequest> Camera3Device::createCaptureRequest( } for (size_t i = 0; i < streams.count; i++) { - int idx = mOutputStreams.indexOfKey(streams.data.u8[i]); + int idx = mOutputStreams.indexOfKey(streams.data.i32[i]); if (idx == NAME_NOT_FOUND) { CLOGE("Request references unknown stream %d", streams.data.u8[i]); @@ -1019,18 +1223,18 @@ status_t Camera3Device::configureStreamsLocked() { ATRACE_CALL(); status_t res; - if (mStatus != STATUS_IDLE) { + if (mStatus != STATUS_UNCONFIGURED && mStatus != STATUS_CONFIGURED) { CLOGE("Not idle"); return INVALID_OPERATION; } if (!mNeedConfig) { ALOGV("%s: Skipping config, no stream changes", __FUNCTION__); - mStatus = STATUS_ACTIVE; return OK; } // Start configuring the streams + ALOGV("%s: Camera %d: Starting stream configuration", __FUNCTION__, mId); camera3_stream_configuration config; @@ -1072,8 +1276,9 @@ status_t Camera3Device::configureStreamsLocked() { // Do the HAL configuration; will potentially touch stream // max_buffers, usage, priv fields. - + ATRACE_BEGIN("camera3->configure_streams"); res = mHal3Device->ops->configure_streams(mHal3Device, &config); + ATRACE_END(); if (res != OK) { SET_ERR_L("Unable to configure streams with HAL: %s (%d)", @@ -1111,11 +1316,18 @@ status_t Camera3Device::configureStreamsLocked() { // across configure_streams() calls mRequestThread->configurationComplete(); - // Finish configuring the streams lazily on first reference + // Update device state - mStatus = STATUS_ACTIVE; mNeedConfig = false; + if (config.num_streams > 0) { + mStatus = STATUS_CONFIGURED; + } else { + mStatus = STATUS_UNCONFIGURED; + } + + ALOGV("%s: Camera %d: Stream configuration complete", __FUNCTION__, mId); + return OK; } @@ -1162,12 +1374,12 @@ void Camera3Device::setErrorStateLockedV(const char *fmt, va_list args) { */ status_t Camera3Device::registerInFlight(int32_t frameNumber, - int32_t numBuffers) { + int32_t requestId, int32_t numBuffers) { ATRACE_CALL(); Mutex::Autolock l(mInFlightLock); ssize_t res; - res = mInFlightMap.add(frameNumber, InFlightRequest(numBuffers)); + res = mInFlightMap.add(frameNumber, InFlightRequest(requestId, numBuffers)); if (res < 0) return res; return OK; @@ -1204,7 +1416,12 @@ void Camera3Device::processCaptureResult(const camera3_capture_result *result) { } InFlightRequest &request = mInFlightMap.editValueAt(idx); timestamp = request.captureTimestamp; - if (timestamp == 0) { + /** + * One of the following must happen before it's legal to call process_capture_result: + * - CAMERA3_MSG_SHUTTER (expected during normal operation) + * - CAMERA3_MSG_ERROR (expected during flush) + */ + if (request.requestStatus == OK && timestamp == 0) { SET_ERR("Called before shutter notify for frame %d", frameNumber); return; @@ -1228,6 +1445,7 @@ void Camera3Device::processCaptureResult(const camera3_capture_result *result) { } if (request.haveResultMetadata && request.numBuffersLeft == 0) { + ATRACE_ASYNC_END("frame capture", frameNumber); mInFlightMap.removeItemsAt(idx, 1); } @@ -1271,8 +1489,7 @@ void Camera3Device::processCaptureResult(const camera3_capture_result *result) { if (entry.count == 0) { SET_ERR("No timestamp provided by HAL for frame %d!", frameNumber); - } - if (timestamp != entry.data.i64[0]) { + } else if (timestamp != entry.data.i64[0]) { SET_ERR("Timestamp mismatch between shutter notify and result" " metadata for frame %d (%lld vs %lld respectively)", frameNumber, timestamp, entry.data.i64[0]); @@ -1288,7 +1505,7 @@ void Camera3Device::processCaptureResult(const camera3_capture_result *result) { // Note: stream may be deallocated at this point, if this buffer was the // last reference to it. if (res != OK) { - SET_ERR("Can't return buffer %d for frame %d to its stream: " + ALOGE("Can't return buffer %d for frame %d to its stream: " " %s (%d)", i, frameNumber, strerror(-res), res); } } @@ -1304,6 +1521,7 @@ void Camera3Device::processCaptureResult(const camera3_capture_result *result) { void Camera3Device::notify(const camera3_notify_msg *msg) { + ATRACE_CALL(); NotificationListener *listener; { Mutex::Autolock l(mOutputLock); @@ -1324,6 +1542,19 @@ void Camera3Device::notify(const camera3_notify_msg *msg) { msg->message.error.error_stream); streamId = stream->getId(); } + ALOGV("Camera %d: %s: HAL error, frame %d, stream %d: %d", + mId, __FUNCTION__, msg->message.error.frame_number, + streamId, msg->message.error.error_code); + + // Set request error status for the request in the in-flight tracking + { + Mutex::Autolock l(mInFlightLock); + ssize_t idx = mInFlightMap.indexOfKey(msg->message.error.frame_number); + if (idx >= 0) { + mInFlightMap.editValueAt(idx).requestStatus = msg->message.error.error_code; + } + } + if (listener != NULL) { listener->notifyError(msg->message.error.error_code, msg->message.error.frame_number, streamId); @@ -1346,12 +1577,17 @@ void Camera3Device::notify(const camera3_notify_msg *msg) { mNextShutterFrameNumber++; } + int32_t requestId = -1; + // Set timestamp for the request in the in-flight tracking + // and get the request ID to send upstream { Mutex::Autolock l(mInFlightLock); idx = mInFlightMap.indexOfKey(frameNumber); if (idx >= 0) { - mInFlightMap.editValueAt(idx).captureTimestamp = timestamp; + InFlightRequest &r = mInFlightMap.editValueAt(idx); + r.captureTimestamp = timestamp; + requestId = r.requestId; } } if (idx < 0) { @@ -1359,10 +1595,11 @@ void Camera3Device::notify(const camera3_notify_msg *msg) { frameNumber); break; } - + ALOGVV("Camera %d: %s: Shutter fired for frame %d (id %d) at %lld", + mId, __FUNCTION__, frameNumber, requestId, timestamp); // Call listener, if any if (listener != NULL) { - listener->notifyShutter(frameNumber, timestamp); + listener->notifyShutter(requestId, timestamp); } break; } @@ -1372,14 +1609,28 @@ void Camera3Device::notify(const camera3_notify_msg *msg) { } } +CameraMetadata Camera3Device::getLatestRequestLocked() { + ALOGV("%s", __FUNCTION__); + + CameraMetadata retVal; + + if (mRequestThread != NULL) { + retVal = mRequestThread->getLatestRequest(); + } + + return retVal; +} + /** * RequestThread inner class methods */ Camera3Device::RequestThread::RequestThread(wp<Camera3Device> parent, + sp<StatusTracker> statusTracker, camera3_device_t *hal3Device) : Thread(false), mParent(parent), + mStatusTracker(statusTracker), mHal3Device(hal3Device), mId(getId(parent)), mReconfigured(false), @@ -1387,6 +1638,7 @@ Camera3Device::RequestThread::RequestThread(wp<Camera3Device> parent, mPaused(true), mFrameNumber(0), mLatestRequestId(NAME_NOT_FOUND) { + mStatusId = statusTracker->addComponent(); } void Camera3Device::RequestThread::configurationComplete() { @@ -1399,6 +1651,8 @@ status_t Camera3Device::RequestThread::queueRequest( Mutex::Autolock l(mRequestLock); mRequestQueue.push_back(request); + unpauseForNewRequests(); + return OK; } @@ -1464,6 +1718,9 @@ status_t Camera3Device::RequestThread::setRepeatingRequests( mRepeatingRequests.clear(); mRepeatingRequests.insert(mRepeatingRequests.begin(), requests.begin(), requests.end()); + + unpauseForNewRequests(); + return OK; } @@ -1473,24 +1730,20 @@ status_t Camera3Device::RequestThread::clearRepeatingRequests() { return OK; } +status_t Camera3Device::RequestThread::clear() { + Mutex::Autolock l(mRequestLock); + mRepeatingRequests.clear(); + mRequestQueue.clear(); + mTriggerMap.clear(); + return OK; +} + void Camera3Device::RequestThread::setPaused(bool paused) { Mutex::Autolock l(mPauseLock); mDoPause = paused; mDoPauseSignal.signal(); } -status_t Camera3Device::RequestThread::waitUntilPaused(nsecs_t timeout) { - status_t res; - Mutex::Autolock l(mPauseLock); - while (!mPaused) { - res = mPausedSignal.waitRelative(mPauseLock, timeout); - if (res == TIMED_OUT) { - return res; - } - } - return OK; -} - status_t Camera3Device::RequestThread::waitUntilRequestProcessed( int32_t requestId, nsecs_t timeout) { Mutex::Autolock l(mLatestRequestMutex); @@ -1507,7 +1760,13 @@ status_t Camera3Device::RequestThread::waitUntilRequestProcessed( return OK; } - +void Camera3Device::RequestThread::requestExit() { + // Call parent to set up shutdown + Thread::requestExit(); + // The exit from any possible waits + mDoPauseSignal.signal(); + mRequestSignal.signal(); +} bool Camera3Device::RequestThread::threadLoop() { @@ -1529,6 +1788,18 @@ bool Camera3Device::RequestThread::threadLoop() { camera3_capture_request_t request = camera3_capture_request_t(); Vector<camera3_stream_buffer_t> outputBuffers; + // Get the request ID, if any + int requestId; + camera_metadata_entry_t requestIdEntry = + nextRequest->mSettings.find(ANDROID_REQUEST_ID); + if (requestIdEntry.count > 0) { + requestId = requestIdEntry.data.i32[0]; + } else { + ALOGW("%s: Did not have android.request.id set in the request", + __FUNCTION__); + requestId = NAME_NOT_FOUND; + } + // Insert any queued triggers (before metadata is locked) int32_t triggerCount; res = insertTriggers(nextRequest); @@ -1546,6 +1817,19 @@ bool Camera3Device::RequestThread::threadLoop() { // If the request is the same as last, or we had triggers last time if (mPrevRequest != nextRequest || triggersMixedIn) { /** + * HAL workaround: + * Insert a dummy trigger ID if a trigger is set but no trigger ID is + */ + res = addDummyTriggerIds(nextRequest); + if (res != OK) { + SET_ERR("RequestThread: Unable to insert dummy trigger IDs " + "(capture request %d, HAL device: %s (%d)", + (mFrameNumber+1), strerror(-res), res); + cleanUpFailedRequest(request, nextRequest, outputBuffers); + return false; + } + + /** * The request should be presorted so accesses in HAL * are O(logn). Sidenote, sorting a sorted metadata is nop. */ @@ -1582,7 +1866,7 @@ bool Camera3Device::RequestThread::threadLoop() { request.input_buffer = &inputBuffer; res = nextRequest->mInputStream->getInputBuffer(&inputBuffer); if (res != OK) { - SET_ERR("RequestThread: Can't get input buffer, skipping request:" + ALOGE("RequestThread: Can't get input buffer, skipping request:" " %s (%d)", strerror(-res), res); cleanUpFailedRequest(request, nextRequest, outputBuffers); return true; @@ -1598,8 +1882,8 @@ bool Camera3Device::RequestThread::threadLoop() { res = nextRequest->mOutputStreams.editItemAt(i)-> getBuffer(&outputBuffers.editItemAt(i)); if (res != OK) { - SET_ERR("RequestThread: Can't get output buffer, skipping request:" - "%s (%d)", strerror(-res), res); + ALOGE("RequestThread: Can't get output buffer, skipping request:" + " %s (%d)", strerror(-res), res); cleanUpFailedRequest(request, nextRequest, outputBuffers); return true; } @@ -1616,7 +1900,7 @@ bool Camera3Device::RequestThread::threadLoop() { return false; } - res = parent->registerInFlight(request.frame_number, + res = parent->registerInFlight(request.frame_number, requestId, request.num_output_buffers); if (res != OK) { SET_ERR("RequestThread: Unable to register new in-flight request:" @@ -1625,9 +1909,20 @@ bool Camera3Device::RequestThread::threadLoop() { return false; } - // Submit request and block until ready for next one + // Inform waitUntilRequestProcessed thread of a new request ID + { + Mutex::Autolock al(mLatestRequestMutex); + + mLatestRequestId = requestId; + mLatestRequestSignal.signal(); + } + // Submit request and block until ready for next one + ATRACE_ASYNC_BEGIN("frame capture", request.frame_number); + ATRACE_BEGIN("camera3->process_capture_request"); res = mHal3Device->ops->process_capture_request(mHal3Device, &request); + ATRACE_END(); + if (res != OK) { SET_ERR("RequestThread: Unable to submit capture request %d to HAL" " device: %s (%d)", request.frame_number, strerror(-res), res); @@ -1635,6 +1930,14 @@ bool Camera3Device::RequestThread::threadLoop() { return false; } + // Update the latest request sent to HAL + if (request.settings != NULL) { // Don't update them if they were unchanged + Mutex::Autolock al(mLatestRequestMutex); + + camera_metadata_t* cloned = clone_camera_metadata(request.settings); + mLatestRequest.acquire(cloned); + } + if (request.settings != NULL) { nextRequest->mSettings.unlock(request.settings); } @@ -1649,24 +1952,6 @@ bool Camera3Device::RequestThread::threadLoop() { } mPrevTriggers = triggerCount; - // Read android.request.id from the request settings metadata - // - inform waitUntilRequestProcessed thread of a new request ID - { - Mutex::Autolock al(mLatestRequestMutex); - - camera_metadata_entry_t requestIdEntry = - nextRequest->mSettings.find(ANDROID_REQUEST_ID); - if (requestIdEntry.count > 0) { - mLatestRequestId = requestIdEntry.data.i32[0]; - } else { - ALOGW("%s: Did not have android.request.id set in the request", - __FUNCTION__); - mLatestRequestId = NAME_NOT_FOUND; - } - - mLatestRequestSignal.signal(); - } - // Return input buffer back to framework if (request.input_buffer != NULL) { Camera3Stream *stream = @@ -1682,9 +1967,15 @@ bool Camera3Device::RequestThread::threadLoop() { } } + return true; +} +CameraMetadata Camera3Device::RequestThread::getLatestRequest() const { + Mutex::Autolock al(mLatestRequestMutex); - return true; + ALOGV("RequestThread::%s", __FUNCTION__); + + return mLatestRequest; } void Camera3Device::RequestThread::cleanUpFailedRequest( @@ -1733,12 +2024,17 @@ sp<Camera3Device::CaptureRequest> res = mRequestSignal.waitRelative(mRequestLock, kRequestTimeout); - if (res == TIMED_OUT) { - // Signal that we're paused by starvation + if ((mRequestQueue.empty() && mRepeatingRequests.empty()) || + exitPending()) { Mutex::Autolock pl(mPauseLock); if (mPaused == false) { + ALOGV("%s: RequestThread: Going idle", __FUNCTION__); mPaused = true; - mPausedSignal.signal(); + // Let the tracker know + sp<StatusTracker> statusTracker = mStatusTracker.promote(); + if (statusTracker != 0) { + statusTracker->markComponentIdle(mStatusId, Fence::NO_FENCE); + } } // Stop waiting for now and let thread management happen return NULL; @@ -1754,8 +2050,17 @@ sp<Camera3Device::CaptureRequest> mRequestQueue.erase(firstRequest); } - // Not paused + // In case we've been unpaused by setPaused clearing mDoPause, need to + // update internal pause state (capture/setRepeatingRequest unpause + // directly). Mutex::Autolock pl(mPauseLock); + if (mPaused) { + ALOGV("%s: RequestThread: Unpaused", __FUNCTION__); + sp<StatusTracker> statusTracker = mStatusTracker.promote(); + if (statusTracker != 0) { + statusTracker->markComponentActive(mStatusId); + } + } mPaused = false; // Check if we've reconfigured since last time, and reset the preview @@ -1772,13 +2077,18 @@ bool Camera3Device::RequestThread::waitIfPaused() { status_t res; Mutex::Autolock l(mPauseLock); while (mDoPause) { - // Signal that we're paused by request if (mPaused == false) { mPaused = true; - mPausedSignal.signal(); + ALOGV("%s: RequestThread: Paused", __FUNCTION__); + // Let the tracker know + sp<StatusTracker> statusTracker = mStatusTracker.promote(); + if (statusTracker != 0) { + statusTracker->markComponentIdle(mStatusId, Fence::NO_FENCE); + } } + res = mDoPauseSignal.waitRelative(mPauseLock, kRequestTimeout); - if (res == TIMED_OUT) { + if (res == TIMED_OUT || exitPending()) { return true; } } @@ -1787,6 +2097,24 @@ bool Camera3Device::RequestThread::waitIfPaused() { return false; } +void Camera3Device::RequestThread::unpauseForNewRequests() { + // With work to do, mark thread as unpaused. + // If paused by request (setPaused), don't resume, to avoid + // extra signaling/waiting overhead to waitUntilPaused + mRequestSignal.signal(); + Mutex::Autolock p(mPauseLock); + if (!mDoPause) { + ALOGV("%s: RequestThread: Going active", __FUNCTION__); + if (mPaused) { + sp<StatusTracker> statusTracker = mStatusTracker.promote(); + if (statusTracker != 0) { + statusTracker->markComponentActive(mStatusId); + } + } + mPaused = false; + } +} + void Camera3Device::RequestThread::setErrorState(const char *fmt, ...) { sp<Camera3Device> parent = mParent.promote(); if (parent != NULL) { @@ -1932,6 +2260,40 @@ status_t Camera3Device::RequestThread::removeTriggers( return OK; } +status_t Camera3Device::RequestThread::addDummyTriggerIds( + const sp<CaptureRequest> &request) { + // Trigger ID 0 has special meaning in the HAL2 spec, so avoid it here + static const int32_t dummyTriggerId = 1; + status_t res; + + CameraMetadata &metadata = request->mSettings; + + // If AF trigger is active, insert a dummy AF trigger ID if none already + // exists + camera_metadata_entry afTrigger = metadata.find(ANDROID_CONTROL_AF_TRIGGER); + camera_metadata_entry afId = metadata.find(ANDROID_CONTROL_AF_TRIGGER_ID); + if (afTrigger.count > 0 && + afTrigger.data.u8[0] != ANDROID_CONTROL_AF_TRIGGER_IDLE && + afId.count == 0) { + res = metadata.update(ANDROID_CONTROL_AF_TRIGGER_ID, &dummyTriggerId, 1); + if (res != OK) return res; + } + + // If AE precapture trigger is active, insert a dummy precapture trigger ID + // if none already exists + camera_metadata_entry pcTrigger = + metadata.find(ANDROID_CONTROL_AE_PRECAPTURE_TRIGGER); + camera_metadata_entry pcId = metadata.find(ANDROID_CONTROL_AE_PRECAPTURE_ID); + if (pcTrigger.count > 0 && + pcTrigger.data.u8[0] != ANDROID_CONTROL_AE_PRECAPTURE_TRIGGER_IDLE && + pcId.count == 0) { + res = metadata.update(ANDROID_CONTROL_AE_PRECAPTURE_ID, + &dummyTriggerId, 1); + if (res != OK) return res; + } + + return OK; +} /** diff --git a/services/camera/libcameraservice/Camera3Device.h b/services/camera/libcameraservice/device3/Camera3Device.h index 2328f89..12252c8 100644 --- a/services/camera/libcameraservice/Camera3Device.h +++ b/services/camera/libcameraservice/device3/Camera3Device.h @@ -22,13 +22,11 @@ #include <utils/List.h> #include <utils/Mutex.h> #include <utils/Thread.h> +#include <utils/KeyedVector.h> +#include <hardware/camera3.h> -#include "CameraDeviceBase.h" -#include "camera3/Camera3Stream.h" -#include "camera3/Camera3OutputStream.h" -#include "camera3/Camera3ZslStream.h" - -#include "hardware/camera3.h" +#include "common/CameraDeviceBase.h" +#include "device3/StatusTracker.h" /** * Function pointer types with C calling convention to @@ -46,6 +44,15 @@ extern "C" { namespace android { +namespace camera3 { + +class Camera3Stream; +class Camera3ZslStream; +class Camera3OutputStreamInterface; +class Camera3StreamInterface; + +} + /** * CameraDevice for HAL devices with version CAMERA_DEVICE_API_VERSION_3_0 */ @@ -118,27 +125,49 @@ class Camera3Device : virtual status_t pushReprocessBuffer(int reprocessStreamId, buffer_handle_t *buffer, wp<BufferReleasedListener> listener); + virtual status_t flush(); + + // Methods called by subclasses + void notifyStatus(bool idle); // updates from StatusTracker + private: + static const size_t kDumpLockAttempts = 10; + static const size_t kDumpSleepDuration = 100000; // 0.10 sec static const size_t kInFlightWarnLimit = 20; static const nsecs_t kShutdownTimeout = 5000000000; // 5 sec + static const nsecs_t kActiveTimeout = 500000000; // 500 ms struct RequestTrigger; + // A lock to enforce serialization on the input/configure side + // of the public interface. + // Only locked by public methods inherited from CameraDeviceBase. + // Not locked by methods guarded by mOutputLock, since they may act + // concurrently to the input/configure side of the interface. + // Must be locked before mLock if both will be locked by a method + Mutex mInterfaceLock; + + // The main lock on internal state Mutex mLock; + // Camera device ID + const int mId; + /**** Scope for mLock ****/ - const int mId; camera3_device_t *mHal3Device; CameraMetadata mDeviceInfo; vendor_tag_query_ops_t mVendorTagOps; - enum { + enum Status { STATUS_ERROR, STATUS_UNINITIALIZED, - STATUS_IDLE, + STATUS_UNCONFIGURED, + STATUS_CONFIGURED, STATUS_ACTIVE } mStatus; + Vector<Status> mRecentStatusUpdates; + Condition mStatusChanged; // Tracking cause of fatal errors when in STATUS_ERROR String8 mErrorCause; @@ -152,6 +181,10 @@ class Camera3Device : int mNextStreamId; bool mNeedConfig; + // Whether to send state updates upstream + // Pause when doing transparent reconfiguration + bool mPauseStateNotify; + // Need to hold on to stream references until configure completes. Vector<sp<camera3::Camera3StreamInterface> > mDeletedStreams; @@ -167,10 +200,38 @@ class Camera3Device : typedef List<sp<CaptureRequest> > RequestList; /** - * Lock-held version of waitUntilDrained. Will transition to IDLE on - * success. + * Get the last request submitted to the hal by the request thread. + * + * Takes mLock. + */ + virtual CameraMetadata getLatestRequestLocked(); + + /** + * Pause processing and flush everything, but don't tell the clients. + * This is for reconfiguring outputs transparently when according to the + * CameraDeviceBase interface we shouldn't need to. + * Must be called with mLock and mInterfaceLock both held. + */ + status_t internalPauseAndWaitLocked(); + + /** + * Resume work after internalPauseAndWaitLocked() + * Must be called with mLock and mInterfaceLock both held. + */ + status_t internalResumeLocked(); + + /** + * Wait until status tracker tells us we've transitioned to the target state + * set, which is either ACTIVE when active==true or IDLE (which is any + * non-ACTIVE state) when active==false. + * + * Needs to be called with mLock and mInterfaceLock held. This means there + * can ever only be one waiter at most. + * + * During the wait mLock is released. + * */ - status_t waitUntilDrainedLocked(); + status_t waitUntilStateThenRelock(bool active, nsecs_t timeout); /** * Do common work for setting up a streaming or single capture request. @@ -200,6 +261,12 @@ class Camera3Device : void setErrorStateLocked(const char *fmt, ...); void setErrorStateLockedV(const char *fmt, va_list args); + /** + * Debugging trylock/spin method + * Try to acquire a lock a few times with sleeps between before giving up. + */ + bool tryLockSpinRightRound(Mutex& lock); + struct RequestTrigger { // Metadata tag number, e.g. android.control.aePrecaptureTrigger uint32_t metadataTag; @@ -225,6 +292,7 @@ class Camera3Device : public: RequestThread(wp<Camera3Device> parent, + sp<camera3::StatusTracker> statusTracker, camera3_device_t *hal3Device); /** @@ -243,6 +311,11 @@ class Camera3Device : status_t queueRequest(sp<CaptureRequest> request); /** + * Remove all queued and repeating requests, and pending triggers + */ + status_t clear(); + + /** * Queue a trigger to be dispatched with the next outgoing * process_capture_request. The settings for that request only * will be temporarily rewritten to add the trigger tag/value. @@ -257,13 +330,6 @@ class Camera3Device : void setPaused(bool paused); /** - * Wait until thread is paused, either due to setPaused(true) - * or due to lack of input requests. Returns TIMED_OUT in case - * the thread does not pause within the timeout. - */ - status_t waitUntilPaused(nsecs_t timeout); - - /** * Wait until thread processes the capture request with settings' * android.request.id == requestId. * @@ -272,6 +338,18 @@ class Camera3Device : */ status_t waitUntilRequestProcessed(int32_t requestId, nsecs_t timeout); + /** + * Shut down the thread. Shutdown is asynchronous, so thread may + * still be running once this method returns. + */ + virtual void requestExit(); + + /** + * Get the latest request that was sent to the HAL + * with process_capture_request. + */ + CameraMetadata getLatestRequest() const; + protected: virtual bool threadLoop(); @@ -286,6 +364,10 @@ class Camera3Device : // restoring the old field values for those tags. status_t removeTriggers(const sp<CaptureRequest> &request); + // HAL workaround: Make sure a trigger ID always exists if + // a trigger does + status_t addDummyTriggerIds(const sp<CaptureRequest> &request); + static const nsecs_t kRequestTimeout = 50e6; // 50 ms // Waits for a request, or returns NULL if times out. @@ -301,14 +383,18 @@ class Camera3Device : // Pause handling bool waitIfPaused(); + void unpauseForNewRequests(); // Relay error to parent device object setErrorState void setErrorState(const char *fmt, ...); wp<Camera3Device> mParent; + wp<camera3::StatusTracker> mStatusTracker; camera3_device_t *mHal3Device; - const int mId; + const int mId; // The camera ID + int mStatusId; // The RequestThread's component ID for + // status tracking Mutex mRequestLock; Condition mRequestSignal; @@ -329,10 +415,11 @@ class Camera3Device : uint32_t mFrameNumber; - Mutex mLatestRequestMutex; + mutable Mutex mLatestRequestMutex; Condition mLatestRequestSignal; // android.request.id for latest process_capture_request int32_t mLatestRequestId; + CameraMetadata mLatestRequest; typedef KeyedVector<uint32_t/*tag*/, RequestTrigger> TriggerMap; Mutex mTriggerMutex; @@ -347,22 +434,30 @@ class Camera3Device : */ struct InFlightRequest { + // android.request.id for the request + int requestId; // Set by notify() SHUTTER call. nsecs_t captureTimestamp; + int requestStatus; // Set by process_capture_result call with valid metadata bool haveResultMetadata; // Decremented by calls to process_capture_result with valid output // buffers int numBuffersLeft; + // Default constructor needed by KeyedVector InFlightRequest() : + requestId(0), captureTimestamp(0), + requestStatus(OK), haveResultMetadata(false), numBuffersLeft(0) { } - explicit InFlightRequest(int numBuffers) : + InFlightRequest(int id, int numBuffers) : + requestId(id), captureTimestamp(0), + requestStatus(OK), haveResultMetadata(false), numBuffersLeft(numBuffers) { } @@ -373,7 +468,13 @@ class Camera3Device : Mutex mInFlightLock; // Protects mInFlightMap InFlightMap mInFlightMap; - status_t registerInFlight(int32_t frameNumber, int32_t numBuffers); + status_t registerInFlight(int32_t frameNumber, int32_t requestId, + int32_t numBuffers); + + /** + * Tracking for idle detection + */ + sp<camera3::StatusTracker> mStatusTracker; /** * Output result queue and current HAL device 3A state diff --git a/services/camera/libcameraservice/camera3/Camera3IOStreamBase.cpp b/services/camera/libcameraservice/device3/Camera3IOStreamBase.cpp index 0850566..da51228 100644 --- a/services/camera/libcameraservice/camera3/Camera3IOStreamBase.cpp +++ b/services/camera/libcameraservice/device3/Camera3IOStreamBase.cpp @@ -23,7 +23,8 @@ #include <utils/Log.h> #include <utils/Trace.h> -#include "Camera3IOStreamBase.h" +#include "device3/Camera3IOStreamBase.h" +#include "device3/StatusTracker.h" namespace android { @@ -62,53 +63,6 @@ bool Camera3IOStreamBase::hasOutstandingBuffersLocked() const { return false; } -status_t Camera3IOStreamBase::waitUntilIdle(nsecs_t timeout) { - status_t res; - { - Mutex::Autolock l(mLock); - while (mDequeuedBufferCount > 0) { - if (timeout != TIMEOUT_NEVER) { - nsecs_t startTime = systemTime(); - res = mBufferReturnedSignal.waitRelative(mLock, timeout); - if (res == TIMED_OUT) { - return res; - } else if (res != OK) { - ALOGE("%s: Error waiting for outstanding buffers: %s (%d)", - __FUNCTION__, strerror(-res), res); - return res; - } - nsecs_t deltaTime = systemTime() - startTime; - if (timeout <= deltaTime) { - timeout = 0; - } else { - timeout -= deltaTime; - } - } else { - res = mBufferReturnedSignal.wait(mLock); - if (res != OK) { - ALOGE("%s: Error waiting for outstanding buffers: %s (%d)", - __FUNCTION__, strerror(-res), res); - return res; - } - } - } - } - - // No lock - - unsigned int timeoutMs; - if (timeout == TIMEOUT_NEVER) { - timeoutMs = Fence::TIMEOUT_NEVER; - } else if (timeout == 0) { - timeoutMs = 0; - } else { - // Round up to wait at least 1 ms - timeoutMs = (timeout + 999999) / 1000000; - } - - return mCombinedFence->wait(timeoutMs); -} - void Camera3IOStreamBase::dump(int fd, const Vector<String16> &args) const { (void) args; String8 lines; @@ -190,6 +144,14 @@ void Camera3IOStreamBase::handoutBufferLocked(camera3_stream_buffer &buffer, buffer.release_fence = releaseFence; buffer.status = status; + // Inform tracker about becoming busy + if (mDequeuedBufferCount == 0 && mState != STATE_IN_CONFIG && + mState != STATE_IN_RECONFIG) { + sp<StatusTracker> statusTracker = mStatusTracker.promote(); + if (statusTracker != 0) { + statusTracker->markComponentActive(mStatusId); + } + } mDequeuedBufferCount++; } @@ -252,20 +214,32 @@ status_t Camera3IOStreamBase::returnAnyBufferLocked( sp<Fence> releaseFence; res = returnBufferCheckedLocked(buffer, timestamp, output, &releaseFence); - if (res != OK) { - return res; - } + // Res may be an error, but we still want to decrement our owned count + // to enable clean shutdown. So we'll just return the error but otherwise + // carry on - mCombinedFence = Fence::merge(mName, mCombinedFence, releaseFence); + if (releaseFence != 0) { + mCombinedFence = Fence::merge(mName, mCombinedFence, releaseFence); + } mDequeuedBufferCount--; + if (mDequeuedBufferCount == 0 && mState != STATE_IN_CONFIG && + mState != STATE_IN_RECONFIG) { + ALOGV("%s: Stream %d: All buffers returned; now idle", __FUNCTION__, + mId); + sp<StatusTracker> statusTracker = mStatusTracker.promote(); + if (statusTracker != 0) { + statusTracker->markComponentIdle(mStatusId, mCombinedFence); + } + } + mBufferReturnedSignal.signal(); if (output) { mLastTimestamp = timestamp; } - return OK; + return res; } diff --git a/services/camera/libcameraservice/camera3/Camera3IOStreamBase.h b/services/camera/libcameraservice/device3/Camera3IOStreamBase.h index 74c4484..fcb9d04 100644 --- a/services/camera/libcameraservice/camera3/Camera3IOStreamBase.h +++ b/services/camera/libcameraservice/device3/Camera3IOStreamBase.h @@ -43,7 +43,6 @@ class Camera3IOStreamBase : * Camera3Stream interface */ - virtual status_t waitUntilIdle(nsecs_t timeout); virtual void dump(int fd, const Vector<String16> &args) const; protected: @@ -77,6 +76,8 @@ class Camera3IOStreamBase : virtual size_t getBufferCountLocked(); + virtual status_t getEndpointUsage(uint32_t *usage) = 0; + status_t getBufferPreconditionCheckLocked() const; status_t returnBufferPreconditionCheckLocked() const; diff --git a/services/camera/libcameraservice/camera3/Camera3InputStream.cpp b/services/camera/libcameraservice/device3/Camera3InputStream.cpp index 13e9c83..5aa9a3e 100644 --- a/services/camera/libcameraservice/camera3/Camera3InputStream.cpp +++ b/services/camera/libcameraservice/device3/Camera3InputStream.cpp @@ -115,7 +115,6 @@ status_t Camera3InputStream::returnBufferCheckedLocked( bufferFound = true; bufferItem = tmp; mBuffersInFlight.erase(it); - mDequeuedBufferCount--; } } } @@ -148,12 +147,11 @@ status_t Camera3InputStream::returnBufferCheckedLocked( if (res != OK) { ALOGE("%s: Stream %d: Error releasing buffer back to buffer queue:" " %s (%d)", __FUNCTION__, mId, strerror(-res), res); - return res; } *releaseFenceOut = releaseFence; - return OK; + return res; } status_t Camera3InputStream::returnInputBufferLocked( @@ -182,10 +180,6 @@ status_t Camera3InputStream::disconnectLocked() { return OK; } -sp<IGraphicBufferProducer> Camera3InputStream::getProducerInterface() const { - return mConsumer->getProducerInterface(); -} - void Camera3InputStream::dump(int fd, const Vector<String16> &args) const { (void) args; String8 lines; @@ -211,9 +205,9 @@ status_t Camera3InputStream::configureQueueLocked() { mFrameCount = 0; if (mConsumer.get() == 0) { - mConsumer = new BufferItemConsumer(camera3_stream::usage, - mTotalBufferCount, - /*synchronousMode*/true); + sp<BufferQueue> bq = new BufferQueue(); + mConsumer = new BufferItemConsumer(bq, camera3_stream::usage, + mTotalBufferCount); mConsumer->setName(String8::format("Camera3-InputStream-%d", mId)); } @@ -234,6 +228,12 @@ status_t Camera3InputStream::configureQueueLocked() { return OK; } +status_t Camera3InputStream::getEndpointUsage(uint32_t *usage) { + // Per HAL3 spec, input streams have 0 for their initial usage field. + *usage = 0; + return OK; +} + }; // namespace camera3 }; // namespace android diff --git a/services/camera/libcameraservice/camera3/Camera3InputStream.h b/services/camera/libcameraservice/device3/Camera3InputStream.h index 8adda88..681d684 100644 --- a/services/camera/libcameraservice/camera3/Camera3InputStream.h +++ b/services/camera/libcameraservice/device3/Camera3InputStream.h @@ -44,13 +44,6 @@ class Camera3InputStream : public Camera3IOStreamBase { virtual void dump(int fd, const Vector<String16> &args) const; - /** - * Get the producer interface for this stream, to hand off to a producer. - * The producer must be connected to the provided interface before - * finishConfigure is called on this stream. - */ - sp<IGraphicBufferProducer> getProducerInterface() const; - private: typedef BufferItemConsumer::BufferItem BufferItem; @@ -79,6 +72,8 @@ class Camera3InputStream : public Camera3IOStreamBase { virtual status_t configureQueueLocked(); + virtual status_t getEndpointUsage(uint32_t *usage); + }; // class Camera3InputStream }; // namespace camera3 diff --git a/services/camera/libcameraservice/camera3/Camera3OutputStream.cpp b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp index 2efeede..682755d 100644 --- a/services/camera/libcameraservice/camera3/Camera3OutputStream.cpp +++ b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp @@ -92,7 +92,22 @@ status_t Camera3OutputStream::getBufferLocked(camera3_stream_buffer *buffer) { ANativeWindowBuffer* anb; int fenceFd; - res = mConsumer->dequeueBuffer(mConsumer.get(), &anb, &fenceFd); + /** + * Release the lock briefly to avoid deadlock for below scenario: + * Thread 1: StreamingProcessor::startStream -> Camera3Stream::isConfiguring(). + * This thread acquired StreamingProcessor lock and try to lock Camera3Stream lock. + * Thread 2: Camera3Stream::returnBuffer->StreamingProcessor::onFrameAvailable(). + * This thread acquired Camera3Stream lock and bufferQueue lock, and try to lock + * StreamingProcessor lock. + * Thread 3: Camera3Stream::getBuffer(). This thread acquired Camera3Stream lock + * and try to lock bufferQueue lock. + * Then there is circular locking dependency. + */ + sp<ANativeWindow> currentConsumer = mConsumer; + mLock.unlock(); + + res = currentConsumer->dequeueBuffer(currentConsumer.get(), &anb, &fenceFd); + mLock.lock(); if (res != OK) { ALOGE("%s: Stream %d: Can't dequeue next output buffer: %s (%d)", __FUNCTION__, mId, strerror(-res), res); @@ -198,12 +213,11 @@ status_t Camera3OutputStream::returnBufferCheckedLocked( mLock.lock(); if (res != OK) { close(anwReleaseFence); - return res; } *releaseFenceOut = releaseFence; - return OK; + return res; } void Camera3OutputStream::dump(int fd, const Vector<String16> &args) const { @@ -301,8 +315,13 @@ status_t Camera3OutputStream::configureQueueLocked() { return res; } - ALOGV("%s: Consumer wants %d buffers", __FUNCTION__, - maxConsumerBuffers); + ALOGV("%s: Consumer wants %d buffers, HAL wants %d", __FUNCTION__, + maxConsumerBuffers, camera3_stream::max_buffers); + if (camera3_stream::max_buffers == 0) { + ALOGE("%s: Camera HAL requested max_buffer count: %d, requires at least 1", + __FUNCTION__, camera3_stream::max_buffers); + return INVALID_OPERATION; + } mTotalBufferCount = maxConsumerBuffers + camera3_stream::max_buffers; mDequeuedBufferCount = 0; @@ -359,6 +378,17 @@ status_t Camera3OutputStream::disconnectLocked() { return OK; } +status_t Camera3OutputStream::getEndpointUsage(uint32_t *usage) { + + status_t res; + int32_t u = 0; + res = mConsumer->query(mConsumer.get(), + NATIVE_WINDOW_CONSUMER_USAGE_BITS, &u); + *usage = u; + + return res; +} + }; // namespace camera3 }; // namespace android diff --git a/services/camera/libcameraservice/camera3/Camera3OutputStream.h b/services/camera/libcameraservice/device3/Camera3OutputStream.h index 774fbdd..6cbb9f4 100644 --- a/services/camera/libcameraservice/camera3/Camera3OutputStream.h +++ b/services/camera/libcameraservice/device3/Camera3OutputStream.h @@ -92,6 +92,9 @@ class Camera3OutputStream : virtual status_t configureQueueLocked(); virtual status_t disconnectLocked(); + + virtual status_t getEndpointUsage(uint32_t *usage); + }; // class Camera3OutputStream } // namespace camera3 diff --git a/services/camera/libcameraservice/camera3/Camera3OutputStreamInterface.h b/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.h index aae72cf..aae72cf 100644 --- a/services/camera/libcameraservice/camera3/Camera3OutputStreamInterface.h +++ b/services/camera/libcameraservice/device3/Camera3OutputStreamInterface.h diff --git a/services/camera/libcameraservice/camera3/Camera3Stream.cpp b/services/camera/libcameraservice/device3/Camera3Stream.cpp index f05658a..6d2cf94 100644 --- a/services/camera/libcameraservice/camera3/Camera3Stream.cpp +++ b/services/camera/libcameraservice/device3/Camera3Stream.cpp @@ -20,13 +20,18 @@ #include <utils/Log.h> #include <utils/Trace.h> -#include "Camera3Stream.h" +#include "device3/Camera3Stream.h" +#include "device3/StatusTracker.h" namespace android { namespace camera3 { Camera3Stream::~Camera3Stream() { + sp<StatusTracker> statusTracker = mStatusTracker.promote(); + if (statusTracker != 0 && mStatusId != StatusTracker::NO_STATUS_ID) { + statusTracker->removeComponent(mStatusId); + } } Camera3Stream* Camera3Stream::cast(camera3_stream *stream) { @@ -44,7 +49,8 @@ Camera3Stream::Camera3Stream(int id, mId(id), mName(String8::format("Camera3Stream[%d]", id)), mMaxSize(maxSize), - mState(STATE_CONSTRUCTED) { + mState(STATE_CONSTRUCTED), + mStatusId(StatusTracker::NO_STATUS_ID) { camera3_stream::stream_type = type; camera3_stream::width = width; @@ -77,7 +83,9 @@ int Camera3Stream::getFormat() const { } camera3_stream* Camera3Stream::startConfiguration() { + ATRACE_CALL(); Mutex::Autolock l(mLock); + status_t res; switch (mState) { case STATE_ERROR: @@ -107,8 +115,24 @@ camera3_stream* Camera3Stream::startConfiguration() { return NULL; } - oldUsage = usage; - oldMaxBuffers = max_buffers; + oldUsage = camera3_stream::usage; + oldMaxBuffers = camera3_stream::max_buffers; + + res = getEndpointUsage(&(camera3_stream::usage)); + if (res != OK) { + ALOGE("%s: Cannot query consumer endpoint usage!", + __FUNCTION__); + return NULL; + } + + // Stop tracking if currently doing so + if (mStatusId != StatusTracker::NO_STATUS_ID) { + sp<StatusTracker> statusTracker = mStatusTracker.promote(); + if (statusTracker != 0) { + statusTracker->removeComponent(mStatusId); + } + mStatusId = StatusTracker::NO_STATUS_ID; + } if (mState == STATE_CONSTRUCTED) { mState = STATE_IN_CONFIG; @@ -125,6 +149,7 @@ bool Camera3Stream::isConfiguring() const { } status_t Camera3Stream::finishConfiguration(camera3_device *hal3Device) { + ATRACE_CALL(); Mutex::Autolock l(mLock); switch (mState) { case STATE_ERROR: @@ -144,11 +169,17 @@ status_t Camera3Stream::finishConfiguration(camera3_device *hal3Device) { return INVALID_OPERATION; } + // Register for idle tracking + sp<StatusTracker> statusTracker = mStatusTracker.promote(); + if (statusTracker != 0) { + mStatusId = statusTracker->addComponent(); + } + // Check if the stream configuration is unchanged, and skip reallocation if // so. As documented in hardware/camera3.h:configure_streams(). if (mState == STATE_IN_RECONFIG && - oldUsage == usage && - oldMaxBuffers == max_buffers) { + oldUsage == camera3_stream::usage && + oldMaxBuffers == camera3_stream::max_buffers) { mState = STATE_CONFIGURED; return OK; } @@ -255,6 +286,18 @@ bool Camera3Stream::hasOutstandingBuffers() const { return hasOutstandingBuffersLocked(); } +status_t Camera3Stream::setStatusTracker(sp<StatusTracker> statusTracker) { + Mutex::Autolock l(mLock); + sp<StatusTracker> oldTracker = mStatusTracker.promote(); + if (oldTracker != 0 && mStatusId != StatusTracker::NO_STATUS_ID) { + oldTracker->removeComponent(mStatusId); + } + mStatusId = StatusTracker::NO_STATUS_ID; + mStatusTracker = statusTracker; + + return OK; +} + status_t Camera3Stream::disconnect() { ATRACE_CALL(); Mutex::Autolock l(mLock); @@ -312,8 +355,10 @@ status_t Camera3Stream::registerBuffersLocked(camera3_device *hal3Device) { // Got all buffers, register with HAL ALOGV("%s: Registering %d buffers with camera HAL", __FUNCTION__, bufferCount); + ATRACE_BEGIN("camera3->register_stream_buffers"); res = hal3Device->ops->register_stream_buffers(hal3Device, &bufferSet); + ATRACE_END(); } // Return all valid buffers to stream, in ERROR state to indicate diff --git a/services/camera/libcameraservice/camera3/Camera3Stream.h b/services/camera/libcameraservice/device3/Camera3Stream.h index 69d81e4..6eeb721 100644 --- a/services/camera/libcameraservice/camera3/Camera3Stream.h +++ b/services/camera/libcameraservice/device3/Camera3Stream.h @@ -190,12 +190,11 @@ class Camera3Stream : enum { TIMEOUT_NEVER = -1 }; + /** - * Wait until the HAL is done with all of this stream's buffers, including - * signalling all release fences. Returns TIMED_OUT if the timeout is exceeded, - * OK on success. Pass in TIMEOUT_NEVER for timeout to indicate an indefinite wait. + * Set the status tracker to notify about idle transitions */ - virtual status_t waitUntilIdle(nsecs_t timeout) = 0; + virtual status_t setStatusTracker(sp<StatusTracker> statusTracker); /** * Disconnect stream from its non-HAL endpoint. After this, @@ -263,6 +262,15 @@ class Camera3Stream : // Get the total number of buffers in the queue virtual size_t getBufferCountLocked() = 0; + // Get the usage flags for the other endpoint, or return + // INVALID_OPERATION if they cannot be obtained. + virtual status_t getEndpointUsage(uint32_t *usage) = 0; + + // Tracking for idle state + wp<StatusTracker> mStatusTracker; + // Status tracker component ID + int mStatusId; + private: uint32_t oldUsage; uint32_t oldMaxBuffers; diff --git a/services/camera/libcameraservice/camera3/Camera3StreamBufferListener.h b/services/camera/libcameraservice/device3/Camera3StreamBufferListener.h index 62ea6c0..62ea6c0 100644 --- a/services/camera/libcameraservice/camera3/Camera3StreamBufferListener.h +++ b/services/camera/libcameraservice/device3/Camera3StreamBufferListener.h diff --git a/services/camera/libcameraservice/camera3/Camera3StreamInterface.h b/services/camera/libcameraservice/device3/Camera3StreamInterface.h index 4768536..c93ae15 100644 --- a/services/camera/libcameraservice/camera3/Camera3StreamInterface.h +++ b/services/camera/libcameraservice/device3/Camera3StreamInterface.h @@ -26,6 +26,8 @@ namespace android { namespace camera3 { +class StatusTracker; + /** * An interface for managing a single stream of input and/or output data from * the camera device. @@ -128,13 +130,11 @@ class Camera3StreamInterface : public virtual RefBase { enum { TIMEOUT_NEVER = -1 }; + /** - * Wait until the HAL is done with all of this stream's buffers, including - * signalling all release fences. Returns TIMED_OUT if the timeout is - * exceeded, OK on success. Pass in TIMEOUT_NEVER for timeout to indicate - * an indefinite wait. + * Set the state tracker to use for signaling idle transitions. */ - virtual status_t waitUntilIdle(nsecs_t timeout) = 0; + virtual status_t setStatusTracker(sp<StatusTracker> statusTracker) = 0; /** * Disconnect stream from its non-HAL endpoint. After this, diff --git a/services/camera/libcameraservice/camera3/Camera3ZslStream.cpp b/services/camera/libcameraservice/device3/Camera3ZslStream.cpp index 8790c8c..04f5dc5 100644 --- a/services/camera/libcameraservice/camera3/Camera3ZslStream.cpp +++ b/services/camera/libcameraservice/device3/Camera3ZslStream.cpp @@ -113,11 +113,11 @@ Camera3ZslStream::Camera3ZslStream(int id, uint32_t width, uint32_t height, Camera3OutputStream(id, CAMERA3_STREAM_BIDIRECTIONAL, width, height, HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED), - mDepth(depth), - mProducer(new RingBufferConsumer(GRALLOC_USAGE_HW_CAMERA_ZSL, - depth)) { + mDepth(depth) { - mConsumer = new Surface(mProducer->getProducerInterface()); + sp<BufferQueue> bq = new BufferQueue(); + mProducer = new RingBufferConsumer(bq, GRALLOC_USAGE_HW_CAMERA_ZSL, depth); + mConsumer = new Surface(bq); } Camera3ZslStream::~Camera3ZslStream() { diff --git a/services/camera/libcameraservice/camera3/Camera3ZslStream.h b/services/camera/libcameraservice/device3/Camera3ZslStream.h index c7f4490..c7f4490 100644 --- a/services/camera/libcameraservice/camera3/Camera3ZslStream.h +++ b/services/camera/libcameraservice/device3/Camera3ZslStream.h diff --git a/services/camera/libcameraservice/device3/StatusTracker.cpp b/services/camera/libcameraservice/device3/StatusTracker.cpp new file mode 100644 index 0000000..ab5419f --- /dev/null +++ b/services/camera/libcameraservice/device3/StatusTracker.cpp @@ -0,0 +1,219 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "Camera3-Status" +#define ATRACE_TAG ATRACE_TAG_CAMERA +//#define LOG_NDEBUG 0 + +// This is needed for stdint.h to define INT64_MAX in C++ +#define __STDC_LIMIT_MACROS + +#include <utils/Log.h> +#include <utils/Trace.h> +#include <ui/Fence.h> + +#include "device3/StatusTracker.h" +#include "device3/Camera3Device.h" + +namespace android { + +namespace camera3 { + +StatusTracker::StatusTracker(wp<Camera3Device> parent) : + mComponentsChanged(false), + mParent(parent), + mNextComponentId(0), + mIdleFence(new Fence()), + mDeviceState(IDLE) { +} + +StatusTracker::~StatusTracker() { +} + +int StatusTracker::addComponent() { + int id; + ssize_t err; + { + Mutex::Autolock l(mLock); + id = mNextComponentId++; + ALOGV("%s: Adding new component %d", __FUNCTION__, id); + + err = mStates.add(id, IDLE); + ALOGE_IF(err < 0, "%s: Can't add new component %d: %s (%d)", + __FUNCTION__, id, strerror(-err), err); + } + + if (err >= 0) { + Mutex::Autolock pl(mPendingLock); + mComponentsChanged = true; + mPendingChangeSignal.signal(); + } + + return err < 0 ? err : id; +} + +void StatusTracker::removeComponent(int id) { + ssize_t idx; + { + Mutex::Autolock l(mLock); + ALOGV("%s: Removing component %d", __FUNCTION__, id); + idx = mStates.removeItem(id); + } + + if (idx >= 0) { + Mutex::Autolock pl(mPendingLock); + mComponentsChanged = true; + mPendingChangeSignal.signal(); + } + + return; +} + + +void StatusTracker::markComponentIdle(int id, const sp<Fence>& componentFence) { + markComponent(id, IDLE, componentFence); +} + +void StatusTracker::markComponentActive(int id) { + markComponent(id, ACTIVE, Fence::NO_FENCE); +} + +void StatusTracker::markComponent(int id, ComponentState state, + const sp<Fence>& componentFence) { + ALOGV("%s: Component %d is now %s", __FUNCTION__, id, + state == IDLE ? "idle" : "active"); + Mutex::Autolock l(mPendingLock); + + StateChange newState = { + id, + state, + componentFence + }; + + mPendingChangeQueue.add(newState); + mPendingChangeSignal.signal(); +} + +void StatusTracker::requestExit() { + // First mark thread dead + Thread::requestExit(); + // Then exit any waits + mPendingChangeSignal.signal(); +} + +StatusTracker::ComponentState StatusTracker::getDeviceStateLocked() { + for (size_t i = 0; i < mStates.size(); i++) { + if (mStates.valueAt(i) == ACTIVE) { + ALOGV("%s: Component %d not idle", __FUNCTION__, + mStates.keyAt(i)); + return ACTIVE; + } + } + // - If not yet signaled, getSignalTime returns INT64_MAX + // - If invalid fence or error, returns -1 + // - Otherwise returns time of signalling. + // Treat -1 as 'signalled', since HAL may not be using fences, and want + // to be able to idle in case of errors. + nsecs_t signalTime = mIdleFence->getSignalTime(); + bool fencesDone = signalTime != INT64_MAX; + + ALOGV_IF(!fencesDone, "%s: Fences still to wait on", __FUNCTION__); + + return fencesDone ? IDLE : ACTIVE; +} + +bool StatusTracker::threadLoop() { + status_t res; + + // Wait for state updates + { + Mutex::Autolock pl(mPendingLock); + while (mPendingChangeQueue.size() == 0 && !mComponentsChanged) { + res = mPendingChangeSignal.waitRelative(mPendingLock, + kWaitDuration); + if (exitPending()) return false; + if (res != OK) { + if (res != TIMED_OUT) { + ALOGE("%s: Error waiting on state changes: %s (%d)", + __FUNCTION__, strerror(-res), res); + } + // TIMED_OUT is expected + break; + } + } + } + + // After new pending states appear, or timeout, check if we're idle. Even + // with timeout, need to check to account for fences that may still be + // clearing out + sp<Camera3Device> parent; + { + Mutex::Autolock pl(mPendingLock); + Mutex::Autolock l(mLock); + + // Collect all pending state updates and see if the device + // collectively transitions between idle and active for each one + + // First pass for changed components or fence completions + ComponentState prevState = getDeviceStateLocked(); + if (prevState != mDeviceState) { + // Only collect changes to overall device state + mStateTransitions.add(prevState); + } + // For each pending component state update, check if we've transitioned + // to a new overall device state + for (size_t i = 0; i < mPendingChangeQueue.size(); i++) { + const StateChange &newState = mPendingChangeQueue[i]; + ssize_t idx = mStates.indexOfKey(newState.id); + // Ignore notices for unknown components + if (idx >= 0) { + // Update single component state + mStates.replaceValueAt(idx, newState.state); + mIdleFence = Fence::merge(String8("idleFence"), + mIdleFence, newState.fence); + // .. and see if overall device state has changed + ComponentState newState = getDeviceStateLocked(); + if (newState != prevState) { + mStateTransitions.add(newState); + } + prevState = newState; + } + } + mPendingChangeQueue.clear(); + mComponentsChanged = false; + + // Store final state after all pending state changes are done with + + mDeviceState = prevState; + parent = mParent.promote(); + } + + // Notify parent for all intermediate transitions + if (mStateTransitions.size() > 0 && parent.get()) { + for (size_t i = 0; i < mStateTransitions.size(); i++) { + bool idle = (mStateTransitions[i] == IDLE); + ALOGV("Camera device is now %s", idle ? "idle" : "active"); + parent->notifyStatus(idle); + } + } + mStateTransitions.clear(); + + return true; +} + +} // namespace android + +} // namespace camera3 diff --git a/services/camera/libcameraservice/device3/StatusTracker.h b/services/camera/libcameraservice/device3/StatusTracker.h new file mode 100644 index 0000000..49cecb3 --- /dev/null +++ b/services/camera/libcameraservice/device3/StatusTracker.h @@ -0,0 +1,130 @@ +/* + * Copyright (C) 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_SERVERS_CAMERA3_STATUSTRACKER_H +#define ANDROID_SERVERS_CAMERA3_STATUSTRACKER_H + +#include <utils/Condition.h> +#include <utils/Errors.h> +#include <utils/List.h> +#include <utils/Mutex.h> +#include <utils/Thread.h> +#include <utils/KeyedVector.h> +#include <hardware/camera3.h> + +#include "common/CameraDeviceBase.h" + +namespace android { + +class Camera3Device; +class Fence; + +namespace camera3 { + +/** + * State tracking for idle and other collective state transitions. + * Collects idle notifications from different sources and calls the + * parent when all of them become idle. + * + * The parent is responsible for synchronizing the status updates with its + * internal state correctly, which means the notifyStatus call to the parent may + * block for a while. + */ +class StatusTracker: public Thread { + public: + StatusTracker(wp<Camera3Device> parent); + ~StatusTracker(); + + // An always-invalid component ID + static const int NO_STATUS_ID = -1; + + // Add a component to track; returns non-negative unique ID for the new + // component on success, negative error code on failure. + // New components start in the idle state. + int addComponent(); + + // Remove existing component from idle tracking. Ignores unknown IDs + void removeComponent(int id); + + // Set the state of a tracked component to be idle. Ignores unknown IDs; can + // accept a fence to wait on to complete idle. The fence is merged with any + // previous fences given, which means they all must signal before the + // component is considered idle. + void markComponentIdle(int id, const sp<Fence>& componentFence); + + // Set the state of a tracked component to be active. Ignores unknown IDs. + void markComponentActive(int id); + + virtual void requestExit(); + protected: + + virtual bool threadLoop(); + + private: + enum ComponentState { + IDLE, + ACTIVE + }; + + void markComponent(int id, ComponentState state, + const sp<Fence>& componentFence); + + // Guards mPendingChange, mPendingStates, mComponentsChanged + Mutex mPendingLock; + + Condition mPendingChangeSignal; + + struct StateChange { + int id; + ComponentState state; + sp<Fence> fence; + }; + // A queue of yet-to-be-processed state changes to components + Vector<StateChange> mPendingChangeQueue; + bool mComponentsChanged; + + wp<Camera3Device> mParent; + + // Guards rest of internals. Must be locked after mPendingLock if both used. + Mutex mLock; + + int mNextComponentId; + + // Current component states + KeyedVector<int, ComponentState> mStates; + // Merged fence for all processed state changes + sp<Fence> mIdleFence; + // Current overall device state + ComponentState mDeviceState; + + // Private to threadLoop + + // Determine current overall device state + // We're IDLE iff + // - All components are currently IDLE + // - The merged fence for all component updates has signalled + ComponentState getDeviceStateLocked(); + + Vector<ComponentState> mStateTransitions; + + static const nsecs_t kWaitDuration = 250000000LL; // 250 ms +}; + +} // namespace camera3 + +} // namespace android + +#endif diff --git a/services/camera/libcameraservice/gui/RingBufferConsumer.cpp b/services/camera/libcameraservice/gui/RingBufferConsumer.cpp index cd39bad..ebc7ea7 100644 --- a/services/camera/libcameraservice/gui/RingBufferConsumer.cpp +++ b/services/camera/libcameraservice/gui/RingBufferConsumer.cpp @@ -34,14 +34,14 @@ typedef android::RingBufferConsumer::PinnedBufferItem PinnedBufferItem; namespace android { -RingBufferConsumer::RingBufferConsumer(uint32_t consumerUsage, +RingBufferConsumer::RingBufferConsumer(const sp<IGraphicBufferConsumer>& consumer, + uint32_t consumerUsage, int bufferCount) : - ConsumerBase(new BufferQueue(true)), + ConsumerBase(consumer), mBufferCount(bufferCount) { - mBufferQueue->setConsumerUsageBits(consumerUsage); - mBufferQueue->setSynchronousMode(true); - mBufferQueue->setMaxAcquiredBufferCount(bufferCount); + mConsumer->setConsumerUsageBits(consumerUsage); + mConsumer->setMaxAcquiredBufferCount(bufferCount); assert(bufferCount > 0); } @@ -52,7 +52,7 @@ RingBufferConsumer::~RingBufferConsumer() { void RingBufferConsumer::setName(const String8& name) { Mutex::Autolock _l(mMutex); mName = name; - mBufferQueue->setConsumerName(name); + mConsumer->setConsumerName(name); } sp<PinnedBufferItem> RingBufferConsumer::pinSelectedBuffer( @@ -214,7 +214,11 @@ status_t RingBufferConsumer::releaseOldestBufferLocked(size_t* pinnedFrames) { // In case the object was never pinned, pass the acquire fence // back to the release fence. If the fence was already waited on, // it'll just be a no-op to wait on it again. - err = addReleaseFenceLocked(item.mBuf, item.mFence); + + // item.mGraphicBuffer was populated with the proper graphic-buffer + // at acquire even if it was previously acquired + err = addReleaseFenceLocked(item.mBuf, + item.mGraphicBuffer, item.mFence); if (err != OK) { BI_LOGE("Failed to add release fence to buffer " @@ -226,7 +230,9 @@ status_t RingBufferConsumer::releaseOldestBufferLocked(size_t* pinnedFrames) { BI_LOGV("Attempting to release buffer timestamp %lld, frame %lld", item.mTimestamp, item.mFrameNumber); - err = releaseBufferLocked(item.mBuf, + // item.mGraphicBuffer was populated with the proper graphic-buffer + // at acquire even if it was previously acquired + err = releaseBufferLocked(item.mBuf, item.mGraphicBuffer, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR); if (err != OK) { @@ -278,7 +284,7 @@ void RingBufferConsumer::onFrameAvailable() { /** * Acquire new frame */ - err = acquireBufferLocked(&item); + err = acquireBufferLocked(&item, 0); if (err != OK) { if (err != NO_BUFFER_AVAILABLE) { BI_LOGE("Error acquiring buffer: %s (%d)", strerror(err), err); @@ -310,7 +316,8 @@ void RingBufferConsumer::unpinBuffer(const BufferItem& item) { RingBufferItem& find = *it; if (item.mGraphicBuffer == find.mGraphicBuffer) { - status_t res = addReleaseFenceLocked(item.mBuf, item.mFence); + status_t res = addReleaseFenceLocked(item.mBuf, + item.mGraphicBuffer, item.mFence); if (res != OK) { BI_LOGE("Failed to add release fence to buffer " @@ -336,17 +343,17 @@ void RingBufferConsumer::unpinBuffer(const BufferItem& item) { status_t RingBufferConsumer::setDefaultBufferSize(uint32_t w, uint32_t h) { Mutex::Autolock _l(mMutex); - return mBufferQueue->setDefaultBufferSize(w, h); + return mConsumer->setDefaultBufferSize(w, h); } status_t RingBufferConsumer::setDefaultBufferFormat(uint32_t defaultFormat) { Mutex::Autolock _l(mMutex); - return mBufferQueue->setDefaultBufferFormat(defaultFormat); + return mConsumer->setDefaultBufferFormat(defaultFormat); } status_t RingBufferConsumer::setConsumerUsage(uint32_t usage) { Mutex::Autolock _l(mMutex); - return mBufferQueue->setConsumerUsageBits(usage); + return mConsumer->setConsumerUsageBits(usage); } } // namespace android diff --git a/services/camera/libcameraservice/gui/RingBufferConsumer.h b/services/camera/libcameraservice/gui/RingBufferConsumer.h index 454fbae..b4ad824 100644 --- a/services/camera/libcameraservice/gui/RingBufferConsumer.h +++ b/services/camera/libcameraservice/gui/RingBufferConsumer.h @@ -63,7 +63,7 @@ class RingBufferConsumer : public ConsumerBase, // the consumer usage flags passed to the graphics allocator. The // bufferCount parameter specifies how many buffers can be pinned for user // access at the same time. - RingBufferConsumer(uint32_t consumerUsage, + RingBufferConsumer(const sp<IGraphicBufferConsumer>& consumer, uint32_t consumerUsage, int bufferCount = BufferQueue::MIN_UNDEQUEUED_BUFFERS); virtual ~RingBufferConsumer(); @@ -72,8 +72,6 @@ class RingBufferConsumer : public ConsumerBase, // log messages. void setName(const String8& name); - sp<IGraphicBufferProducer> getProducerInterface() const { return getBufferQueue(); } - // setDefaultBufferSize is used to set the size of buffers returned by // requestBuffers when a with and height of zero is requested. status_t setDefaultBufferSize(uint32_t w, uint32_t h); |