From 67e7f543c7f1c4fe4ee1989ceb0aebe44a63b49e Mon Sep 17 00:00:00 2001 From: Ronghua Wu Date: Fri, 13 Mar 2015 10:47:08 -0700 Subject: media: use ResourceManagerService with MediaCodec Bug: 19620911 Change-Id: I235a97e5195b28645b7834dda0dd77307d93f4a8 --- include/media/stagefright/MediaCodec.h | 38 +++ media/libstagefright/MediaCodec.cpp | 259 +++++++++++++++++++-- media/mediaserver/main_mediaserver.cpp | 2 + .../ResourceManagerService.cpp | 44 +++- .../test/ResourceManagerService_test.cpp | 68 +++++- 5 files changed, 380 insertions(+), 31 deletions(-) diff --git a/include/media/stagefright/MediaCodec.h b/include/media/stagefright/MediaCodec.h index 3e3c276..012a8f4 100644 --- a/include/media/stagefright/MediaCodec.h +++ b/include/media/stagefright/MediaCodec.h @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -34,6 +35,8 @@ struct IBatteryStats; struct ICrypto; struct IMemory; struct MemoryDealer; +class IResourceManagerClient; +class IResourceManagerService; struct SoftwareRenderer; struct Surface; @@ -230,6 +233,30 @@ private: bool mOwnedByClient; }; + struct ResourceManagerServiceProxy : public IBinder::DeathRecipient { + ResourceManagerServiceProxy(); + ~ResourceManagerServiceProxy(); + + void init(); + + // implements DeathRecipient + virtual void binderDied(const wp& /*who*/); + + void addResource( + int pid, + int64_t clientId, + const sp client, + const Vector &resources); + + void removeResource(int64_t clientId); + + bool reclaimResource(int callingPid, const Vector &resources); + + private: + Mutex mLock; + sp mService; + }; + State mState; sp mLooper; sp mCodecLooper; @@ -245,14 +272,22 @@ private: sp mCallback; sp mDealer; + sp mResourceManagerClient; + sp mResourceManagerService; + bool mBatteryStatNotified; bool mIsVideo; + int32_t mVideoWidth; + int32_t mVideoHeight; // initial create parameters AString mInitName; bool mInitNameIsType; bool mInitIsEncoder; + // configure parameter + sp mConfigureMsg; + // Used only to synchronize asynchronous getBufferAndFormat // across all the other (synchronous) buffer state change // operations, such as de/queueIn/OutputBuffer, start and @@ -320,6 +355,9 @@ private: void updateBatteryStat(); bool isExecuting() const; + uint64_t getGraphicBufferSize(); + void addResource(const char *type, uint64_t value); + /* called to get the last codec error when the sticky flag is set. * if no such codec error is found, returns UNKNOWN_ERROR. */ diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp index 5538cb0..40818f9 100644 --- a/media/libstagefright/MediaCodec.cpp +++ b/media/libstagefright/MediaCodec.cpp @@ -23,10 +23,12 @@ #include #include +#include #include #include #include #include +#include #include #include #include @@ -47,6 +49,45 @@ namespace android { +static inline int getCallingPid() { + return IPCThreadState::self()->getCallingPid(); +} + +static int64_t getId(sp client) { + return (int64_t) client.get(); +} + +static bool isResourceError(status_t err) { + return (err == OMX_ErrorInsufficientResources); +} + +static const int kMaxRetry = 2; + +struct ResourceManagerClient : public BnResourceManagerClient { + ResourceManagerClient(MediaCodec* codec) : mMediaCodec(codec) {} + + virtual bool reclaimResource() { + sp codec = mMediaCodec.promote(); + if (codec == NULL) { + // codec is already gone. + return true; + } + status_t err = codec->release(); + if (err != OK) { + ALOGW("ResourceManagerClient failed to release codec with err %d", err); + } + return (err == OK); + } + +protected: + virtual ~ResourceManagerClient() {} + +private: + wp mMediaCodec; + + DISALLOW_EVIL_CONSTRUCTORS(ResourceManagerClient); +}; + struct MediaCodec::BatteryNotifier : public Singleton { BatteryNotifier(); virtual ~BatteryNotifier(); @@ -178,6 +219,65 @@ void MediaCodec::BatteryNotifier::onBatteryStatServiceDied() { // started. } +MediaCodec::ResourceManagerServiceProxy::ResourceManagerServiceProxy() { +} + +MediaCodec::ResourceManagerServiceProxy::~ResourceManagerServiceProxy() { + if (mService != NULL) { + IInterface::asBinder(mService)->unlinkToDeath(this); + } +} + +void MediaCodec::ResourceManagerServiceProxy::init() { + sp sm = defaultServiceManager(); + sp binder = sm->getService(String16("media.resource_manager")); + mService = interface_cast(binder); + if (mService == NULL) { + ALOGE("Failed to get ResourceManagerService"); + return; + } + if (IInterface::asBinder(mService)->linkToDeath(this) != OK) { + mService.clear(); + ALOGE("Failed to linkToDeath to ResourceManagerService."); + return; + } +} + +void MediaCodec::ResourceManagerServiceProxy::binderDied(const wp& /*who*/) { + ALOGW("ResourceManagerService died."); + Mutex::Autolock _l(mLock); + mService.clear(); +} + +void MediaCodec::ResourceManagerServiceProxy::addResource( + int pid, + int64_t clientId, + const sp client, + const Vector &resources) { + Mutex::Autolock _l(mLock); + if (mService == NULL) { + return; + } + mService->addResource(pid, clientId, client, resources); +} + +void MediaCodec::ResourceManagerServiceProxy::removeResource(int64_t clientId) { + Mutex::Autolock _l(mLock); + if (mService == NULL) { + return; + } + mService->removeResource(clientId); +} + +bool MediaCodec::ResourceManagerServiceProxy::reclaimResource( + int callingPid, const Vector &resources) { + Mutex::Autolock _l(mLock); + if (mService == NULL) { + return false; + } + return mService->reclaimResource(callingPid, resources); +} + // static sp MediaCodec::CreateByType( const sp &looper, const char *mime, bool encoder, status_t *err) { @@ -208,10 +308,14 @@ MediaCodec::MediaCodec(const sp &looper) mCodec(NULL), mReplyID(0), mFlags(0), + mResourceManagerClient(new ResourceManagerClient(this)), + mResourceManagerService(new ResourceManagerServiceProxy()), mStickyError(OK), mSoftRenderer(NULL), mBatteryStatNotified(false), mIsVideo(false), + mVideoWidth(0), + mVideoHeight(0), mDequeueInputTimeoutGeneration(0), mDequeueInputReplyID(0), mDequeueOutputTimeoutGeneration(0), @@ -221,6 +325,7 @@ MediaCodec::MediaCodec(const sp &looper) MediaCodec::~MediaCodec() { CHECK_EQ(mState, UNINITIALIZED); + mResourceManagerService->removeResource(getId(mResourceManagerClient)); } // static @@ -247,6 +352,8 @@ void MediaCodec::PostReplyWithError(const sp &replyID, int32_t err) } status_t MediaCodec::init(const AString &name, bool nameIsType, bool encoder) { + mResourceManagerService->init(); + // save init parameters for reset mInitName = name; mInitNameIsType = nameIsType; @@ -266,12 +373,13 @@ status_t MediaCodec::init(const AString &name, bool nameIsType, bool encoder) { return NAME_NOT_FOUND; } - bool needDedicatedLooper = false; + bool secureCodec = false; if (nameIsType && !strncasecmp(name.c_str(), "video/", 6)) { - needDedicatedLooper = true; + mIsVideo = true; } else { AString tmp = name; if (tmp.endsWith(".secure")) { + secureCodec = true; tmp.erase(tmp.size() - 7, 7); } const sp mcl = MediaCodecList::getInstance(); @@ -282,14 +390,15 @@ status_t MediaCodec::init(const AString &name, bool nameIsType, bool encoder) { info->getSupportedMimes(&mimes); for (size_t i = 0; i < mimes.size(); i++) { if (mimes[i].startsWith("video/")) { - needDedicatedLooper = true; + mIsVideo = true; break; } } } } - if (needDedicatedLooper) { + if (mIsVideo) { + // video codec needs dedicated looper if (mCodecLooper == NULL) { mCodecLooper = new ALooper; mCodecLooper->setName("CodecLooper"); @@ -313,8 +422,25 @@ status_t MediaCodec::init(const AString &name, bool nameIsType, bool encoder) { msg->setInt32("encoder", encoder); } - sp response; - return PostAndAwaitResponse(msg, &response); + status_t err; + Vector resources; + const char *type = secureCodec ? kResourceSecureCodec : kResourceNonSecureCodec; + resources.push_back(MediaResource(String8(type), 1)); + for (int i = 0; i <= kMaxRetry; ++i) { + if (i > 0) { + // Don't try to reclaim resource for the first time. + if (!mResourceManagerService->reclaimResource(getCallingPid(), resources)) { + break; + } + } + + sp response; + err = PostAndAwaitResponse(msg, &response); + if (!isResourceError(err)) { + break; + } + } + return err; } status_t MediaCodec::setCallback(const sp &callback) { @@ -332,6 +458,11 @@ status_t MediaCodec::configure( uint32_t flags) { sp msg = new AMessage(kWhatConfigure, this); + if (mIsVideo) { + format->findInt32("width", &mVideoWidth); + format->findInt32("height", &mVideoHeight); + } + msg->setMessage("format", format); msg->setInt32("flags", flags); @@ -345,20 +476,41 @@ status_t MediaCodec::configure( msg->setPointer("crypto", crypto.get()); } - sp response; - status_t err = PostAndAwaitResponse(msg, &response); + // save msg for reset + mConfigureMsg = msg; - if (err != OK && err != INVALID_OPERATION) { - // MediaCodec now set state to UNINITIALIZED upon any fatal error. - // To maintain backward-compatibility, do a reset() to put codec - // back into INITIALIZED state. - // But don't reset if the err is INVALID_OPERATION, which means - // the configure failure is due to wrong state. + status_t err; + Vector resources; + const char *type = (mFlags & kFlagIsSecure) ? + kResourceSecureCodec : kResourceNonSecureCodec; + resources.push_back(MediaResource(String8(type), 1)); + // Don't know the buffer size at this point, but it's fine to use 1 because + // the reclaimResource call doesn't consider the requester's buffer size for now. + resources.push_back(MediaResource(String8(kResourceGraphicMemory), 1)); + for (int i = 0; i <= kMaxRetry; ++i) { + if (i > 0) { + // Don't try to reclaim resource for the first time. + if (!mResourceManagerService->reclaimResource(getCallingPid(), resources)) { + break; + } + } - ALOGE("configure failed with err 0x%08x, resetting...", err); - reset(); + sp response; + err = PostAndAwaitResponse(msg, &response); + if (err != OK && err != INVALID_OPERATION) { + // MediaCodec now set state to UNINITIALIZED upon any fatal error. + // To maintain backward-compatibility, do a reset() to put codec + // back into INITIALIZED state. + // But don't reset if the err is INVALID_OPERATION, which means + // the configure failure is due to wrong state. + + ALOGE("configure failed with err 0x%08x, resetting...", err); + reset(); + } + if (!isResourceError(err)) { + break; + } } - return err; } @@ -382,11 +534,65 @@ status_t MediaCodec::createInputSurface( return err; } +uint64_t MediaCodec::getGraphicBufferSize() { + if (!mIsVideo) { + return 0; + } + + uint64_t size = 0; + size_t portNum = sizeof(mPortBuffers) / sizeof((mPortBuffers)[0]); + for (size_t i = 0; i < portNum; ++i) { + // TODO: this is just an estimation, we should get the real buffer size from ACodec. + size += mPortBuffers[i].size() * mVideoWidth * mVideoHeight * 3 / 2; + } + return size; +} + +void MediaCodec::addResource(const char *type, uint64_t value) { + Vector resources; + resources.push_back(MediaResource(String8(type), value)); + mResourceManagerService->addResource( + getCallingPid(), getId(mResourceManagerClient), mResourceManagerClient, resources); +} + status_t MediaCodec::start() { sp msg = new AMessage(kWhatStart, this); - sp response; - return PostAndAwaitResponse(msg, &response); + status_t err; + Vector resources; + const char *type = (mFlags & kFlagIsSecure) ? + kResourceSecureCodec : kResourceNonSecureCodec; + resources.push_back(MediaResource(String8(type), 1)); + // Don't know the buffer size at this point, but it's fine to use 1 because + // the reclaimResource call doesn't consider the requester's buffer size for now. + resources.push_back(MediaResource(String8(kResourceGraphicMemory), 1)); + for (int i = 0; i <= kMaxRetry; ++i) { + if (i > 0) { + // Don't try to reclaim resource for the first time. + if (!mResourceManagerService->reclaimResource(getCallingPid(), resources)) { + break; + } + // Recover codec from previous error before retry start. + err = reset(); + if (err != OK) { + ALOGE("retrying start: failed to reset codec"); + break; + } + sp response; + err = PostAndAwaitResponse(mConfigureMsg, &response); + if (err != OK) { + ALOGE("retrying start: failed to configure codec"); + break; + } + } + + sp response; + err = PostAndAwaitResponse(msg, &response); + if (!isResourceError(err)) { + break; + } + } + return err; } status_t MediaCodec::stop() { @@ -960,11 +1166,15 @@ void MediaCodec::onMessageReceived(const sp &msg) { mFlags &= ~kFlagUsesSoftwareRenderer; } + String8 resourceType; if (mComponentName.endsWith(".secure")) { mFlags |= kFlagIsSecure; + resourceType = String8(kResourceSecureCodec); } else { mFlags &= ~kFlagIsSecure; + resourceType = String8(kResourceNonSecureCodec); } + addResource(resourceType, 1); (new AMessage)->postReply(mReplyID); break; @@ -1077,6 +1287,9 @@ void MediaCodec::onMessageReceived(const sp &msg) { // We're always allocating output buffers after // allocating input buffers, so this is a good // indication that now all buffers are allocated. + if (mIsVideo) { + addResource(kResourceGraphicMemory, getGraphicBufferSize()); + } setState(STARTED); (new AMessage)->postReply(mReplyID); } else { @@ -1256,6 +1469,8 @@ void MediaCodec::onMessageReceived(const sp &msg) { } mFlags &= ~kFlagIsComponentAllocated; + mResourceManagerService->removeResource(getId(mResourceManagerClient)); + (new AMessage)->postReply(mReplyID); break; } @@ -2357,12 +2572,6 @@ status_t MediaCodec::amendOutputFormatWithCodecSpecificData( void MediaCodec::updateBatteryStat() { if (mState == CONFIGURED && !mBatteryStatNotified) { - AString mime; - CHECK(mOutputFormat != NULL && - mOutputFormat->findString("mime", &mime)); - - mIsVideo = mime.startsWithIgnoreCase("video/"); - BatteryNotifier& notifier(BatteryNotifier::getInstance()); if (mIsVideo) { diff --git a/media/mediaserver/main_mediaserver.cpp b/media/mediaserver/main_mediaserver.cpp index 99572f8..06b3c6e 100644 --- a/media/mediaserver/main_mediaserver.cpp +++ b/media/mediaserver/main_mediaserver.cpp @@ -33,6 +33,7 @@ #include "CameraService.h" #include "MediaLogService.h" #include "MediaPlayerService.h" +#include "ResourceManagerService.h" #include "service/AudioPolicyService.h" #include "SoundTriggerHwService.h" #include "RadioService.h" @@ -128,6 +129,7 @@ int main(int argc __unused, char** argv) ALOGI("ServiceManager: %p", sm.get()); AudioFlinger::instantiate(); MediaPlayerService::instantiate(); + ResourceManagerService::instantiate(); CameraService::instantiate(); AudioPolicyService::instantiate(); SoundTriggerHwService::instantiate(); diff --git a/services/mediaresourcemanager/ResourceManagerService.cpp b/services/mediaresourcemanager/ResourceManagerService.cpp index 7296d47..75a69ed 100644 --- a/services/mediaresourcemanager/ResourceManagerService.cpp +++ b/services/mediaresourcemanager/ResourceManagerService.cpp @@ -126,6 +126,7 @@ void ResourceManagerService::addResource( Mutex::Autolock lock(mLock); ResourceInfos& infos = getResourceInfosForEdit(pid, mMap); ResourceInfo& info = getResourceInfoForEdit(clientId, client, infos); + // TODO: do the merge instead of append. info.resources.appendVector(resources); } @@ -197,19 +198,58 @@ bool ResourceManagerService::reclaimResource( } } } + + if (clients.size() == 0) { + // if we are here, run the third pass to free one codec with the same type. + for (size_t i = 0; i < resources.size(); ++i) { + String8 type = resources[i].mType; + if (type == kResourceSecureCodec || type == kResourceNonSecureCodec) { + sp client; + if (!getLowestPriorityBiggestClient_l(callingPid, type, &client)) { + return false; + } + clients.push_back(client); + } + } + } } if (clients.size() == 0) { return false; } + sp failedClient; for (size_t i = 0; i < clients.size(); ++i) { ALOGV("reclaimResource from client %p", clients[i].get()); if (!clients[i]->reclaimResource()) { - return false; + failedClient = clients[i]; + break; } } - return true; + + { + Mutex::Autolock lock(mLock); + bool found = false; + for (size_t i = 0; i < mMap.size(); ++i) { + ResourceInfos &infos = mMap.editValueAt(i); + for (size_t j = 0; j < infos.size();) { + if (infos[j].client == failedClient) { + j = infos.removeAt(j); + found = true; + } else { + ++j; + } + } + if (found) { + break; + } + } + if (!found) { + ALOGV("didn't find failed client"); + } + } + + return (failedClient == NULL); } bool ResourceManagerService::getAllClients_l( diff --git a/services/mediaresourcemanager/test/ResourceManagerService_test.cpp b/services/mediaresourcemanager/test/ResourceManagerService_test.cpp index b73e1bc..48d1395 100644 --- a/services/mediaresourcemanager/test/ResourceManagerService_test.cpp +++ b/services/mediaresourcemanager/test/ResourceManagerService_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015 The Android Open Source Project + * Copyright 2015 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -118,6 +118,20 @@ protected: client3->reset(); } + // test set up + // --------------------------------------------------------------------------------- + // pid priority client type number + // --------------------------------------------------------------------------------- + // kTestPid1(30) 30 mTestClient1 secure codec 1 + // graphic memory 200 + // graphic memory 200 + // --------------------------------------------------------------------------------- + // kTestPid2(20) 20 mTestClient2 non-secure codec 1 + // graphic memory 300 + // ------------------------------------------- + // mTestClient3 secure codec 1 + // graphic memory 100 + // --------------------------------------------------------------------------------- void addResource() { // kTestPid1 mTestClient1 Vector resources1; @@ -202,10 +216,12 @@ protected: int lowPriorityPid = 100; EXPECT_FALSE(mService->getAllClients_l(lowPriorityPid, type, &clients)); int midPriorityPid = 25; - EXPECT_FALSE(mService->getAllClients_l(lowPriorityPid, type, &clients)); + // some higher priority process (e.g. kTestPid2) owns the resource, so getAllClients_l + // will fail. + EXPECT_FALSE(mService->getAllClients_l(midPriorityPid, type, &clients)); int highPriorityPid = 10; - EXPECT_TRUE(mService->getAllClients_l(10, unknowType, &clients)); - EXPECT_TRUE(mService->getAllClients_l(10, type, &clients)); + EXPECT_TRUE(mService->getAllClients_l(highPriorityPid, unknowType, &clients)); + EXPECT_TRUE(mService->getAllClients_l(highPriorityPid, type, &clients)); EXPECT_EQ(2u, clients.size()); EXPECT_EQ(mTestClient3, clients[0]); @@ -308,6 +324,30 @@ protected: // nothing left EXPECT_FALSE(mService->reclaimResource(10, resources)); } + + // ### secure codecs can coexist and secure codec can coexist with non-secure codec ### + { + addResource(); + mService->mSupportsMultipleSecureCodecs = true; + mService->mSupportsSecureWithNonSecureCodec = true; + + Vector resources; + resources.push_back(MediaResource(String8(kResourceSecureCodec), 1)); + + EXPECT_TRUE(mService->reclaimResource(10, resources)); + // secure codec from lowest process got reclaimed + verifyClients(true, false, false); + + // call again should reclaim another secure codec from lowest process + EXPECT_TRUE(mService->reclaimResource(10, resources)); + verifyClients(false, false, true); + + // nothing left + EXPECT_FALSE(mService->reclaimResource(10, resources)); + + // clean up client 2 which still has non secure codec left + mService->removeResource((int64_t) mTestClient2.get()); + } } void testReclaimResourceNonSecure() { @@ -360,6 +400,26 @@ protected: // nothing left EXPECT_FALSE(mService->reclaimResource(10, resources)); } + + // ### secure codec can coexist with non-secure codec ### + { + addResource(); + mService->mSupportsSecureWithNonSecureCodec = true; + + Vector resources; + resources.push_back(MediaResource(String8(kResourceNonSecureCodec), 1)); + + EXPECT_TRUE(mService->reclaimResource(10, resources)); + // one non secure codec from lowest process got reclaimed + verifyClients(false, true, false); + + // nothing left + EXPECT_FALSE(mService->reclaimResource(10, resources)); + + // clean up client 1 and 3 which still have secure codec left + mService->removeResource((int64_t) mTestClient1.get()); + mService->removeResource((int64_t) mTestClient3.get()); + } } void testGetLowestPriorityBiggestClient() { -- cgit v1.1