Avoid deadlock in OMX::freeNode by making sure OMXCodecObserver does not hold the last reference of OMXCodec object - do not merge

cherry-picked from: I4c79b66a900c527e3ae6a833f76d5da1b75c5a89

bug - 3336424

Change-Id: I2d8ecb79a5422342988c195c012c9e6327ac457a

3 files changed, 18 insertions, 7 deletions

diff --git a/include/media/stagefright/OMXCodec.h b/include/media/stagefright/OMXCodec.h
index fed6761..8274dfb 100644
--- a/include/media/stagefright/OMXCodec.h
+++ b/include/media/stagefright/OMXCodec.h
@@ -60,8 +60,6 @@ struct OMXCodec : public MediaSource,
 
     virtual status_t pause();
 
-    void on_message(const omx_message &msg);
-
     // from MediaBufferObserver
     virtual void signalBufferReturned(MediaBuffer *buffer);
 
@@ -69,6 +67,13 @@ protected:
     virtual ~OMXCodec();
 
 private:
+
+    // Make sure mLock is accessible to OMXCodecObserver
+    friend class OMXCodecObserver;
+
+    // Call this with mLock hold
+    void on_message(const omx_message &msg);
+
     enum State {
         DEAD,
         LOADED,
diff --git a/media/libstagefright/OMXCodec.cpp b/media/libstagefright/OMXCodec.cpp
index 980da77..b5d00bf 100644
--- a/media/libstagefright/OMXCodec.cpp
+++ b/media/libstagefright/OMXCodec.cpp
@@ -234,7 +234,9 @@ struct OMXCodecObserver : public BnOMXObserver {
         sp<OMXCodec> codec = mTarget.promote();
 
         if (codec.get() != NULL) {
+            Mutex::Autolock autoLock(codec->mLock);
             codec->on_message(msg);
+            codec.clear();
         }
     }
 
@@ -1672,8 +1674,6 @@ status_t OMXCodec::allocateBuffersOnPort(OMX_U32 portIndex) {
 }
 
 void OMXCodec::on_message(const omx_message &msg) {
-    Mutex::Autolock autoLock(mLock);
-
     switch (msg.type) {
         case omx_message::EVENT:
         {
diff --git a/media/libstagefright/omx/OMX.cpp b/media/libstagefright/omx/OMX.cpp
index f19c16a..2ba63f7 100644
--- a/media/libstagefright/omx/OMX.cpp
+++ b/media/libstagefright/omx/OMX.cpp
@@ -89,6 +89,9 @@ OMX::CallbackDispatcher::~CallbackDispatcher() {
         mQueueChanged.signal();
     }
 
+    // Don't call join on myself
+    CHECK(mThread != pthread_self());
+
     void *dummy;
     pthread_join(mThread, &dummy);
 }
@@ -249,9 +252,12 @@ status_t OMX::freeNode(node_id node) {
 
     status_t err = instance->freeNode(mMaster);
 
-    index = mDispatchers.indexOfKey(node);
-    CHECK(index >= 0);
-    mDispatchers.removeItemsAt(index);
+    {
+        Mutex::Autolock autoLock(mLock);
+        index = mDispatchers.indexOfKey(node);
+        CHECK(index >= 0);
+        mDispatchers.removeItemsAt(index);
+    }
 
     return err;
 }