Add timestamps to video frames to improve A/V sync.

Bug 1927069.

8 files changed, 55 insertions, 11 deletions

diff --git a/camera/libcameraservice/CameraService.cpp b/camera/libcameraservice/CameraService.cpp
index 022fe5a..e4b6791 100644
--- a/camera/libcameraservice/CameraService.cpp
+++ b/camera/libcameraservice/CameraService.cpp
@@ -504,8 +504,7 @@ status_t CameraService::Client::startRecordingMode()
     }
 
     // start recording mode
-    ret = mHardware->startRecording(recordingCallback,
-                                    mCameraService.get());
+    ret = mHardware->startRecording(recordingCallback, mCameraService.get());
     if (ret != NO_ERROR) {
         LOGE("mHardware->startRecording() failed with status %d", ret);
     }
@@ -798,7 +797,7 @@ void CameraService::Client::previewCallback(const sp<IMemory>& mem, void* user)
 }
 
 // recording callback
-void CameraService::Client::recordingCallback(const sp<IMemory>& mem, void* user)
+void CameraService::Client::recordingCallback(nsecs_t timestamp, const sp<IMemory>& mem, void* user)
 {
     LOGV("recordingCallback");
     sp<Client> client = getClientFromCookie(user);
@@ -806,7 +805,7 @@ void CameraService::Client::recordingCallback(const sp<IMemory>& mem, void* user
         return;
     }
     // The strong pointer guarantees the client will exist, but no lock is held.
-    client->postRecordingFrame(mem);
+    client->postRecordingFrame(timestamp, mem);
 }
 
 // take a picture - image is returned in callback
@@ -1072,14 +1071,14 @@ void CameraService::Client::copyFrameAndPostCopiedFrame(sp<IMemoryHeap> heap, si
     mCameraClient->dataCallback(CAMERA_MSG_PREVIEW_FRAME, frame);
 }
 
-void CameraService::Client::postRecordingFrame(const sp<IMemory>& frame)
+void CameraService::Client::postRecordingFrame(nsecs_t timestamp, const sp<IMemory>& frame)
 {
     LOGV("postRecordingFrame");
     if (frame == 0) {
         LOGW("frame is a null pointer");
         return;
     }
-    mCameraClient->dataCallback(CAMERA_MSG_VIDEO_FRAME, frame);
+    mCameraClient->dataCallbackTimestamp(timestamp, CAMERA_MSG_VIDEO_FRAME, frame);
 }
 
 void CameraService::Client::postPreviewFrame(const sp<IMemory>& mem)
diff --git a/camera/libcameraservice/CameraService.h b/camera/libcameraservice/CameraService.h
index 0f07673..ea93789 100644
--- a/camera/libcameraservice/CameraService.h
+++ b/camera/libcameraservice/CameraService.h
@@ -132,7 +132,7 @@ private:
 
                     status_t    checkPid();
 
-        static      void        recordingCallback(const sp<IMemory>& mem, void* user);
+        static      void        recordingCallback(nsecs_t timestamp, const sp<IMemory>& mem, void* user);
         static      void        previewCallback(const sp<IMemory>& mem, void* user);
         static      void        shutterCallback(void *user);
         static      void        yuvPictureCallback(const sp<IMemory>& mem, void* user);
@@ -144,7 +144,7 @@ private:
                     void        postRaw(const sp<IMemory>& mem);
                     void        postJpeg(const sp<IMemory>& mem);
                     void        postPreviewFrame(const sp<IMemory>& mem);
-                    void        postRecordingFrame(const sp<IMemory>& frame);
+                    void        postRecordingFrame(nsecs_t timestamp, const sp<IMemory>& frame);
                     void        copyFrameAndPostCopiedFrame(sp<IMemoryHeap> heap, size_t offset, size_t size);
                     void        postError(status_t error);
                     void        postAutoFocus(bool focused);
diff --git a/core/jni/android_hardware_Camera.cpp b/core/jni/android_hardware_Camera.cpp
index 77a8a72..57b5aa6 100644
--- a/core/jni/android_hardware_Camera.cpp
+++ b/core/jni/android_hardware_Camera.cpp
@@ -55,6 +55,7 @@ public:
     ~JNICameraContext() { release(); }
     virtual void notify(int32_t msgType, int32_t ext1, int32_t ext2);
     virtual void postData(int32_t msgType, const sp<IMemory>& dataPtr);
+    virtual void postDataTimestamp(nsecs_t timestamp, int32_t msgType, const sp<IMemory>& dataPtr);
     sp<Camera> getCamera() { Mutex::Autolock _l(mLock); return mCamera; }
     void release();
 
@@ -188,6 +189,12 @@ void JNICameraContext::postData(int32_t msgType, const sp<IMemory>& dataPtr)
     }
 }
 
+void JNICameraContext::postDataTimestamp(nsecs_t timestamp, int32_t msgType, const sp<IMemory>& dataPtr)
+{
+    // TODO: plumb up to Java. For now, just drop the timestamp
+    postData(msgType, dataPtr);
+}
+
 // connect to camera service
 static void android_hardware_Camera_native_setup(JNIEnv *env, jobject thiz, jobject weak_this)
 {
diff --git a/include/ui/Camera.h b/include/ui/Camera.h
index e3544ab..afb07b5 100644
--- a/include/ui/Camera.h
+++ b/include/ui/Camera.h
@@ -18,6 +18,7 @@
 #ifndef ANDROID_HARDWARE_CAMERA_H
 #define ANDROID_HARDWARE_CAMERA_H
 
+#include <utils/Timers.h>
 #include <ui/ICameraClient.h>
 
 namespace android {
@@ -94,6 +95,7 @@ class CameraListener: virtual public RefBase
 public:
     virtual void notify(int32_t msgType, int32_t ext1, int32_t ext2) = 0;
     virtual void postData(int32_t msgType, const sp<IMemory>& dataPtr) = 0;
+    virtual void postDataTimestamp(nsecs_t timestamp, int32_t msgType, const sp<IMemory>& dataPtr) = 0;
 };
 
 class Camera : public BnCameraClient, public IBinder::DeathRecipient
@@ -155,6 +157,7 @@ public:
     // ICameraClient interface
     virtual void        notifyCallback(int32_t msgType, int32_t ext, int32_t ext2);
     virtual void        dataCallback(int32_t msgType, const sp<IMemory>& dataPtr);
+    virtual void        dataCallbackTimestamp(nsecs_t timestamp, int32_t msgType, const sp<IMemory>& dataPtr);
 
     sp<ICamera>         remote();
 
diff --git a/include/ui/CameraHardwareInterface.h b/include/ui/CameraHardwareInterface.h
index 73036f0..822b4a8 100644
--- a/include/ui/CameraHardwareInterface.h
+++ b/include/ui/CameraHardwareInterface.h
@@ -28,7 +28,7 @@ namespace android {
 typedef void (*preview_callback)(const sp<IMemory>& mem, void* user);
 
 /** Callback for startRecord() */
-typedef void (*recording_callback)(const sp<IMemory>& mem, void* user);
+typedef void (*recording_callback)(nsecs_t timestamp, const sp<IMemory>& mem, void* user);
 
 /** Callback for takePicture() */
 typedef void (*shutter_callback)(void* user);
diff --git a/include/ui/ICameraClient.h b/include/ui/ICameraClient.h
index c4bdd07..1001c71 100644
--- a/include/ui/ICameraClient.h
+++ b/include/ui/ICameraClient.h
@@ -21,6 +21,7 @@
 #include <utils/IInterface.h>
 #include <utils/Parcel.h>
 #include <utils/IMemory.h>
+#include <utils/Timers.h>
 
 namespace android {
 
@@ -31,7 +32,7 @@ public:
 
     virtual void            notifyCallback(int32_t msgType, int32_t ext1, int32_t ext2) = 0;
     virtual void            dataCallback(int32_t msgType, const sp<IMemory>& data) = 0;
-
+    virtual void            dataCallbackTimestamp(nsecs_t timestamp, int32_t msgType, const sp<IMemory>& data) = 0;
 };
 
 // ----------------------------------------------------------------------------
diff --git a/libs/ui/Camera.cpp b/libs/ui/Camera.cpp
index 975594f..5015379 100644
--- a/libs/ui/Camera.cpp
+++ b/libs/ui/Camera.cpp
@@ -310,6 +310,19 @@ void Camera::dataCallback(int32_t msgType, const sp<IMemory>& dataPtr)
     }
 }
 
+// callback from camera service when timestamped frame is ready
+void Camera::dataCallbackTimestamp(nsecs_t timestamp, int32_t msgType, const sp<IMemory>& dataPtr)
+{
+    sp<CameraListener> listener;
+    {
+        Mutex::Autolock _l(mLock);
+        listener = mListener;
+    }
+    if (listener != NULL) {
+        listener->postDataTimestamp(timestamp, msgType, dataPtr);
+    }
+}
+
 void Camera::binderDied(const wp<IBinder>& who) {
     LOGW("ICamera died");
     notifyCallback(CAMERA_MSG_ERROR, CAMERA_ERROR_SERVER_DIED, 0);
diff --git a/libs/ui/ICameraClient.cpp b/libs/ui/ICameraClient.cpp
index c6cf75c..59a6cf2 100644
--- a/libs/ui/ICameraClient.cpp
+++ b/libs/ui/ICameraClient.cpp
@@ -27,6 +27,7 @@ namespace android {
 enum {
     NOTIFY_CALLBACK = IBinder::FIRST_CALL_TRANSACTION,
     DATA_CALLBACK,
+    DATA_CALLBACK_TIMESTAMP,
 };
 
 class BpCameraClient: public BpInterface<ICameraClient>
@@ -60,6 +61,17 @@ public:
         remote()->transact(DATA_CALLBACK, data, &reply, IBinder::FLAG_ONEWAY);
     }
 
+    // generic data callback from camera service to app with image data
+    void dataCallbackTimestamp(nsecs_t timestamp, int32_t msgType, const sp<IMemory>& imageData)
+    {
+        LOGV("dataCallback");
+        Parcel data, reply;
+        data.writeInterfaceToken(ICameraClient::getInterfaceDescriptor());
+        data.writeInt64(timestamp);
+        data.writeInt32(msgType);
+        data.writeStrongBinder(imageData->asBinder());
+        remote()->transact(DATA_CALLBACK_TIMESTAMP, data, &reply, IBinder::FLAG_ONEWAY);
+    }
 };
 
 IMPLEMENT_META_INTERFACE(CameraClient, "android.hardware.ICameraClient");
@@ -86,13 +98,22 @@ status_t BnCameraClient::onTransact(
             return NO_ERROR;
         } break;
         case DATA_CALLBACK: {
-            LOGV("RAW_CALLBACK");
+            LOGV("DATA_CALLBACK");
             CHECK_INTERFACE(ICameraClient, data, reply);
             int32_t msgType = data.readInt32();
             sp<IMemory> imageData = interface_cast<IMemory>(data.readStrongBinder());
             dataCallback(msgType, imageData);
             return NO_ERROR;
         } break;
+        case DATA_CALLBACK_TIMESTAMP: {
+            LOGV("DATA_CALLBACK_TIMESTAMP");
+            CHECK_INTERFACE(ICameraClient, data, reply);
+            nsecs_t timestamp = data.readInt64();
+            int32_t msgType = data.readInt32();
+            sp<IMemory> imageData = interface_cast<IMemory>(data.readStrongBinder());
+            dataCallbackTimestamp(timestamp, msgType, imageData);
+            return NO_ERROR;
+        } break;
         default:
             return BBinder::onTransact(code, data, reply, flags);
     }