Add support for timestamps into SurfaceTexture.

API addition: The timestamps are represented as nanoseconds from some
arbitrary time point. Like the SurfaceTexture transform matrix, the
timestamp retrieved by getTimestamp is for the last frame sent to the
GL texture using updateTexImage().

Camera HAL change: Expect vendors to set these timestamps using
native_window_set_buffers_timestamp().  For now, they are
autogenerated by SurfaceTextureClient if set_buffers_timestamp() is
never called, but such timing is likely not accurate enough to pass a
CTS test.

bug:3300707

Change-Id: Ife131a0c2a826ac27342e11b8a6c42ff49e1bea7

4 files changed, 65 insertions, 10 deletions

diff --git a/libs/gui/ISurfaceTexture.cpp b/libs/gui/ISurfaceTexture.cpp
index d661fd5..bc14ad5 100644
--- a/libs/gui/ISurfaceTexture.cpp
+++ b/libs/gui/ISurfaceTexture.cpp
@@ -88,10 +88,11 @@ public:
         return result;
     }
 
-    virtual status_t queueBuffer(int buf) {
+    virtual status_t queueBuffer(int buf, int64_t timestamp) {
         Parcel data, reply;
         data.writeInterfaceToken(ISurfaceTexture::getInterfaceDescriptor());
         data.writeInt32(buf);
+        data.writeInt64(timestamp);
         remote()->transact(QUEUE_BUFFER, data, &reply);
         status_t result = reply.readInt32();
         return result;
@@ -174,7 +175,8 @@ status_t BnSurfaceTexture::onTransact(
         case QUEUE_BUFFER: {
             CHECK_INTERFACE(ISurfaceTexture, data, reply);
             int buf = data.readInt32();
-            status_t result = queueBuffer(buf);
+            int64_t timestamp = data.readInt64();
+            status_t result = queueBuffer(buf, timestamp);
             reply->writeInt32(result);
             return NO_ERROR;
         } break;
@@ -196,7 +198,6 @@ status_t BnSurfaceTexture::onTransact(
             return NO_ERROR;
         } break;
         case SET_TRANSFORM: {
-            Rect reg;
             CHECK_INTERFACE(ISurfaceTexture, data, reply);
             uint32_t transform = data.readInt32();
             status_t result = setTransform(transform);
diff --git a/libs/gui/SurfaceTexture.cpp b/libs/gui/SurfaceTexture.cpp
index 5c6d71b..cdaca47 100644
--- a/libs/gui/SurfaceTexture.cpp
+++ b/libs/gui/SurfaceTexture.cpp
@@ -76,9 +76,15 @@ static float mtxRot270[16] = {
 static void mtxMul(float out[16], const float a[16], const float b[16]);
 
 SurfaceTexture::SurfaceTexture(GLuint tex) :
-    mBufferCount(MIN_BUFFER_SLOTS), mCurrentTexture(INVALID_BUFFER_SLOT),
-    mCurrentTransform(0), mLastQueued(INVALID_BUFFER_SLOT),
-    mLastQueuedTransform(0), mNextTransform(0), mTexName(tex) {
+    mBufferCount(MIN_BUFFER_SLOTS),
+    mCurrentTexture(INVALID_BUFFER_SLOT),
+    mCurrentTransform(0),
+    mCurrentTimestamp(0),
+    mLastQueued(INVALID_BUFFER_SLOT),
+    mLastQueuedTransform(0),
+    mLastQueuedTimestamp(0),
+    mNextTransform(0),
+    mTexName(tex) {
     LOGV("SurfaceTexture::SurfaceTexture");
     for (int i = 0; i < NUM_BUFFER_SLOTS; i++) {
         mSlots[i].mEglImage = EGL_NO_IMAGE_KHR;
@@ -153,7 +159,7 @@ status_t SurfaceTexture::dequeueBuffer(int *buf) {
     return OK;
 }
 
-status_t SurfaceTexture::queueBuffer(int buf) {
+status_t SurfaceTexture::queueBuffer(int buf, int64_t timestamp) {
     LOGV("SurfaceTexture::queueBuffer");
     Mutex::Autolock lock(mMutex);
     if (buf < 0 || mBufferCount <= buf) {
@@ -172,6 +178,7 @@ status_t SurfaceTexture::queueBuffer(int buf) {
     mLastQueued = buf;
     mLastQueuedCrop = mNextCrop;
     mLastQueuedTransform = mNextTransform;
+    mLastQueuedTimestamp = timestamp;
     if (mFrameAvailableListener != 0) {
         mFrameAvailableListener->onFrameAvailable();
     }
@@ -246,12 +253,13 @@ status_t SurfaceTexture::updateTexImage() {
         mCurrentTextureBuf = mSlots[mCurrentTexture].mGraphicBuffer;
         mCurrentCrop = mLastQueuedCrop;
         mCurrentTransform = mLastQueuedTransform;
+        mCurrentTimestamp = mLastQueuedTimestamp;
     }
     return OK;
 }
 
 void SurfaceTexture::getTransformMatrix(float mtx[16]) {
-    LOGV("SurfaceTexture::updateTexImage");
+    LOGV("SurfaceTexture::getTransformMatrix");
     Mutex::Autolock lock(mMutex);
 
     float xform[16];
@@ -342,6 +350,12 @@ void SurfaceTexture::getTransformMatrix(float mtx[16]) {
     mtxMul(mtx, mtxFlipV, mtxBeforeFlipV);
 }
 
+nsecs_t SurfaceTexture::getTimestamp() {
+    LOGV("SurfaceTexture::getTimestamp");
+    Mutex::Autolock lock(mMutex);
+    return mCurrentTimestamp;
+}
+
 void SurfaceTexture::setFrameAvailableListener(
         const sp<FrameAvailableListener>& l) {
     LOGV("SurfaceTexture::setFrameAvailableListener");
diff --git a/libs/gui/SurfaceTextureClient.cpp b/libs/gui/SurfaceTextureClient.cpp
index 7f1d9cb..a4812d0 100644
--- a/libs/gui/SurfaceTextureClient.cpp
+++ b/libs/gui/SurfaceTextureClient.cpp
@@ -26,7 +26,8 @@ namespace android {
 SurfaceTextureClient::SurfaceTextureClient(
         const sp<ISurfaceTexture>& surfaceTexture):
         mSurfaceTexture(surfaceTexture), mAllocator(0), mReqWidth(1),
-        mReqHeight(1), mReqFormat(DEFAULT_FORMAT), mReqUsage(0), mMutex() {
+        mReqHeight(1), mReqFormat(DEFAULT_FORMAT), mReqUsage(0),
+        mTimestamp(NATIVE_WINDOW_TIMESTAMP_AUTO), mMutex() {
     // Initialize the ANativeWindow function pointers.
     ANativeWindow::setSwapInterval  = setSwapInterval;
     ANativeWindow::dequeueBuffer    = dequeueBuffer;
@@ -135,9 +136,17 @@ int SurfaceTextureClient::lockBuffer(android_native_buffer_t* buffer) {
 int SurfaceTextureClient::queueBuffer(android_native_buffer_t* buffer) {
     LOGV("SurfaceTextureClient::queueBuffer");
     Mutex::Autolock lock(mMutex);
+    int64_t timestamp;
+    if (mTimestamp == NATIVE_WINDOW_TIMESTAMP_AUTO) {
+        timestamp = systemTime(SYSTEM_TIME_MONOTONIC);
+        LOGV("SurfaceTextureClient::queueBuffer making up timestamp: %.2f ms",
+             timestamp / 1000000.f);
+    } else {
+        timestamp = mTimestamp;
+    }
     for (int i = 0; i < NUM_BUFFER_SLOTS; i++) {
         if (mSlots[i]->handle == buffer->handle) {
-            return mSurfaceTexture->queueBuffer(i);
+            return mSurfaceTexture->queueBuffer(i, timestamp);
         }
     }
     LOGE("queueBuffer: unknown buffer queued");
@@ -196,6 +205,9 @@ int SurfaceTextureClient::perform(int operation, va_list args)
     case NATIVE_WINDOW_SET_BUFFERS_TRANSFORM:
         res = dispatchSetBuffersTransform(args);
         break;
+    case NATIVE_WINDOW_SET_BUFFERS_TIMESTAMP:
+        res = dispatchSetBuffersTimestamp(args);
+        break;
     default:
         res = NAME_NOT_FOUND;
         break;
@@ -240,6 +252,11 @@ int SurfaceTextureClient::dispatchSetBuffersTransform(va_list args) {
     return setBuffersTransform(transform);
 }
 
+int SurfaceTextureClient::dispatchSetBuffersTimestamp(va_list args) {
+    int64_t timestamp = va_arg(args, int64_t);
+    return setBuffersTimestamp(timestamp);
+}
+
 int SurfaceTextureClient::connect(int api) {
     LOGV("SurfaceTextureClient::connect");
     // XXX: Implement this!
@@ -323,6 +340,14 @@ int SurfaceTextureClient::setBuffersTransform(int transform)
     return err;
 }
 
+int SurfaceTextureClient::setBuffersTimestamp(int64_t timestamp)
+{
+    LOGV("SurfaceTextureClient::setBuffersTimestamp");
+    Mutex::Autolock lock(mMutex);
+    mTimestamp = timestamp;
+    return NO_ERROR;
+}
+
 void SurfaceTextureClient::freeAllBuffers() {
     for (int i = 0; i < NUM_BUFFER_SLOTS; i++) {
         mSlots[i] = 0;
diff --git a/libs/surfaceflinger_client/Surface.cpp b/libs/surfaceflinger_client/Surface.cpp
index 21d509a..0dfbf01 100644
--- a/libs/surfaceflinger_client/Surface.cpp
+++ b/libs/surfaceflinger_client/Surface.cpp
@@ -753,6 +753,9 @@ int Surface::perform(int operation, va_list args)
     case NATIVE_WINDOW_SET_BUFFERS_TRANSFORM:
         res = dispatch_set_buffers_transform( args );
         break;
+    case NATIVE_WINDOW_SET_BUFFERS_TIMESTAMP:
+        res = dispatch_set_buffers_timestamp( args );
+        break;
     default:
         res = NAME_NOT_FOUND;
         break;
@@ -792,6 +795,11 @@ int Surface::dispatch_set_buffers_transform(va_list args) {
     return setBuffersTransform(transform);
 }
 
+int Surface::dispatch_set_buffers_timestamp(va_list args) {
+    int64_t timestamp = va_arg(args, int64_t);
+    return setBuffersTimestamp(timestamp);
+}
+
 void Surface::setUsage(uint32_t reqUsage)
 {
     Mutex::Autolock _l(mSurfaceLock);
@@ -910,6 +918,13 @@ int Surface::setBuffersTransform(int transform)
     return NO_ERROR;
 }
 
+int Surface::setBuffersTimestamp(int64_t timestamp)
+{
+    // Surface doesn't really have anything meaningful to do with timestamps
+    // so they'll just be dropped here.
+    return NO_ERROR;
+}
+
 // ----------------------------------------------------------------------------
 
 int Surface::getConnectedApi() const