camera2: Fix LEGACY mode timestamps.

Bug: 15116722

- Add CaptureCollector class to accumulate buffer timestamps
  and manage lifecycle callbacks for each request.
- Set correct timestamps for buffers, results, and callbacks.

Change-Id: I75fa1049cf100d9d14c5ba8992be93ba1048df19

6 files changed, 594 insertions, 92 deletions

diff --git a/core/java/android/hardware/camera2/legacy/CameraDeviceState.java b/core/java/android/hardware/camera2/legacy/CameraDeviceState.java
index ab7e844..2fa9d85 100644
--- a/core/java/android/hardware/camera2/legacy/CameraDeviceState.java
+++ b/core/java/android/hardware/camera2/legacy/CameraDeviceState.java
@@ -65,7 +65,7 @@ public class CameraDeviceState {
         void onError(int errorCode, RequestHolder holder);
         void onConfiguring();
         void onIdle();
-        void onCaptureStarted(RequestHolder holder);
+        void onCaptureStarted(RequestHolder holder, long timestamp);
         void onCaptureResult(CameraMetadataNative result, RequestHolder holder);
     }
 
@@ -125,11 +125,12 @@ public class CameraDeviceState {
      * </p>
      *
      * @param request A {@link RequestHolder} containing the request for the current capture.
+     * @param timestamp The timestamp of the capture start in nanoseconds.
      * @return {@link CameraBinderDecorator#NO_ERROR}, or an error if one has occurred.
      */
-    public synchronized int setCaptureStart(final RequestHolder request) {
+    public synchronized int setCaptureStart(final RequestHolder request, long timestamp) {
         mCurrentRequest = request;
-        doStateTransition(STATE_CAPTURING);
+        doStateTransition(STATE_CAPTURING, timestamp);
         return mCurrentError;
     }
 
@@ -180,6 +181,10 @@ public class CameraDeviceState {
     }
 
     private void doStateTransition(int newState) {
+        doStateTransition(newState, /*timestamp*/0);
+    }
+
+    private void doStateTransition(int newState, final long timestamp) {
         if (DEBUG) {
             if (newState != mCurrentState) {
                 Log.d(TAG, "Transitioning to state " + newState);
@@ -250,7 +255,7 @@ public class CameraDeviceState {
                     mCurrentHandler.post(new Runnable() {
                         @Override
                         public void run() {
-                            mCurrentListener.onCaptureStarted(mCurrentRequest);
+                            mCurrentListener.onCaptureStarted(mCurrentRequest, timestamp);
                         }
                     });
                 }
diff --git a/core/java/android/hardware/camera2/legacy/CaptureCollector.java b/core/java/android/hardware/camera2/legacy/CaptureCollector.java
new file mode 100644
index 0000000..a9834d0
--- /dev/null
+++ b/core/java/android/hardware/camera2/legacy/CaptureCollector.java
@@ -0,0 +1,453 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package android.hardware.camera2.legacy;
+
+import android.hardware.camera2.impl.CameraMetadataNative;
+import android.util.Log;
+import android.util.Pair;
+
+import java.util.ArrayDeque;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Collect timestamps and state for each {@link CaptureRequest} as it passes through
+ * the Legacy camera pipeline.
+ */
+public class CaptureCollector {
+    private static final String TAG = "CaptureCollector";
+
+    private static final boolean DEBUG = Log.isLoggable(LegacyCameraDevice.DEBUG_PROP, Log.DEBUG);
+
+    private static final int FLAG_RECEIVED_JPEG = 1;
+    private static final int FLAG_RECEIVED_JPEG_TS = 2;
+    private static final int FLAG_RECEIVED_PREVIEW = 4;
+    private static final int FLAG_RECEIVED_PREVIEW_TS = 8;
+    private static final int FLAG_RECEIVED_ALL_JPEG = FLAG_RECEIVED_JPEG | FLAG_RECEIVED_JPEG_TS;
+    private static final int FLAG_RECEIVED_ALL_PREVIEW = FLAG_RECEIVED_PREVIEW |
+            FLAG_RECEIVED_PREVIEW_TS;
+
+    private static final int MAX_JPEGS_IN_FLIGHT = 1;
+
+    private class CaptureHolder {
+        private final RequestHolder mRequest;
+        private final LegacyRequest mLegacy;
+        public final boolean needsJpeg;
+        public final boolean needsPreview;
+
+        private long mTimestamp = 0;
+        private int mReceivedFlags = 0;
+        private boolean mHasStarted = false;
+
+        public CaptureHolder(RequestHolder request, LegacyRequest legacyHolder) {
+            mRequest = request;
+            mLegacy = legacyHolder;
+            needsJpeg = request.hasJpegTargets();
+            needsPreview = request.hasPreviewTargets();
+        }
+
+        public boolean isPreviewCompleted() {
+            return (mReceivedFlags & FLAG_RECEIVED_ALL_PREVIEW) == FLAG_RECEIVED_ALL_PREVIEW;
+        }
+
+        public  boolean isJpegCompleted() {
+            return (mReceivedFlags & FLAG_RECEIVED_ALL_JPEG) == FLAG_RECEIVED_ALL_JPEG;
+        }
+
+        public boolean isCompleted() {
+            return (needsJpeg == isJpegCompleted()) && (needsPreview == isPreviewCompleted());
+        }
+
+        public void tryComplete() {
+            if (needsPreview && isPreviewCompleted()) {
+                CaptureCollector.this.onPreviewCompleted();
+            }
+            if (isCompleted()) {
+                CaptureCollector.this.onRequestCompleted(mRequest, mLegacy, mTimestamp);
+            }
+        }
+
+        public void setJpegTimestamp(long timestamp) {
+            if (DEBUG) {
+                Log.d(TAG, "setJpegTimestamp - called for request " + mRequest.getRequestId());
+            }
+            if (!needsJpeg) {
+                throw new IllegalStateException(
+                        "setJpegTimestamp called for capture with no jpeg targets.");
+            }
+            if (isCompleted()) {
+                throw new IllegalStateException(
+                        "setJpegTimestamp called on already completed request.");
+            }
+
+            mReceivedFlags |= FLAG_RECEIVED_JPEG_TS;
+
+            if (mTimestamp == 0) {
+                mTimestamp = timestamp;
+            }
+
+            if (!mHasStarted) {
+                mHasStarted = true;
+                CaptureCollector.this.mDeviceState.setCaptureStart(mRequest, mTimestamp);
+            }
+
+            tryComplete();
+        }
+
+        public void setJpegProduced() {
+            if (DEBUG) {
+                Log.d(TAG, "setJpegProduced - called for request " + mRequest.getRequestId());
+            }
+            if (!needsJpeg) {
+                throw new IllegalStateException(
+                        "setJpegProduced called for capture with no jpeg targets.");
+            }
+            if (isCompleted()) {
+                throw new IllegalStateException(
+                        "setJpegProduced called on already completed request.");
+            }
+
+            mReceivedFlags |= FLAG_RECEIVED_JPEG;
+            tryComplete();
+        }
+
+        public void setPreviewTimestamp(long timestamp) {
+            if (DEBUG) {
+                Log.d(TAG, "setPreviewTimestamp - called for request " + mRequest.getRequestId());
+            }
+            if (!needsPreview) {
+                throw new IllegalStateException(
+                        "setPreviewTimestamp called for capture with no preview targets.");
+            }
+            if (isCompleted()) {
+                throw new IllegalStateException(
+                        "setPreviewTimestamp called on already completed request.");
+            }
+
+            mReceivedFlags |= FLAG_RECEIVED_PREVIEW_TS;
+
+            if (mTimestamp == 0) {
+                mTimestamp = timestamp;
+            }
+
+            if (!needsJpeg) {
+                if (!mHasStarted) {
+                    mHasStarted = true;
+                    CaptureCollector.this.mDeviceState.setCaptureStart(mRequest, mTimestamp);
+                }
+            }
+
+            tryComplete();
+        }
+
+        public void setPreviewProduced() {
+            if (DEBUG) {
+                Log.d(TAG, "setPreviewProduced - called for request " + mRequest.getRequestId());
+            }
+            if (!needsPreview) {
+                throw new IllegalStateException(
+                        "setPreviewProduced called for capture with no preview targets.");
+            }
+            if (isCompleted()) {
+                throw new IllegalStateException(
+                        "setPreviewProduced called on already completed request.");
+            }
+
+            mReceivedFlags |= FLAG_RECEIVED_PREVIEW;
+            tryComplete();
+        }
+    }
+
+    private final ArrayDeque<CaptureHolder> mJpegCaptureQueue;
+    private final ArrayDeque<CaptureHolder> mJpegProduceQueue;
+    private final ArrayDeque<CaptureHolder> mPreviewCaptureQueue;
+    private final ArrayDeque<CaptureHolder> mPreviewProduceQueue;
+
+    private final ReentrantLock mLock = new ReentrantLock();
+    private final Condition mIsEmpty;
+    private final Condition mPreviewsEmpty;
+    private final Condition mNotFull;
+    private final CameraDeviceState mDeviceState;
+    private final LegacyResultMapper mMapper = new LegacyResultMapper();
+    private int mInFlight = 0;
+    private int mInFlightPreviews = 0;
+    private final int mMaxInFlight;
+
+    /**
+     * Create a new {@link CaptureCollector} that can modify the given {@link CameraDeviceState}.
+     *
+     * @param maxInFlight max allowed in-flight requests.
+     * @param deviceState the {@link CameraDeviceState} to update as requests are processed.
+     */
+    public CaptureCollector(int maxInFlight, CameraDeviceState deviceState) {
+        mMaxInFlight = maxInFlight;
+        mJpegCaptureQueue = new ArrayDeque<>(MAX_JPEGS_IN_FLIGHT);
+        mJpegProduceQueue = new ArrayDeque<>(MAX_JPEGS_IN_FLIGHT);
+        mPreviewCaptureQueue = new ArrayDeque<>(mMaxInFlight);
+        mPreviewProduceQueue = new ArrayDeque<>(mMaxInFlight);
+        mIsEmpty = mLock.newCondition();
+        mNotFull = mLock.newCondition();
+        mPreviewsEmpty = mLock.newCondition();
+        mDeviceState = deviceState;
+    }
+
+    /**
+     * Queue a new request.
+     *
+     * <p>
+     * For requests that use the Camera1 API preview output stream, this will block if there are
+     * already {@code maxInFlight} requests in progress (until at least one prior request has
+     * completed). For requests that use the Camera1 API jpeg callbacks, this will block until
+     * all prior requests have been completed to avoid stopping preview for
+     * {@link android.hardware.Camera#takePicture} before prior preview requests have been
+     * completed.
+     * </p>
+     * @param holder the {@link RequestHolder} for this request.
+     * @param legacy the {@link LegacyRequest} for this request; this will not be mutated.
+     * @param timeout a timeout to use for this call.
+     * @param unit the units to use for the timeout.
+     * @return {@code false} if this method timed out.
+     * @throws InterruptedException if this thread is interrupted.
+     */
+    public boolean queueRequest(RequestHolder holder, LegacyRequest legacy, long timeout,
+                                TimeUnit unit)
+            throws InterruptedException {
+        CaptureHolder h = new CaptureHolder(holder, legacy);
+        long nanos = unit.toNanos(timeout);
+        final ReentrantLock lock = this.mLock;
+        lock.lock();
+        try {
+            if (DEBUG) {
+                Log.d(TAG, "queueRequest  for request " + holder.getRequestId() +
+                        " - " + mInFlight + " requests remain in flight.");
+            }
+            if (h.needsJpeg) {
+                // Wait for all current requests to finish before queueing jpeg.
+                while (mInFlight > 0) {
+                    if (nanos <= 0) {
+                        return false;
+                    }
+                    nanos = mIsEmpty.awaitNanos(nanos);
+                }
+                mJpegCaptureQueue.add(h);
+                mJpegProduceQueue.add(h);
+            }
+            if (h.needsPreview) {
+                while (mInFlight >= mMaxInFlight) {
+                    if (nanos <= 0) {
+                        return false;
+                    }
+                    nanos = mNotFull.awaitNanos(nanos);
+                }
+                mPreviewCaptureQueue.add(h);
+                mPreviewProduceQueue.add(h);
+                mInFlightPreviews++;
+            }
+
+            if (!(h.needsJpeg || h.needsPreview)) {
+                throw new IllegalStateException("Request must target at least one output surface!");
+            }
+
+            mInFlight++;
+            return true;
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    /**
+     * Wait all queued requests to complete.
+     *
+     * @param timeout a timeout to use for this call.
+     * @param unit the units to use for the timeout.
+     * @return {@code false} if this method timed out.
+     * @throws InterruptedException if this thread is interrupted.
+     */
+    public boolean waitForEmpty(long timeout, TimeUnit unit) throws InterruptedException {
+        long nanos = unit.toNanos(timeout);
+        final ReentrantLock lock = this.mLock;
+        lock.lock();
+        try {
+            while (mInFlight > 0) {
+                if (nanos <= 0) {
+                    return false;
+                }
+                nanos = mIsEmpty.awaitNanos(nanos);
+            }
+            return true;
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    /**
+     * Wait all queued requests that use the Camera1 API preview output to complete.
+     *
+     * @param timeout a timeout to use for this call.
+     * @param unit the units to use for the timeout.
+     * @return {@code false} if this method timed out.
+     * @throws InterruptedException if this thread is interrupted.
+     */
+    public boolean waitForPreviewsEmpty(long timeout, TimeUnit unit) throws InterruptedException {
+        long nanos = unit.toNanos(timeout);
+        final ReentrantLock lock = this.mLock;
+        lock.lock();
+        try {
+            while (mInFlightPreviews > 0) {
+                if (nanos <= 0) {
+                    return false;
+                }
+                nanos = mPreviewsEmpty.awaitNanos(nanos);
+            }
+            return true;
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    /**
+     * Called to alert the {@link CaptureCollector} that the jpeg capture has begun.
+     *
+     * @param timestamp the time of the jpeg capture.
+     * @return the {@link RequestHolder} for the request associated with this capture.
+     */
+    public RequestHolder jpegCaptured(long timestamp) {
+        final ReentrantLock lock = this.mLock;
+        lock.lock();
+        try {
+            CaptureHolder h = mJpegCaptureQueue.poll();
+            if (h == null) {
+                Log.w(TAG, "jpegCaptured called with no jpeg request on queue!");
+                return null;
+            }
+            h.setJpegTimestamp(timestamp);
+            return h.mRequest;
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    /**
+     * Called to alert the {@link CaptureCollector} that the jpeg capture has completed.
+     *
+     * @return a pair containing the {@link RequestHolder} and the timestamp of the capture.
+     */
+    public Pair<RequestHolder, Long> jpegProduced() {
+        final ReentrantLock lock = this.mLock;
+        lock.lock();
+        try {
+            CaptureHolder h = mJpegProduceQueue.poll();
+            if (h == null) {
+                Log.w(TAG, "jpegProduced called with no jpeg request on queue!");
+                return null;
+            }
+            h.setJpegProduced();
+            return new Pair<>(h.mRequest, h.mTimestamp);
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    /**
+     * Check if there are any pending capture requests that use the Camera1 API preview output.
+     *
+     * @return {@code true} if there are pending preview requests.
+     */
+    public boolean hasPendingPreviewCaptures() {
+        final ReentrantLock lock = this.mLock;
+        lock.lock();
+        try {
+            return !mPreviewCaptureQueue.isEmpty();
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    /**
+     * Called to alert the {@link CaptureCollector} that the preview capture has begun.
+     *
+     * @param timestamp the time of the preview capture.
+     * @return a pair containing the {@link RequestHolder} and the timestamp of the capture.
+     */
+    public Pair<RequestHolder, Long> previewCaptured(long timestamp) {
+        final ReentrantLock lock = this.mLock;
+        lock.lock();
+        try {
+            CaptureHolder h = mPreviewCaptureQueue.poll();
+            if (h == null) {
+                Log.w(TAG, "previewCaptured called with no preview request on queue!");
+                return null;
+            }
+            h.setPreviewTimestamp(timestamp);
+            return new Pair<>(h.mRequest, h.mTimestamp);
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    /**
+     * Called to alert the {@link CaptureCollector} that the preview capture has completed.
+     *
+     * @return the {@link RequestHolder} for the request associated with this capture.
+     */
+    public RequestHolder previewProduced() {
+        final ReentrantLock lock = this.mLock;
+        lock.lock();
+        try {
+            CaptureHolder h = mPreviewProduceQueue.poll();
+            if (h == null) {
+                Log.w(TAG, "previewProduced called with no preview request on queue!");
+                return null;
+            }
+            h.setPreviewProduced();
+            return h.mRequest;
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    private void onPreviewCompleted() {
+        mInFlightPreviews--;
+        if (mInFlightPreviews < 0) {
+            throw new IllegalStateException(
+                    "More preview captures completed than requests queued.");
+        }
+        if (mInFlightPreviews == 0) {
+            mPreviewsEmpty.signalAll();
+        }
+    }
+
+    private void onRequestCompleted(RequestHolder request, LegacyRequest legacyHolder,
+                                    long timestamp) {
+        mInFlight--;
+        if (DEBUG) {
+            Log.d(TAG, "Completed request " + request.getRequestId() +
+                    ", " + mInFlight + " requests remain in flight.");
+        }
+        if (mInFlight < 0) {
+            throw new IllegalStateException(
+                    "More captures completed than requests queued.");
+        }
+        mNotFull.signalAll();
+        if (mInFlight == 0) {
+            mIsEmpty.signalAll();
+        }
+        CameraMetadataNative result = mMapper.cachedConvertResultMetadata(
+                legacyHolder, timestamp);
+        mDeviceState.setCaptureResult(request, result);
+    }
+}
diff --git a/core/java/android/hardware/camera2/legacy/GLThreadManager.java b/core/java/android/hardware/camera2/legacy/GLThreadManager.java
index 5d44fd2..06521cf 100644
--- a/core/java/android/hardware/camera2/legacy/GLThreadManager.java
+++ b/core/java/android/hardware/camera2/legacy/GLThreadManager.java
@@ -25,6 +25,8 @@ import android.view.Surface;
 
 import java.util.Collection;
 
+import static com.android.internal.util.Preconditions.*;
+
 /**
  * GLThreadManager handles the thread used for rendering into the configured output surfaces.
  */
@@ -38,6 +40,8 @@ public class GLThreadManager {
     private static final int MSG_DROP_FRAMES = 4;
     private static final int MSG_ALLOW_FRAMES = 5;
 
+    private CaptureCollector mCaptureCollector;
+
     private final SurfaceTextureRenderer mTextureRenderer;
 
     private final RequestHandlerThread mGLHandlerThread;
@@ -51,10 +55,13 @@ public class GLThreadManager {
     private static class ConfigureHolder {
         public final ConditionVariable condition;
         public final Collection<Surface> surfaces;
+        public final CaptureCollector collector;
 
-        public ConfigureHolder(ConditionVariable condition, Collection<Surface> surfaces) {
+        public ConfigureHolder(ConditionVariable condition, Collection<Surface> surfaces,
+                               CaptureCollector collector) {
             this.condition = condition;
             this.surfaces = surfaces;
+            this.collector = collector;
         }
     }
 
@@ -74,6 +81,7 @@ public class GLThreadManager {
                     ConfigureHolder configure = (ConfigureHolder) msg.obj;
                     mTextureRenderer.cleanupEGLContext();
                     mTextureRenderer.configureSurfaces(configure.surfaces);
+                    mCaptureCollector = checkNotNull(configure.collector);
                     configure.condition.open();
                     mConfigured = true;
                     break;
@@ -88,7 +96,7 @@ public class GLThreadManager {
                     if (!mConfigured) {
                         Log.e(TAG, "Dropping frame, EGL context not configured!");
                     }
-                    mTextureRenderer.drawIntoSurfaces((Collection<Surface>) msg.obj);
+                    mTextureRenderer.drawIntoSurfaces(mCaptureCollector);
                     break;
                 case MSG_CLEANUP:
                     mTextureRenderer.cleanupEGLContext();
@@ -158,16 +166,11 @@ public class GLThreadManager {
     }
 
     /**
-     * Queue a new call to draw into a given set of surfaces.
-     *
-     * <p>
-     * The set of surfaces passed here must be a subset of the set of surfaces passed in
-     * the last call to {@link #setConfigurationAndWait}.
-     * </p>
-     *
-     * @param targets a collection of {@link android.view.Surface}s to draw into.
+     * Queue a new call to draw into the surfaces specified in the next available preview
+     * request from the {@link CaptureCollector} passed to
+     * {@link #setConfigurationAndWait(java.util.Collection, CaptureCollector)};
      */
-    public void queueNewFrame(Collection<Surface> targets) {
+    public void queueNewFrame() {
         Handler handler = mGLHandlerThread.getHandler();
 
         /**
@@ -175,7 +178,7 @@ public class GLThreadManager {
          * are produced, drop frames rather than allowing the queue to back up.
          */
         if (!handler.hasMessages(MSG_NEW_FRAME)) {
-            handler.sendMessage(handler.obtainMessage(MSG_NEW_FRAME, targets));
+            handler.sendMessage(handler.obtainMessage(MSG_NEW_FRAME));
         } else {
             Log.e(TAG, "GLThread dropping frame.  Not consuming frames quickly enough!");
         }
@@ -186,12 +189,14 @@ public class GLThreadManager {
      * this configuration has been applied.
      *
      * @param surfaces a collection of {@link android.view.Surface}s to configure.
+     * @param collector a {@link CaptureCollector} to retrieve requests from.
      */
-    public void setConfigurationAndWait(Collection<Surface> surfaces) {
+    public void setConfigurationAndWait(Collection<Surface> surfaces, CaptureCollector collector) {
+        checkNotNull(collector, "collector must not be null");
         Handler handler = mGLHandlerThread.getHandler();
 
         final ConditionVariable condition = new ConditionVariable(/*closed*/false);
-        ConfigureHolder configure = new ConfigureHolder(condition, surfaces);
+        ConfigureHolder configure = new ConfigureHolder(condition, surfaces, collector);
 
         Message m = handler.obtainMessage(MSG_NEW_CONFIGURATION, /*arg1*/0, /*arg2*/0, configure);
         handler.sendMessage(m);
diff --git a/core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java b/core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java
index 71d3d4b..cbf4a3d 100644
--- a/core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java
+++ b/core/java/android/hardware/camera2/legacy/LegacyCameraDevice.java
@@ -135,10 +135,9 @@ public class LegacyCameraDevice implements AutoCloseable {
         }
 
         @Override
-        public void onCaptureStarted(RequestHolder holder) {
+        public void onCaptureStarted(RequestHolder holder, final long timestamp) {
             final CaptureResultExtras extras = getExtrasFromRequest(holder);
 
-            final long timestamp = System.nanoTime();
             mResultHandler.post(new Runnable() {
                 @Override
                 public void run() {
@@ -146,7 +145,6 @@ public class LegacyCameraDevice implements AutoCloseable {
                         Log.d(TAG, "doing onCaptureStarted callback.");
                     }
                     try {
-                        // TODO: Don't fake timestamp
                         mDeviceCallbacks.onCaptureStarted(extras, timestamp);
                     } catch (RemoteException e) {
                         throw new IllegalStateException(
@@ -167,7 +165,6 @@ public class LegacyCameraDevice implements AutoCloseable {
                         Log.d(TAG, "doing onCaptureResult callback.");
                     }
                     try {
-                        // TODO: Don't fake metadata
                         mDeviceCallbacks.onResultReceived(result, extras);
                     } catch (RemoteException e) {
                         throw new IllegalStateException(
@@ -483,6 +480,12 @@ public class LegacyCameraDevice implements AutoCloseable {
         return new Size(dimens[0], dimens[1]);
     }
 
+    static void setNextTimestamp(Surface surface, long timestamp)
+            throws BufferQueueAbandonedException {
+        checkNotNull(surface);
+        LegacyExceptionUtils.throwOnError(nativeSetNextTimestamp(surface, timestamp));
+    }
+
     private static native int nativeDetectSurfaceType(Surface surface);
 
     private static native int nativeDetectSurfaceDimens(Surface surface,
@@ -506,4 +509,5 @@ public class LegacyCameraDevice implements AutoCloseable {
     private static native int nativeDetectTextureDimens(SurfaceTexture surfaceTexture,
             /*out*/int[/*2*/] dimens);
 
+    private static native int nativeSetNextTimestamp(Surface surface, long timestamp);
 }
diff --git a/core/java/android/hardware/camera2/legacy/RequestThreadManager.java b/core/java/android/hardware/camera2/legacy/RequestThreadManager.java
index cc7a90e..066b416 100644
--- a/core/java/android/hardware/camera2/legacy/RequestThreadManager.java
+++ b/core/java/android/hardware/camera2/legacy/RequestThreadManager.java
@@ -38,6 +38,8 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.TimeUnit;
 
 import static com.android.internal.util.Preconditions.*;
 
@@ -62,22 +64,20 @@ public class RequestThreadManager {
     private final CameraCharacteristics mCharacteristics;
 
     private final CameraDeviceState mDeviceState;
+    private final CaptureCollector mCaptureCollector;
 
     private static final int MSG_CONFIGURE_OUTPUTS = 1;
     private static final int MSG_SUBMIT_CAPTURE_REQUEST = 2;
     private static final int MSG_CLEANUP = 3;
 
+    private static final int MAX_IN_FLIGHT_REQUESTS = 2;
+
     private static final int PREVIEW_FRAME_TIMEOUT = 300; // ms
     private static final int JPEG_FRAME_TIMEOUT = 3000; // ms (same as CTS for API2)
 
     private static final float ASPECT_RATIO_TOLERANCE = 0.01f;
     private boolean mPreviewRunning = false;
 
-    private volatile long mLastJpegTimestamp;
-    private volatile long mLastPreviewTimestamp;
-    private volatile RequestHolder mInFlightPreview;
-    private volatile RequestHolder mInFlightJpeg;
-
     private final List<Surface> mPreviewOutputs = new ArrayList<>();
     private final List<Surface> mCallbackOutputs = new ArrayList<>();
     private GLThreadManager mGLThreadManager;
@@ -167,16 +167,16 @@ public class RequestThreadManager {
     }
 
     private final ConditionVariable mReceivedJpeg = new ConditionVariable(false);
-    private final ConditionVariable mReceivedPreview = new ConditionVariable(false);
 
     private final Camera.PictureCallback mJpegCallback = new Camera.PictureCallback() {
         @Override
         public void onPictureTaken(byte[] data, Camera camera) {
             Log.i(TAG, "Received jpeg.");
-            RequestHolder holder = mInFlightJpeg;
+            Pair<RequestHolder, Long> captureInfo = mCaptureCollector.jpegProduced();
+            RequestHolder holder = captureInfo.first;
+            long timestamp = captureInfo.second;
             if (holder == null) {
-                Log.w(TAG, "Dropping jpeg frame.");
-                mInFlightJpeg = null;
+                Log.e(TAG, "Dropping jpeg frame.");
                 return;
             }
             for (Surface s : holder.getHolderTargets()) {
@@ -184,6 +184,7 @@ public class RequestThreadManager {
                     if (RequestHolder.jpegType(s)) {
                         Log.i(TAG, "Producing jpeg buffer...");
                         LegacyCameraDevice.setSurfaceDimens(s, data.length, /*height*/1);
+                        LegacyCameraDevice.setNextTimestamp(s, timestamp);
                         LegacyCameraDevice.produceFrame(s, data, data.length, /*height*/1,
                                 CameraMetadataNative.NATIVE_JPEG_FORMAT);
                     }
@@ -191,6 +192,7 @@ public class RequestThreadManager {
                     Log.w(TAG, "Surface abandoned, dropping frame. ", e);
                 }
             }
+
             mReceivedJpeg.open();
         }
     };
@@ -198,7 +200,7 @@ public class RequestThreadManager {
     private final Camera.ShutterCallback mJpegShutterCallback = new Camera.ShutterCallback() {
         @Override
         public void onShutter() {
-            mLastJpegTimestamp = SystemClock.elapsedRealtimeNanos();
+            mCaptureCollector.jpegCaptured(SystemClock.elapsedRealtimeNanos());
         }
     };
 
@@ -206,29 +208,10 @@ public class RequestThreadManager {
             new SurfaceTexture.OnFrameAvailableListener() {
                 @Override
                 public void onFrameAvailable(SurfaceTexture surfaceTexture) {
-                    RequestHolder holder = mInFlightPreview;
-
                     if (DEBUG) {
                         mPrevCounter.countAndLog();
                     }
-
-                    if (holder == null) {
-                        mGLThreadManager.queueNewFrame(null);
-                        Log.w(TAG, "Dropping preview frame.");
-                        return;
-                    }
-
-                    mInFlightPreview = null;
-
-                    if (holder.hasPreviewTargets()) {
-                        mGLThreadManager.queueNewFrame(holder.getHolderTargets());
-                    }
-
-                    /**
-                     * TODO: Get timestamp from GL thread after buffer update.
-                     */
-                    mLastPreviewTimestamp = surfaceTexture.getTimestamp();
-                    mReceivedPreview.open();
+                    mGLThreadManager.queueNewFrame();
                 }
             };
 
@@ -256,14 +239,11 @@ public class RequestThreadManager {
             mCamera.setPreviewTexture(mDummyTexture);
             startPreview();
         }
-        mInFlightJpeg = request;
-        // TODO: Hook up shutter callback to CameraDeviceStateListener#onCaptureStarted
         mCamera.takePicture(mJpegShutterCallback, /*raw*/null, mJpegCallback);
         mPreviewRunning = false;
     }
 
     private void doPreviewCapture(RequestHolder request) throws IOException {
-        mInFlightPreview = request;
         if (mPreviewRunning) {
             return; // Already running
         }
@@ -290,8 +270,6 @@ public class RequestThreadManager {
         mPreviewOutputs.clear();
         mCallbackOutputs.clear();
         mPreviewTexture = null;
-        mInFlightPreview = null;
-        mInFlightJpeg = null;
 
         int facing = mCharacteristics.get(CameraCharacteristics.LENS_FACING);
         int orientation = mCharacteristics.get(CameraCharacteristics.SENSOR_ORIENTATION);
@@ -389,7 +367,7 @@ public class RequestThreadManager {
             mGLThreadManager.start();
         }
         mGLThreadManager.waitUntilStarted();
-        mGLThreadManager.setConfigurationAndWait(mPreviewOutputs);
+        mGLThreadManager.setConfigurationAndWait(mPreviewOutputs, mCaptureCollector);
         mGLThreadManager.allowNewFrames();
         mPreviewTexture = mGLThreadManager.getCurrentSurfaceTexture();
         if (mPreviewTexture != null) {
@@ -553,6 +531,18 @@ public class RequestThreadManager {
                     int sizes = config.surfaces != null ? config.surfaces.size() : 0;
                     Log.i(TAG, "Configure outputs: " + sizes +
                             " surfaces configured.");
+
+                    try {
+                        boolean success = mCaptureCollector.waitForEmpty(JPEG_FRAME_TIMEOUT,
+                                TimeUnit.MILLISECONDS);
+                        if (!success) {
+                            Log.e(TAG, "Timed out while queueing configure request.");
+                        }
+                    } catch (InterruptedException e) {
+                        // TODO: report error to CameraDevice
+                        Log.e(TAG, "Interrupted while waiting for requests to complete.");
+                    }
+
                     try {
                         configureOutputs(config.surfaces);
                     } catch (IOException e) {
@@ -571,6 +561,16 @@ public class RequestThreadManager {
                     // Get the next burst from the request queue.
                     Pair<BurstHolder, Long> nextBurst = mRequestQueue.getNext();
                     if (nextBurst == null) {
+                        try {
+                            boolean success = mCaptureCollector.waitForEmpty(JPEG_FRAME_TIMEOUT,
+                                    TimeUnit.MILLISECONDS);
+                            if (!success) {
+                                Log.e(TAG, "Timed out while waiting for empty.");
+                            }
+                        } catch (InterruptedException e) {
+                            // TODO: report error to CameraDevice
+                            Log.e(TAG, "Interrupted while waiting for requests to complete.");
+                        }
                         mDeviceState.setIdle();
                         stopPreview();
                         break;
@@ -603,39 +603,41 @@ public class RequestThreadManager {
                             if (!mParams.same(legacyRequest.parameters)) {
                                 mParams = legacyRequest.parameters;
                                 mCamera.setParameters(mParams);
+
                                 paramsChanged = true;
                             }
                         }
 
-                        mDeviceState.setCaptureStart(holder);
-                        long timestamp = 0;
                         try {
+                            boolean success = mCaptureCollector.queueRequest(holder,
+                                    mLastRequest, JPEG_FRAME_TIMEOUT, TimeUnit.MILLISECONDS);
+
+                            if (!success) {
+                                Log.e(TAG, "Timed out while queueing capture request.");
+                            }
                             if (holder.hasPreviewTargets()) {
-                                mReceivedPreview.close();
                                 doPreviewCapture(holder);
-                                if (!mReceivedPreview.block(PREVIEW_FRAME_TIMEOUT)) {
-                                    // TODO: report error to CameraDevice
-                                    Log.e(TAG, "Hit timeout for preview callback!");
-                                }
-                                timestamp = mLastPreviewTimestamp;
                             }
                             if (holder.hasJpegTargets()) {
+                                success = mCaptureCollector.
+                                        waitForPreviewsEmpty(PREVIEW_FRAME_TIMEOUT *
+                                                MAX_IN_FLIGHT_REQUESTS, TimeUnit.MILLISECONDS);
+                                if (!success) {
+                                    Log.e(TAG, "Timed out waiting for prior requests to complete.");
+                                }
                                 mReceivedJpeg.close();
                                 doJpegCapture(holder);
                                 if (!mReceivedJpeg.block(JPEG_FRAME_TIMEOUT)) {
                                     // TODO: report error to CameraDevice
                                     Log.e(TAG, "Hit timeout for jpeg callback!");
                                 }
-                                mInFlightJpeg = null;
-                                timestamp = mLastJpegTimestamp;
                             }
                         } catch (IOException e) {
-                            // TODO: err handling
+                            // TODO: report error to CameraDevice
                             throw new IOError(e);
-                        }
-
-                        if (timestamp == 0) {
-                            timestamp = SystemClock.elapsedRealtimeNanos();
+                        } catch (InterruptedException e) {
+                            // TODO: report error to CameraDevice
+                            Log.e(TAG, "Interrupted during capture.", e);
                         }
 
                         if (paramsChanged) {
@@ -647,11 +649,6 @@ public class RequestThreadManager {
                             // Update parameters to the latest that we think the camera is using
                             mLastRequest.setParameters(mParams);
                         }
-
-
-                        CameraMetadataNative result = mMapper.cachedConvertResultMetadata(
-                                mLastRequest, timestamp);
-                        mDeviceState.setCaptureResult(holder, result);
                     }
                     if (DEBUG) {
                         long totalTime = SystemClock.elapsedRealtimeNanos() - startTime;
@@ -661,6 +658,16 @@ public class RequestThreadManager {
                     break;
                 case MSG_CLEANUP:
                     mCleanup = true;
+                    try {
+                        boolean success = mCaptureCollector.waitForEmpty(JPEG_FRAME_TIMEOUT,
+                                TimeUnit.MILLISECONDS);
+                        if (!success) {
+                            Log.e(TAG, "Timed out while queueing cleanup request.");
+                        }
+                    } catch (InterruptedException e) {
+                        // TODO: report error to CameraDevice
+                        Log.e(TAG, "Interrupted while waiting for requests to complete.");
+                    }
                     if (mGLThreadManager != null) {
                         mGLThreadManager.quit();
                     }
@@ -693,6 +700,7 @@ public class RequestThreadManager {
         String name = String.format("RequestThread-%d", cameraId);
         TAG = name;
         mDeviceState = checkNotNull(deviceState, "deviceState must not be null");
+        mCaptureCollector = new CaptureCollector(MAX_IN_FLIGHT_REQUESTS, mDeviceState);
         mRequestThread = new RequestHandlerThread(name, mRequestHandlerCb);
     }
 
diff --git a/core/java/android/hardware/camera2/legacy/SurfaceTextureRenderer.java b/core/java/android/hardware/camera2/legacy/SurfaceTextureRenderer.java
index fdf9ba0..0687264 100644
--- a/core/java/android/hardware/camera2/legacy/SurfaceTextureRenderer.java
+++ b/core/java/android/hardware/camera2/legacy/SurfaceTextureRenderer.java
@@ -28,6 +28,7 @@ import android.opengl.GLES20;
 import android.opengl.Matrix;
 import android.text.format.Time;
 import android.util.Log;
+import android.util.Pair;
 import android.util.Size;
 import android.view.Surface;
 import android.os.SystemProperties;
@@ -599,41 +600,63 @@ public class SurfaceTextureRenderer {
 
     /**
      * Draw the current buffer in the {@link SurfaceTexture} returned from
-     * {@link #getSurfaceTexture()} into the given set of target surfaces.
+     * {@link #getSurfaceTexture()} into the set of target {@link Surface}s
+     * in the next request from the given {@link CaptureCollector}, or drop
+     * the frame if none is available.
      *
      * <p>
-     * The given surfaces must be a subset of the surfaces set in the last
-     * {@link #configureSurfaces(java.util.Collection)} call.
+     * Any {@link Surface}s targeted must be a subset of the {@link Surface}s
+     * set in the last {@link #configureSurfaces(java.util.Collection)} call.
      * </p>
      *
-     * @param targetSurfaces the surfaces to draw to.
+     * @param targetCollector the surfaces to draw to.
      */
-    public void drawIntoSurfaces(Collection<Surface> targetSurfaces) {
+    public void drawIntoSurfaces(CaptureCollector targetCollector) {
         if ((mSurfaces == null || mSurfaces.size() == 0)
                 && (mConversionSurfaces == null || mConversionSurfaces.size() == 0)) {
             return;
         }
 
+        boolean doTiming = targetCollector.hasPendingPreviewCaptures();
         checkGlError("before updateTexImage");
 
-        if (targetSurfaces == null) {
-            mSurfaceTexture.updateTexImage();
-            return;
+        if (doTiming) {
+            beginGlTiming();
         }
 
-        beginGlTiming();
-
         mSurfaceTexture.updateTexImage();
 
         long timestamp = mSurfaceTexture.getTimestamp();
-        addGlTimestamp(timestamp);
+
+        Pair<RequestHolder, Long> captureHolder = targetCollector.previewCaptured(timestamp);
+
+        // No preview request queued, drop frame.
+        if (captureHolder == null) {
+            Log.w(TAG, "Dropping preview frame.");
+            if (doTiming) {
+                endGlTiming();
+            }
+            return;
+        }
+
+        RequestHolder request = captureHolder.first;
+
+        Collection<Surface> targetSurfaces = request.getHolderTargets();
+        if (doTiming) {
+            addGlTimestamp(timestamp);
+        }
 
         List<Long> targetSurfaceIds = LegacyCameraDevice.getSurfaceIds(targetSurfaces);
         for (EGLSurfaceHolder holder : mSurfaces) {
             if (LegacyCameraDevice.containsSurfaceId(holder.surface, targetSurfaceIds)) {
                 makeCurrent(holder.eglSurface);
-                drawFrame(mSurfaceTexture, holder.width, holder.height);
-                swapBuffers(holder.eglSurface);
+                try {
+                    LegacyCameraDevice.setNextTimestamp(holder.surface, captureHolder.second);
+                    drawFrame(mSurfaceTexture, holder.width, holder.height);
+                    swapBuffers(holder.eglSurface);
+                } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) {
+                    Log.w(TAG, "Surface abandoned, dropping frame. ", e);
+                }
             }
         }
         for (EGLSurfaceHolder holder : mConversionSurfaces) {
@@ -647,6 +670,7 @@ public class SurfaceTextureRenderer {
 
                 try {
                     int format = LegacyCameraDevice.detectSurfaceType(holder.surface);
+                    LegacyCameraDevice.setNextTimestamp(holder.surface, captureHolder.second);
                     LegacyCameraDevice.produceFrame(holder.surface, mPBufferPixels.array(),
                             holder.width, holder.height, format);
                     swapBuffers(holder.eglSurface);
@@ -655,8 +679,11 @@ public class SurfaceTextureRenderer {
                 }
             }
         }
+        targetCollector.previewProduced();
 
-        endGlTiming();
+        if (doTiming) {
+            endGlTiming();
+        }
     }
 
     /**