Support streaming of compressed assets > 1 megabyte

Compressed assets larger than one megabyte are now decompressed on demand
rather than being decompressed in their entirety and held in memory.  Reading
the data in order is relatively efficient, as is seeking forward in the stream.
Seeking backwards is supported, but requires reprocessing the compressed data
from the beginning, so is very inefficient.

In addition, the size limit on compressed assets has been eliminated.

Change-Id: I6e68247957e6c53e7e8ba70d12764695f1723bad

3 files changed, 271 insertions, 30 deletions

diff --git a/libs/utils/Android.mk b/libs/utils/Android.mk
index 8bd5823..2e20268 100644
--- a/libs/utils/Android.mk
+++ b/libs/utils/Android.mk
@@ -33,6 +33,7 @@ commonSources:= \
 	SharedBuffer.cpp \
 	Static.cpp \
 	StopWatch.cpp \
+	StreamingZipInflater.cpp \
 	String8.cpp \
 	String16.cpp \
 	StringArray.cpp \
@@ -131,4 +132,4 @@ endif
 # team really wants is to build the stuff defined by this makefile.
 ifeq (,$(ONE_SHOT_MAKEFILE))
 include $(call first-makefiles-under,$(LOCAL_PATH))
-endif
\ No newline at end of file
+endif
diff --git a/libs/utils/Asset.cpp b/libs/utils/Asset.cpp
index 4295123..cef7db4 100644
--- a/libs/utils/Asset.cpp
+++ b/libs/utils/Asset.cpp
@@ -24,6 +24,7 @@
 #include <utils/Asset.h>
 #include <utils/Atomic.h>
 #include <utils/FileMap.h>
+#include <utils/StreamingZipInflater.h>
 #include <utils/ZipUtils.h>
 #include <utils/ZipFileRO.h>
 #include <utils/Log.h>
@@ -659,7 +660,7 @@ const void* _FileAsset::ensureAlignment(FileMap* map)
  */
 _CompressedAsset::_CompressedAsset(void)
     : mStart(0), mCompressedLen(0), mUncompressedLen(0), mOffset(0),
-      mMap(NULL), mFd(-1), mBuf(NULL)
+      mMap(NULL), mFd(-1), mZipInflater(NULL), mBuf(NULL)
 {
 }
 
@@ -698,6 +699,10 @@ status_t _CompressedAsset::openChunk(int fd, off_t offset,
     mFd = fd;
     assert(mBuf == NULL);
 
+    if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) {
+        mZipInflater = new StreamingZipInflater(mFd, offset, uncompressedLen, compressedLen);
+    }
+
     return NO_ERROR;
 }
 
@@ -724,6 +729,9 @@ status_t _CompressedAsset::openChunk(FileMap* dataMap, int compressionMethod,
     mUncompressedLen = uncompressedLen;
     assert(mOffset == 0);
 
+    if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) {
+        mZipInflater = new StreamingZipInflater(dataMap, uncompressedLen);
+    }
     return NO_ERROR;
 }
 
@@ -739,26 +747,29 @@ ssize_t _CompressedAsset::read(void* buf, size_t count)
 
     assert(mOffset >= 0 && mOffset <= mUncompressedLen);
 
-    // TODO: if mAccessMode == ACCESS_STREAMING, use zlib more cleverly
+    /* If we're relying on a streaming inflater, go through that */
+    if (mZipInflater) {
+        actual = mZipInflater->read(buf, count);
+    } else {
+        if (mBuf == NULL) {
+            if (getBuffer(false) == NULL)
+                return -1;
+        }
+        assert(mBuf != NULL);
 
-    if (mBuf == NULL) {
-        if (getBuffer(false) == NULL)
-            return -1;
-    }
-    assert(mBuf != NULL);
+        /* adjust count if we're near EOF */
+        maxLen = mUncompressedLen - mOffset;
+        if (count > maxLen)
+            count = maxLen;
 
-    /* adjust count if we're near EOF */
-    maxLen = mUncompressedLen - mOffset;
-    if (count > maxLen)
-        count = maxLen;
+        if (!count)
+            return 0;
 
-    if (!count)
-        return 0;
-
-    /* copy from buffer */
-    //printf("comp buf read\n");
-    memcpy(buf, (char*)mBuf + mOffset, count);
-    actual = count;
+        /* copy from buffer */
+        //printf("comp buf read\n");
+        memcpy(buf, (char*)mBuf + mOffset, count);
+        actual = count;
+    }
 
     mOffset += actual;
     return actual;
@@ -780,6 +791,9 @@ off_t _CompressedAsset::seek(off_t offset, int whence)
     if (newPosn == (off_t) -1)
         return newPosn;
 
+    if (mZipInflater) {
+        mZipInflater->seekAbsolute(newPosn);
+    }
     mOffset = newPosn;
     return mOffset;
 }
@@ -793,10 +807,12 @@ void _CompressedAsset::close(void)
         mMap->release();
         mMap = NULL;
     }
-    if (mBuf != NULL) {
-        delete[] mBuf;
-        mBuf = NULL;
-    }
+
+    delete[] mBuf;
+    mBuf = NULL;
+
+    delete mZipInflater;
+    mZipInflater = NULL;
 
     if (mFd > 0) {
         ::close(mFd);
@@ -817,12 +833,6 @@ const void* _CompressedAsset::getBuffer(bool wordAligned)
     if (mBuf != NULL)
         return mBuf;
 
-    if (mUncompressedLen > UNCOMPRESS_DATA_MAX) {
-        LOGD("Data exceeds UNCOMPRESS_DATA_MAX (%ld vs %d)\n",
-            (long) mUncompressedLen, UNCOMPRESS_DATA_MAX);
-        goto bail;
-    }
-
     /*
      * Allocate a buffer and read the file into it.
      */
@@ -853,7 +863,13 @@ const void* _CompressedAsset::getBuffer(bool wordAligned)
             goto bail;
     }
 
-    /* success! */
+    /*
+     * Success - now that we have the full asset in RAM we
+     * no longer need the streaming inflater
+     */
+    delete mZipInflater;
+    mZipInflater = NULL;
+
     mBuf = buf;
     buf = NULL;
 
diff --git a/libs/utils/StreamingZipInflater.cpp b/libs/utils/StreamingZipInflater.cpp
new file mode 100644
index 0000000..2ebec93
--- /dev/null
+++ b/libs/utils/StreamingZipInflater.cpp
@@ -0,0 +1,224 @@
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "szipinf"
+#include <utils/Log.h>
+
+#include <utils/FileMap.h>
+#include <utils/StreamingZipInflater.h>
+#include <string.h>
+#include <assert.h>
+
+static inline size_t min(size_t a, size_t b) { return (a < b) ? a : b; }
+
+using namespace android;
+
+/*
+ * Streaming access to compressed asset data in an open fd
+ */
+StreamingZipInflater::StreamingZipInflater(int fd, off_t compDataStart,
+        size_t uncompSize, size_t compSize) {
+    mFd = fd;
+    mDataMap = NULL;
+    mInFileStart = compDataStart;
+    mOutTotalSize = uncompSize;
+    mInTotalSize = compSize;
+
+    mInBufSize = StreamingZipInflater::INPUT_CHUNK_SIZE;
+    mInBuf = new uint8_t[mInBufSize];
+
+    mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE;
+    mOutBuf = new uint8_t[mOutBufSize];
+
+    initInflateState();
+}
+
+/*
+ * Streaming access to compressed data held in an mmapped region of memory
+ */
+StreamingZipInflater::StreamingZipInflater(FileMap* dataMap, size_t uncompSize) {
+    mFd = -1;
+    mDataMap = dataMap;
+    mOutTotalSize = uncompSize;
+    mInTotalSize = dataMap->getDataLength();
+
+    mInBuf = (uint8_t*) dataMap->getDataPtr();
+    mInBufSize = mInTotalSize;
+
+    mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE;
+    mOutBuf = new uint8_t[mOutBufSize];
+
+    initInflateState();
+}
+
+StreamingZipInflater::~StreamingZipInflater() {
+    // tear down the in-flight zip state just in case
+    ::inflateEnd(&mInflateState);
+
+    if (mDataMap == NULL) {
+        delete [] mInBuf;
+    }
+    delete [] mOutBuf;
+}
+
+void StreamingZipInflater::initInflateState() {
+    LOGD("Initializing inflate state");
+
+    memset(&mInflateState, 0, sizeof(mInflateState));
+    mInflateState.zalloc = Z_NULL;
+    mInflateState.zfree = Z_NULL;
+    mInflateState.opaque = Z_NULL;
+    mInflateState.next_in = (Bytef*)mInBuf;
+    mInflateState.next_out = (Bytef*) mOutBuf;
+    mInflateState.avail_out = mOutBufSize;
+    mInflateState.data_type = Z_UNKNOWN;
+
+    mOutLastDecoded = mOutDeliverable = mOutCurPosition = 0;
+    mInNextChunkOffset = 0;
+    mStreamNeedsInit = true;
+
+    if (mDataMap == NULL) {
+        ::lseek(mFd, mInFileStart, SEEK_SET);
+        mInflateState.avail_in = 0; // set when a chunk is read in
+    } else {
+        mInflateState.avail_in = mInBufSize;
+    }
+}
+
+/*
+ * Basic approach:
+ *
+ * 1. If we have undelivered uncompressed data, send it.  At this point
+ *    either we've satisfied the request, or we've exhausted the available
+ *    output data in mOutBuf.
+ *
+ * 2. While we haven't sent enough data to satisfy the request:
+ *    0. if the request is for more data than exists, bail.
+ *    a. if there is no input data to decode, read some into the input buffer
+ *       and readjust the z_stream input pointers
+ *    b. point the output to the start of the output buffer and decode what we can
+ *    c. deliver whatever output data we can
+ */
+ssize_t StreamingZipInflater::read(void* outBuf, size_t count) {
+    uint8_t* dest = (uint8_t*) outBuf;
+    size_t bytesRead = 0;
+    size_t toRead = min(count, size_t(mOutTotalSize - mOutCurPosition));
+    while (toRead > 0) {
+        // First, write from whatever we already have decoded and ready to go
+        size_t deliverable = min(toRead, mOutLastDecoded - mOutDeliverable);
+        if (deliverable > 0) {
+            if (outBuf != NULL) memcpy(dest, mOutBuf + mOutDeliverable, deliverable);
+            mOutDeliverable += deliverable;
+            mOutCurPosition += deliverable;
+            dest += deliverable;
+            bytesRead += deliverable;
+            toRead -= deliverable;
+        }
+
+        // need more data?  time to decode some.
+        if (toRead > 0) {
+            // if we don't have any data to decode, read some in.  If we're working
+            // from mmapped data this won't happen, because the clipping to total size
+            // will prevent reading off the end of the mapped input chunk.
+            if (mInflateState.avail_in == 0) {
+                int err = readNextChunk();
+                if (err < 0) {
+                    LOGE("Unable to access asset data: %d", err);
+                    if (!mStreamNeedsInit) {
+                        ::inflateEnd(&mInflateState);
+                        initInflateState();
+                    }
+                    return -1;
+                }
+            }
+            // we know we've drained whatever is in the out buffer now, so just
+            // start from scratch there, reading all the input we have at present.
+            mInflateState.next_out = (Bytef*) mOutBuf;
+            mInflateState.avail_out = mOutBufSize;
+
+            /*
+            LOGD("Inflating to outbuf: avail_in=%u avail_out=%u next_in=%p next_out=%p",
+                    mInflateState.avail_in, mInflateState.avail_out,
+                    mInflateState.next_in, mInflateState.next_out);
+            */
+            int result = Z_OK;
+            if (mStreamNeedsInit) {
+                LOGI("Initializing zlib to inflate");
+                result = inflateInit2(&mInflateState, -MAX_WBITS);
+                mStreamNeedsInit = false;
+            }
+            if (result == Z_OK) result = ::inflate(&mInflateState, Z_SYNC_FLUSH);
+            if (result < 0) {
+                // Whoops, inflation failed
+                LOGE("Error inflating asset: %d", result);
+                ::inflateEnd(&mInflateState);
+                initInflateState();
+                return -1;
+            } else {
+                if (result == Z_STREAM_END) {
+                    // we know we have to have reached the target size here and will
+                    // not try to read any further, so just wind things up.
+                    ::inflateEnd(&mInflateState);
+                }
+
+                // Note how much data we got, and off we go
+                mOutDeliverable = 0;
+                mOutLastDecoded = mOutBufSize - mInflateState.avail_out;
+            }
+        }
+    }
+    return bytesRead;
+}
+
+int StreamingZipInflater::readNextChunk() {
+    assert(mDataMap == NULL);
+
+    if (mInNextChunkOffset < mInTotalSize) {
+        size_t toRead = min(mInBufSize, mInTotalSize - mInNextChunkOffset);
+        if (toRead > 0) {
+            ssize_t didRead = ::read(mFd, mInBuf, toRead);
+            //LOGD("Reading input chunk, size %08x didread %08x", toRead, didRead);
+            if (didRead < 0) {
+                // TODO: error
+                LOGE("Error reading asset data");
+                return didRead;
+            } else {
+                mInNextChunkOffset += didRead;
+                mInflateState.next_in = (Bytef*) mInBuf;
+                mInflateState.avail_in = didRead;
+            }
+        }
+    }
+    return 0;
+}
+
+// seeking backwards requires uncompressing fom the beginning, so is very
+// expensive.  seeking forwards only requires uncompressing from the current
+// position to the destination.
+off_t StreamingZipInflater::seekAbsolute(off_t absoluteInputPosition) {
+    if (absoluteInputPosition < mOutCurPosition) {
+        // rewind and reprocess the data from the beginning
+        if (!mStreamNeedsInit) {
+            ::inflateEnd(&mInflateState);
+        }
+        initInflateState();
+        read(NULL, absoluteInputPosition);
+    } else if (absoluteInputPosition > mOutCurPosition) {
+        read(NULL, absoluteInputPosition - mOutCurPosition);
+    }
+    // else if the target position *is* our current position, do nothing
+    return absoluteInputPosition;
+}