diff options
author | Christopher Tate <ctate@google.com> | 2010-07-28 15:36:48 -0700 |
---|---|---|
committer | Android (Google) Code Review <android-gerrit@google.com> | 2010-07-28 15:36:48 -0700 |
commit | bae6fe242ff9a97840ff0c5d9d87c0575d261682 (patch) | |
tree | 428e2e84aef9faeaf4a89d9663a42438649fcc36 /libs | |
parent | bc1daf016020ce022ceb971f09e862da5ee82b1d (diff) | |
parent | b100cbf178e91d6652ebbad3ed36684cacb9d10e (diff) | |
download | frameworks_base-bae6fe242ff9a97840ff0c5d9d87c0575d261682.zip frameworks_base-bae6fe242ff9a97840ff0c5d9d87c0575d261682.tar.gz frameworks_base-bae6fe242ff9a97840ff0c5d9d87c0575d261682.tar.bz2 |
Merge "Support streaming of compressed assets > 1 megabyte" into gingerbread
Diffstat (limited to 'libs')
-rw-r--r-- | libs/utils/Android.mk | 3 | ||||
-rw-r--r-- | libs/utils/Asset.cpp | 74 | ||||
-rw-r--r-- | libs/utils/StreamingZipInflater.cpp | 224 |
3 files changed, 271 insertions, 30 deletions
diff --git a/libs/utils/Android.mk b/libs/utils/Android.mk index 8bd5823..2e20268 100644 --- a/libs/utils/Android.mk +++ b/libs/utils/Android.mk @@ -33,6 +33,7 @@ commonSources:= \ SharedBuffer.cpp \ Static.cpp \ StopWatch.cpp \ + StreamingZipInflater.cpp \ String8.cpp \ String16.cpp \ StringArray.cpp \ @@ -131,4 +132,4 @@ endif # team really wants is to build the stuff defined by this makefile. ifeq (,$(ONE_SHOT_MAKEFILE)) include $(call first-makefiles-under,$(LOCAL_PATH)) -endif
\ No newline at end of file +endif diff --git a/libs/utils/Asset.cpp b/libs/utils/Asset.cpp index 4295123..cef7db4 100644 --- a/libs/utils/Asset.cpp +++ b/libs/utils/Asset.cpp @@ -24,6 +24,7 @@ #include <utils/Asset.h> #include <utils/Atomic.h> #include <utils/FileMap.h> +#include <utils/StreamingZipInflater.h> #include <utils/ZipUtils.h> #include <utils/ZipFileRO.h> #include <utils/Log.h> @@ -659,7 +660,7 @@ const void* _FileAsset::ensureAlignment(FileMap* map) */ _CompressedAsset::_CompressedAsset(void) : mStart(0), mCompressedLen(0), mUncompressedLen(0), mOffset(0), - mMap(NULL), mFd(-1), mBuf(NULL) + mMap(NULL), mFd(-1), mZipInflater(NULL), mBuf(NULL) { } @@ -698,6 +699,10 @@ status_t _CompressedAsset::openChunk(int fd, off_t offset, mFd = fd; assert(mBuf == NULL); + if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) { + mZipInflater = new StreamingZipInflater(mFd, offset, uncompressedLen, compressedLen); + } + return NO_ERROR; } @@ -724,6 +729,9 @@ status_t _CompressedAsset::openChunk(FileMap* dataMap, int compressionMethod, mUncompressedLen = uncompressedLen; assert(mOffset == 0); + if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) { + mZipInflater = new StreamingZipInflater(dataMap, uncompressedLen); + } return NO_ERROR; } @@ -739,26 +747,29 @@ ssize_t _CompressedAsset::read(void* buf, size_t count) assert(mOffset >= 0 && mOffset <= mUncompressedLen); - // TODO: if mAccessMode == ACCESS_STREAMING, use zlib more cleverly + /* If we're relying on a streaming inflater, go through that */ + if (mZipInflater) { + actual = mZipInflater->read(buf, count); + } else { + if (mBuf == NULL) { + if (getBuffer(false) == NULL) + return -1; + } + assert(mBuf != NULL); - if (mBuf == NULL) { - if (getBuffer(false) == NULL) - return -1; - } - assert(mBuf != NULL); + /* adjust count if we're near EOF */ + maxLen = mUncompressedLen - mOffset; + if (count > maxLen) + count = maxLen; - /* adjust count if we're near EOF */ - maxLen = mUncompressedLen - mOffset; - if (count > maxLen) - count = maxLen; + if (!count) + return 0; - if (!count) - return 0; - - /* copy from buffer */ - //printf("comp buf read\n"); - memcpy(buf, (char*)mBuf + mOffset, count); - actual = count; + /* copy from buffer */ + //printf("comp buf read\n"); + memcpy(buf, (char*)mBuf + mOffset, count); + actual = count; + } mOffset += actual; return actual; @@ -780,6 +791,9 @@ off_t _CompressedAsset::seek(off_t offset, int whence) if (newPosn == (off_t) -1) return newPosn; + if (mZipInflater) { + mZipInflater->seekAbsolute(newPosn); + } mOffset = newPosn; return mOffset; } @@ -793,10 +807,12 @@ void _CompressedAsset::close(void) mMap->release(); mMap = NULL; } - if (mBuf != NULL) { - delete[] mBuf; - mBuf = NULL; - } + + delete[] mBuf; + mBuf = NULL; + + delete mZipInflater; + mZipInflater = NULL; if (mFd > 0) { ::close(mFd); @@ -817,12 +833,6 @@ const void* _CompressedAsset::getBuffer(bool wordAligned) if (mBuf != NULL) return mBuf; - if (mUncompressedLen > UNCOMPRESS_DATA_MAX) { - LOGD("Data exceeds UNCOMPRESS_DATA_MAX (%ld vs %d)\n", - (long) mUncompressedLen, UNCOMPRESS_DATA_MAX); - goto bail; - } - /* * Allocate a buffer and read the file into it. */ @@ -853,7 +863,13 @@ const void* _CompressedAsset::getBuffer(bool wordAligned) goto bail; } - /* success! */ + /* + * Success - now that we have the full asset in RAM we + * no longer need the streaming inflater + */ + delete mZipInflater; + mZipInflater = NULL; + mBuf = buf; buf = NULL; diff --git a/libs/utils/StreamingZipInflater.cpp b/libs/utils/StreamingZipInflater.cpp new file mode 100644 index 0000000..2ebec93 --- /dev/null +++ b/libs/utils/StreamingZipInflater.cpp @@ -0,0 +1,224 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "szipinf" +#include <utils/Log.h> + +#include <utils/FileMap.h> +#include <utils/StreamingZipInflater.h> +#include <string.h> +#include <assert.h> + +static inline size_t min(size_t a, size_t b) { return (a < b) ? a : b; } + +using namespace android; + +/* + * Streaming access to compressed asset data in an open fd + */ +StreamingZipInflater::StreamingZipInflater(int fd, off_t compDataStart, + size_t uncompSize, size_t compSize) { + mFd = fd; + mDataMap = NULL; + mInFileStart = compDataStart; + mOutTotalSize = uncompSize; + mInTotalSize = compSize; + + mInBufSize = StreamingZipInflater::INPUT_CHUNK_SIZE; + mInBuf = new uint8_t[mInBufSize]; + + mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE; + mOutBuf = new uint8_t[mOutBufSize]; + + initInflateState(); +} + +/* + * Streaming access to compressed data held in an mmapped region of memory + */ +StreamingZipInflater::StreamingZipInflater(FileMap* dataMap, size_t uncompSize) { + mFd = -1; + mDataMap = dataMap; + mOutTotalSize = uncompSize; + mInTotalSize = dataMap->getDataLength(); + + mInBuf = (uint8_t*) dataMap->getDataPtr(); + mInBufSize = mInTotalSize; + + mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE; + mOutBuf = new uint8_t[mOutBufSize]; + + initInflateState(); +} + +StreamingZipInflater::~StreamingZipInflater() { + // tear down the in-flight zip state just in case + ::inflateEnd(&mInflateState); + + if (mDataMap == NULL) { + delete [] mInBuf; + } + delete [] mOutBuf; +} + +void StreamingZipInflater::initInflateState() { + LOGD("Initializing inflate state"); + + memset(&mInflateState, 0, sizeof(mInflateState)); + mInflateState.zalloc = Z_NULL; + mInflateState.zfree = Z_NULL; + mInflateState.opaque = Z_NULL; + mInflateState.next_in = (Bytef*)mInBuf; + mInflateState.next_out = (Bytef*) mOutBuf; + mInflateState.avail_out = mOutBufSize; + mInflateState.data_type = Z_UNKNOWN; + + mOutLastDecoded = mOutDeliverable = mOutCurPosition = 0; + mInNextChunkOffset = 0; + mStreamNeedsInit = true; + + if (mDataMap == NULL) { + ::lseek(mFd, mInFileStart, SEEK_SET); + mInflateState.avail_in = 0; // set when a chunk is read in + } else { + mInflateState.avail_in = mInBufSize; + } +} + +/* + * Basic approach: + * + * 1. If we have undelivered uncompressed data, send it. At this point + * either we've satisfied the request, or we've exhausted the available + * output data in mOutBuf. + * + * 2. While we haven't sent enough data to satisfy the request: + * 0. if the request is for more data than exists, bail. + * a. if there is no input data to decode, read some into the input buffer + * and readjust the z_stream input pointers + * b. point the output to the start of the output buffer and decode what we can + * c. deliver whatever output data we can + */ +ssize_t StreamingZipInflater::read(void* outBuf, size_t count) { + uint8_t* dest = (uint8_t*) outBuf; + size_t bytesRead = 0; + size_t toRead = min(count, size_t(mOutTotalSize - mOutCurPosition)); + while (toRead > 0) { + // First, write from whatever we already have decoded and ready to go + size_t deliverable = min(toRead, mOutLastDecoded - mOutDeliverable); + if (deliverable > 0) { + if (outBuf != NULL) memcpy(dest, mOutBuf + mOutDeliverable, deliverable); + mOutDeliverable += deliverable; + mOutCurPosition += deliverable; + dest += deliverable; + bytesRead += deliverable; + toRead -= deliverable; + } + + // need more data? time to decode some. + if (toRead > 0) { + // if we don't have any data to decode, read some in. If we're working + // from mmapped data this won't happen, because the clipping to total size + // will prevent reading off the end of the mapped input chunk. + if (mInflateState.avail_in == 0) { + int err = readNextChunk(); + if (err < 0) { + LOGE("Unable to access asset data: %d", err); + if (!mStreamNeedsInit) { + ::inflateEnd(&mInflateState); + initInflateState(); + } + return -1; + } + } + // we know we've drained whatever is in the out buffer now, so just + // start from scratch there, reading all the input we have at present. + mInflateState.next_out = (Bytef*) mOutBuf; + mInflateState.avail_out = mOutBufSize; + + /* + LOGD("Inflating to outbuf: avail_in=%u avail_out=%u next_in=%p next_out=%p", + mInflateState.avail_in, mInflateState.avail_out, + mInflateState.next_in, mInflateState.next_out); + */ + int result = Z_OK; + if (mStreamNeedsInit) { + LOGI("Initializing zlib to inflate"); + result = inflateInit2(&mInflateState, -MAX_WBITS); + mStreamNeedsInit = false; + } + if (result == Z_OK) result = ::inflate(&mInflateState, Z_SYNC_FLUSH); + if (result < 0) { + // Whoops, inflation failed + LOGE("Error inflating asset: %d", result); + ::inflateEnd(&mInflateState); + initInflateState(); + return -1; + } else { + if (result == Z_STREAM_END) { + // we know we have to have reached the target size here and will + // not try to read any further, so just wind things up. + ::inflateEnd(&mInflateState); + } + + // Note how much data we got, and off we go + mOutDeliverable = 0; + mOutLastDecoded = mOutBufSize - mInflateState.avail_out; + } + } + } + return bytesRead; +} + +int StreamingZipInflater::readNextChunk() { + assert(mDataMap == NULL); + + if (mInNextChunkOffset < mInTotalSize) { + size_t toRead = min(mInBufSize, mInTotalSize - mInNextChunkOffset); + if (toRead > 0) { + ssize_t didRead = ::read(mFd, mInBuf, toRead); + //LOGD("Reading input chunk, size %08x didread %08x", toRead, didRead); + if (didRead < 0) { + // TODO: error + LOGE("Error reading asset data"); + return didRead; + } else { + mInNextChunkOffset += didRead; + mInflateState.next_in = (Bytef*) mInBuf; + mInflateState.avail_in = didRead; + } + } + } + return 0; +} + +// seeking backwards requires uncompressing fom the beginning, so is very +// expensive. seeking forwards only requires uncompressing from the current +// position to the destination. +off_t StreamingZipInflater::seekAbsolute(off_t absoluteInputPosition) { + if (absoluteInputPosition < mOutCurPosition) { + // rewind and reprocess the data from the beginning + if (!mStreamNeedsInit) { + ::inflateEnd(&mInflateState); + } + initInflateState(); + read(NULL, absoluteInputPosition); + } else if (absoluteInputPosition > mOutCurPosition) { + read(NULL, absoluteInputPosition - mOutCurPosition); + } + // else if the target position *is* our current position, do nothing + return absoluteInputPosition; +} |