diff options
| -rw-r--r-- | include/utils/Asset.h | 11 | ||||
| -rw-r--r-- | include/utils/StreamingZipInflater.h | 82 | ||||
| -rw-r--r-- | libs/utils/Android.mk | 3 | ||||
| -rw-r--r-- | libs/utils/Asset.cpp | 74 | ||||
| -rw-r--r-- | libs/utils/StreamingZipInflater.cpp | 224 |
5 files changed, 355 insertions, 39 deletions
diff --git a/include/utils/Asset.h b/include/utils/Asset.h index 5908bcc..2a09095 100644 --- a/include/utils/Asset.h +++ b/include/utils/Asset.h @@ -61,15 +61,6 @@ public: ACCESS_BUFFER, } AccessMode; - enum { - /* data larger than this does not get uncompressed into a buffer */ -#ifdef HAVE_ANDROID_OS - UNCOMPRESS_DATA_MAX = 1 * 1024 * 1024 -#else - UNCOMPRESS_DATA_MAX = 2 * 1024 * 1024 -#endif - }; - /* * Read data from the current offset. Returns the actual number of * bytes read, 0 on EOF, or -1 on error. @@ -317,6 +308,8 @@ private: FileMap* mMap; // for memory-mapped input int mFd; // for file input + class StreamingZipInflater* mZipInflater; // for streaming large compressed assets + unsigned char* mBuf; // for getBuffer() }; diff --git a/include/utils/StreamingZipInflater.h b/include/utils/StreamingZipInflater.h new file mode 100644 index 0000000..16867d8 --- /dev/null +++ b/include/utils/StreamingZipInflater.h @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __LIBS_STREAMINGZIPINFLATER_H +#define __LIBS_STREAMINGZIPINFLATER_H + +#include <unistd.h> +#include <inttypes.h> +#include <zlib.h> + +namespace android { + +class StreamingZipInflater { +public: + static const size_t INPUT_CHUNK_SIZE = 64 * 1024; + static const size_t OUTPUT_CHUNK_SIZE = 64 * 1024; + + // Flavor that pages in the compressed data from a fd + StreamingZipInflater(int fd, off_t compDataStart, size_t uncompSize, size_t compSize); + + // Flavor that gets the compressed data from an in-memory buffer + StreamingZipInflater(class FileMap* dataMap, size_t uncompSize); + + ~StreamingZipInflater(); + + // read 'count' bytes of uncompressed data from the current position. outBuf may + // be NULL, in which case the data is consumed and discarded. + ssize_t read(void* outBuf, size_t count); + + // seeking backwards requires uncompressing fom the beginning, so is very + // expensive. seeking forwards only requires uncompressing from the current + // position to the destination. + off_t seekAbsolute(off_t absoluteInputPosition); + +private: + void initInflateState(); + int readNextChunk(); + + // where to find the uncompressed data + int mFd; + off_t mInFileStart; // where the compressed data lives in the file + class FileMap* mDataMap; + + z_stream mInflateState; + bool mStreamNeedsInit; + + // output invariants for this asset + uint8_t* mOutBuf; // output buf for decompressed bytes + size_t mOutBufSize; // allocated size of mOutBuf + size_t mOutTotalSize; // total uncompressed size of the blob + + // current output state bookkeeping + off_t mOutCurPosition; // current position in total offset + size_t mOutLastDecoded; // last decoded byte + 1 in mOutbuf + size_t mOutDeliverable; // next undelivered byte of decoded output in mOutBuf + + // input invariants + uint8_t* mInBuf; + size_t mInBufSize; // allocated size of mInBuf; + size_t mInTotalSize; // total size of compressed data for this blob + + // input state bookkeeping + size_t mInNextChunkOffset; // offset from start of blob at which the next input chunk lies + // the z_stream contains state about input block consumption +}; + +} + +#endif diff --git a/libs/utils/Android.mk b/libs/utils/Android.mk index 8bd5823..2e20268 100644 --- a/libs/utils/Android.mk +++ b/libs/utils/Android.mk @@ -33,6 +33,7 @@ commonSources:= \ SharedBuffer.cpp \ Static.cpp \ StopWatch.cpp \ + StreamingZipInflater.cpp \ String8.cpp \ String16.cpp \ StringArray.cpp \ @@ -131,4 +132,4 @@ endif # team really wants is to build the stuff defined by this makefile. ifeq (,$(ONE_SHOT_MAKEFILE)) include $(call first-makefiles-under,$(LOCAL_PATH)) -endif
\ No newline at end of file +endif diff --git a/libs/utils/Asset.cpp b/libs/utils/Asset.cpp index 4295123..cef7db4 100644 --- a/libs/utils/Asset.cpp +++ b/libs/utils/Asset.cpp @@ -24,6 +24,7 @@ #include <utils/Asset.h> #include <utils/Atomic.h> #include <utils/FileMap.h> +#include <utils/StreamingZipInflater.h> #include <utils/ZipUtils.h> #include <utils/ZipFileRO.h> #include <utils/Log.h> @@ -659,7 +660,7 @@ const void* _FileAsset::ensureAlignment(FileMap* map) */ _CompressedAsset::_CompressedAsset(void) : mStart(0), mCompressedLen(0), mUncompressedLen(0), mOffset(0), - mMap(NULL), mFd(-1), mBuf(NULL) + mMap(NULL), mFd(-1), mZipInflater(NULL), mBuf(NULL) { } @@ -698,6 +699,10 @@ status_t _CompressedAsset::openChunk(int fd, off_t offset, mFd = fd; assert(mBuf == NULL); + if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) { + mZipInflater = new StreamingZipInflater(mFd, offset, uncompressedLen, compressedLen); + } + return NO_ERROR; } @@ -724,6 +729,9 @@ status_t _CompressedAsset::openChunk(FileMap* dataMap, int compressionMethod, mUncompressedLen = uncompressedLen; assert(mOffset == 0); + if (uncompressedLen > StreamingZipInflater::OUTPUT_CHUNK_SIZE) { + mZipInflater = new StreamingZipInflater(dataMap, uncompressedLen); + } return NO_ERROR; } @@ -739,26 +747,29 @@ ssize_t _CompressedAsset::read(void* buf, size_t count) assert(mOffset >= 0 && mOffset <= mUncompressedLen); - // TODO: if mAccessMode == ACCESS_STREAMING, use zlib more cleverly + /* If we're relying on a streaming inflater, go through that */ + if (mZipInflater) { + actual = mZipInflater->read(buf, count); + } else { + if (mBuf == NULL) { + if (getBuffer(false) == NULL) + return -1; + } + assert(mBuf != NULL); - if (mBuf == NULL) { - if (getBuffer(false) == NULL) - return -1; - } - assert(mBuf != NULL); + /* adjust count if we're near EOF */ + maxLen = mUncompressedLen - mOffset; + if (count > maxLen) + count = maxLen; - /* adjust count if we're near EOF */ - maxLen = mUncompressedLen - mOffset; - if (count > maxLen) - count = maxLen; + if (!count) + return 0; - if (!count) - return 0; - - /* copy from buffer */ - //printf("comp buf read\n"); - memcpy(buf, (char*)mBuf + mOffset, count); - actual = count; + /* copy from buffer */ + //printf("comp buf read\n"); + memcpy(buf, (char*)mBuf + mOffset, count); + actual = count; + } mOffset += actual; return actual; @@ -780,6 +791,9 @@ off_t _CompressedAsset::seek(off_t offset, int whence) if (newPosn == (off_t) -1) return newPosn; + if (mZipInflater) { + mZipInflater->seekAbsolute(newPosn); + } mOffset = newPosn; return mOffset; } @@ -793,10 +807,12 @@ void _CompressedAsset::close(void) mMap->release(); mMap = NULL; } - if (mBuf != NULL) { - delete[] mBuf; - mBuf = NULL; - } + + delete[] mBuf; + mBuf = NULL; + + delete mZipInflater; + mZipInflater = NULL; if (mFd > 0) { ::close(mFd); @@ -817,12 +833,6 @@ const void* _CompressedAsset::getBuffer(bool wordAligned) if (mBuf != NULL) return mBuf; - if (mUncompressedLen > UNCOMPRESS_DATA_MAX) { - LOGD("Data exceeds UNCOMPRESS_DATA_MAX (%ld vs %d)\n", - (long) mUncompressedLen, UNCOMPRESS_DATA_MAX); - goto bail; - } - /* * Allocate a buffer and read the file into it. */ @@ -853,7 +863,13 @@ const void* _CompressedAsset::getBuffer(bool wordAligned) goto bail; } - /* success! */ + /* + * Success - now that we have the full asset in RAM we + * no longer need the streaming inflater + */ + delete mZipInflater; + mZipInflater = NULL; + mBuf = buf; buf = NULL; diff --git a/libs/utils/StreamingZipInflater.cpp b/libs/utils/StreamingZipInflater.cpp new file mode 100644 index 0000000..2ebec93 --- /dev/null +++ b/libs/utils/StreamingZipInflater.cpp @@ -0,0 +1,224 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "szipinf" +#include <utils/Log.h> + +#include <utils/FileMap.h> +#include <utils/StreamingZipInflater.h> +#include <string.h> +#include <assert.h> + +static inline size_t min(size_t a, size_t b) { return (a < b) ? a : b; } + +using namespace android; + +/* + * Streaming access to compressed asset data in an open fd + */ +StreamingZipInflater::StreamingZipInflater(int fd, off_t compDataStart, + size_t uncompSize, size_t compSize) { + mFd = fd; + mDataMap = NULL; + mInFileStart = compDataStart; + mOutTotalSize = uncompSize; + mInTotalSize = compSize; + + mInBufSize = StreamingZipInflater::INPUT_CHUNK_SIZE; + mInBuf = new uint8_t[mInBufSize]; + + mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE; + mOutBuf = new uint8_t[mOutBufSize]; + + initInflateState(); +} + +/* + * Streaming access to compressed data held in an mmapped region of memory + */ +StreamingZipInflater::StreamingZipInflater(FileMap* dataMap, size_t uncompSize) { + mFd = -1; + mDataMap = dataMap; + mOutTotalSize = uncompSize; + mInTotalSize = dataMap->getDataLength(); + + mInBuf = (uint8_t*) dataMap->getDataPtr(); + mInBufSize = mInTotalSize; + + mOutBufSize = StreamingZipInflater::OUTPUT_CHUNK_SIZE; + mOutBuf = new uint8_t[mOutBufSize]; + + initInflateState(); +} + +StreamingZipInflater::~StreamingZipInflater() { + // tear down the in-flight zip state just in case + ::inflateEnd(&mInflateState); + + if (mDataMap == NULL) { + delete [] mInBuf; + } + delete [] mOutBuf; +} + +void StreamingZipInflater::initInflateState() { + LOGD("Initializing inflate state"); + + memset(&mInflateState, 0, sizeof(mInflateState)); + mInflateState.zalloc = Z_NULL; + mInflateState.zfree = Z_NULL; + mInflateState.opaque = Z_NULL; + mInflateState.next_in = (Bytef*)mInBuf; + mInflateState.next_out = (Bytef*) mOutBuf; + mInflateState.avail_out = mOutBufSize; + mInflateState.data_type = Z_UNKNOWN; + + mOutLastDecoded = mOutDeliverable = mOutCurPosition = 0; + mInNextChunkOffset = 0; + mStreamNeedsInit = true; + + if (mDataMap == NULL) { + ::lseek(mFd, mInFileStart, SEEK_SET); + mInflateState.avail_in = 0; // set when a chunk is read in + } else { + mInflateState.avail_in = mInBufSize; + } +} + +/* + * Basic approach: + * + * 1. If we have undelivered uncompressed data, send it. At this point + * either we've satisfied the request, or we've exhausted the available + * output data in mOutBuf. + * + * 2. While we haven't sent enough data to satisfy the request: + * 0. if the request is for more data than exists, bail. + * a. if there is no input data to decode, read some into the input buffer + * and readjust the z_stream input pointers + * b. point the output to the start of the output buffer and decode what we can + * c. deliver whatever output data we can + */ +ssize_t StreamingZipInflater::read(void* outBuf, size_t count) { + uint8_t* dest = (uint8_t*) outBuf; + size_t bytesRead = 0; + size_t toRead = min(count, size_t(mOutTotalSize - mOutCurPosition)); + while (toRead > 0) { + // First, write from whatever we already have decoded and ready to go + size_t deliverable = min(toRead, mOutLastDecoded - mOutDeliverable); + if (deliverable > 0) { + if (outBuf != NULL) memcpy(dest, mOutBuf + mOutDeliverable, deliverable); + mOutDeliverable += deliverable; + mOutCurPosition += deliverable; + dest += deliverable; + bytesRead += deliverable; + toRead -= deliverable; + } + + // need more data? time to decode some. + if (toRead > 0) { + // if we don't have any data to decode, read some in. If we're working + // from mmapped data this won't happen, because the clipping to total size + // will prevent reading off the end of the mapped input chunk. + if (mInflateState.avail_in == 0) { + int err = readNextChunk(); + if (err < 0) { + LOGE("Unable to access asset data: %d", err); + if (!mStreamNeedsInit) { + ::inflateEnd(&mInflateState); + initInflateState(); + } + return -1; + } + } + // we know we've drained whatever is in the out buffer now, so just + // start from scratch there, reading all the input we have at present. + mInflateState.next_out = (Bytef*) mOutBuf; + mInflateState.avail_out = mOutBufSize; + + /* + LOGD("Inflating to outbuf: avail_in=%u avail_out=%u next_in=%p next_out=%p", + mInflateState.avail_in, mInflateState.avail_out, + mInflateState.next_in, mInflateState.next_out); + */ + int result = Z_OK; + if (mStreamNeedsInit) { + LOGI("Initializing zlib to inflate"); + result = inflateInit2(&mInflateState, -MAX_WBITS); + mStreamNeedsInit = false; + } + if (result == Z_OK) result = ::inflate(&mInflateState, Z_SYNC_FLUSH); + if (result < 0) { + // Whoops, inflation failed + LOGE("Error inflating asset: %d", result); + ::inflateEnd(&mInflateState); + initInflateState(); + return -1; + } else { + if (result == Z_STREAM_END) { + // we know we have to have reached the target size here and will + // not try to read any further, so just wind things up. + ::inflateEnd(&mInflateState); + } + + // Note how much data we got, and off we go + mOutDeliverable = 0; + mOutLastDecoded = mOutBufSize - mInflateState.avail_out; + } + } + } + return bytesRead; +} + +int StreamingZipInflater::readNextChunk() { + assert(mDataMap == NULL); + + if (mInNextChunkOffset < mInTotalSize) { + size_t toRead = min(mInBufSize, mInTotalSize - mInNextChunkOffset); + if (toRead > 0) { + ssize_t didRead = ::read(mFd, mInBuf, toRead); + //LOGD("Reading input chunk, size %08x didread %08x", toRead, didRead); + if (didRead < 0) { + // TODO: error + LOGE("Error reading asset data"); + return didRead; + } else { + mInNextChunkOffset += didRead; + mInflateState.next_in = (Bytef*) mInBuf; + mInflateState.avail_in = didRead; + } + } + } + return 0; +} + +// seeking backwards requires uncompressing fom the beginning, so is very +// expensive. seeking forwards only requires uncompressing from the current +// position to the destination. +off_t StreamingZipInflater::seekAbsolute(off_t absoluteInputPosition) { + if (absoluteInputPosition < mOutCurPosition) { + // rewind and reprocess the data from the beginning + if (!mStreamNeedsInit) { + ::inflateEnd(&mInflateState); + } + initInflateState(); + read(NULL, absoluteInputPosition); + } else if (absoluteInputPosition > mOutCurPosition) { + read(NULL, absoluteInputPosition - mOutCurPosition); + } + // else if the target position *is* our current position, do nothing + return absoluteInputPosition; +} |