From cacf65446dde64dc47c9a4d9a6af002f564274ea Mon Sep 17 00:00:00 2001 From: Steve Kondik Date: Wed, 10 Jun 2015 13:02:50 -0700 Subject: updater: Add LZMA support to blockimg * This adds support for LZMA compressed blockimages. The space savings from doing this is significant and will reduce our CDN costs by quite a bit. * I'll spend the saved $$$ on beer. Change-Id: I6679220aaa29592fe6bbccd4136f0c239e45e2ae --- minzip/Android.mk | 4 ++- minzip/Zip.c | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ minzip/Zip.h | 12 ++++++++ updater/Android.mk | 2 +- updater/blockimg.c | 7 ++++- 5 files changed, 105 insertions(+), 3 deletions(-) diff --git a/minzip/Android.mk b/minzip/Android.mk index 22eabfb..1605a53 100644 --- a/minzip/Android.mk +++ b/minzip/Android.mk @@ -10,9 +10,11 @@ LOCAL_SRC_FILES := \ LOCAL_C_INCLUDES := \ external/zlib \ - external/safe-iop/include + external/safe-iop/include \ + external/lzma/xz-embedded LOCAL_STATIC_LIBRARIES := libselinux +LOCAL_STATIC_LIBRARIES += libxz LOCAL_MODULE := libminzip diff --git a/minzip/Zip.c b/minzip/Zip.c index bdb565c..7823c38 100644 --- a/minzip/Zip.c +++ b/minzip/Zip.c @@ -5,6 +5,7 @@ */ #include "safe_iop.h" #include "zlib.h" +#include "xz_config.h" #include #include @@ -501,6 +502,68 @@ static bool processStoredEntry(const ZipArchive *pArchive, return processFunction(pArchive->addr + pEntry->offset, pEntry->uncompLen, cookie); } +static bool processXZEntry(const ZipArchive *pArchive, + const ZipEntry *pEntry, ProcessZipEntryContentsFunction processFunction, + void *cookie) +{ + unsigned char out[32*1024]; + struct xz_buf b; + struct xz_dec *s; + enum xz_ret ret; + + printf("ok!\n"); + xz_crc32_init(); + xz_crc64_init(); + s = xz_dec_init(XZ_DYNALLOC, 1 << 26); + if (s == NULL) { + LOGE("XZ decompression alloc failed\n"); + goto bail; + } + + b.in = pArchive->addr + pEntry->offset; + b.in_pos = 0; + b.in_size = pEntry->compLen; + b.out = out; + b.out_pos = 0; + b.out_size = sizeof(out); + + do { + ret = xz_dec_run(s, &b); + + LOGVV("+++ b.in_pos = %zu b.out_pos = %zu ret=%d\n", b.in_pos, b.out_pos, ret); + if (b.out_pos == sizeof(out)) { + LOGVV("+++ processing %d bytes\n", b.out_pos); + bool err = processFunction(out, b.out_pos, cookie); + if (!err) { + LOGW("Process function elected to fail (in xz_dec)\n"); + goto xz_bail; + } + b.out_pos = 0; + } + + } while (ret == XZ_OK); + + assert(ret == XZ_STREAM_END); + + bool err = processFunction(out, b.out_pos, cookie); + if (!err) { + LOGW("Process function elected to fail (in xz_dec)\n"); + goto xz_bail; + } + + +xz_bail: + xz_dec_end(s); + +bail: + if (b.in_pos != (unsigned long)pEntry->compLen) { + LOGW("Size mismatch on file after xz_dec (%ld vs %zu)\n", + pEntry->compLen, b.in_pos); + //return false; + } + return true; +} + static bool processDeflatedEntry(const ZipArchive *pArchive, const ZipEntry *pEntry, ProcessZipEntryContentsFunction processFunction, void *cookie) @@ -638,6 +701,26 @@ static bool copyProcessFunction(const unsigned char *data, int dataLen, } /* + * Similar to mzProcessZipEntryContents, but explicitly process the stream + * using XZ/LZMA before calling processFunction. + * + * This is a separate function for use by the updater. LZMA provides huge + * size reductions vs deflate, but isn't actually supported by the ZIP format. + * We need to process it using as little memory as possible. + */ +bool mzProcessZipEntryContentsXZ(const ZipArchive *pArchive, + const ZipEntry *pEntry, ProcessZipEntryContentsFunction processFunction, + void *cookie) +{ + if (pEntry->compression == STORED) { + return processXZEntry(pArchive, pEntry, processFunction, cookie); + } + LOGE("Explicit XZ decoding of entry '%s' unsupported for type %d", + pEntry->fileName, pEntry->compression); + return false; +} + +/* * Read an entry into a buffer allocated by the caller. */ bool mzReadZipEntry(const ZipArchive* pArchive, const ZipEntry* pEntry, diff --git a/minzip/Zip.h b/minzip/Zip.h index 86d8db5..92c20a0 100644 --- a/minzip/Zip.h +++ b/minzip/Zip.h @@ -114,6 +114,18 @@ bool mzProcessZipEntryContents(const ZipArchive *pArchive, void *cookie); /* + * Similar to mzProcessZipEntryContents, but explicitly process the stream + * using XZ/LZMA before calling processFunction. + * + * This is a separate function for use by the updater. LZMA provides huge + * size reductions vs deflate, but isn't actually supported by the ZIP format. + * We need to process it using as little memory as possible. + */ +bool mzProcessZipEntryContentsXZ(const ZipArchive *pArchive, + const ZipEntry *pEntry, ProcessZipEntryContentsFunction processFunction, + void *cookie); + +/* * Read an entry into a buffer allocated by the caller. */ bool mzReadZipEntry(const ZipArchive* pArchive, const ZipEntry* pEntry, diff --git a/updater/Android.mk b/updater/Android.mk index 6e224fb..470eb5d 100644 --- a/updater/Android.mk +++ b/updater/Android.mk @@ -41,7 +41,7 @@ endif LOCAL_STATIC_LIBRARIES += $(TARGET_RECOVERY_UPDATER_LIBS) $(TARGET_RECOVERY_UPDATER_EXTRA_LIBS) LOCAL_STATIC_LIBRARIES += libapplypatch libedify libmtdutils libminzip libz -LOCAL_STATIC_LIBRARIES += libmincrypt libbz +LOCAL_STATIC_LIBRARIES += libmincrypt libbz libxz LOCAL_STATIC_LIBRARIES += libcutils liblog libc LOCAL_STATIC_LIBRARIES += libselinux tune2fs_static_libraries := \ diff --git a/updater/blockimg.c b/updater/blockimg.c index a93bfcb..be56fbd 100644 --- a/updater/blockimg.c +++ b/updater/blockimg.c @@ -341,7 +341,12 @@ static bool receive_new_data(const unsigned char* data, int size, void* cookie) static void* unzip_new_data(void* cookie) { NewThreadInfo* nti = (NewThreadInfo*) cookie; - mzProcessZipEntryContents(nti->za, nti->entry, receive_new_data, nti); + if (strncmp(".xz", nti->entry->fileName + (nti->entry->fileNameLen - 3), 3) == 0) { + mzProcessZipEntryContentsXZ(nti->za, nti->entry, receive_new_data, nti); + } else { + mzProcessZipEntryContents(nti->za, nti->entry, receive_new_data, nti); + } + return NULL; } -- cgit v1.1