updater: Add LZMA support to blockimg

 * This adds support for LZMA compressed blockimages. The space savings
   from doing this is significant and will reduce our CDN costs by quite
   a bit.
 * I'll spend the saved $$$ on beer.

Change-Id: I6679220aaa29592fe6bbccd4136f0c239e45e2ae

5 files changed, 105 insertions, 3 deletions

diff --git a/minzip/Android.mk b/minzip/Android.mk
index 22eabfb..1605a53 100644
--- a/minzip/Android.mk
+++ b/minzip/Android.mk
@@ -10,9 +10,11 @@ LOCAL_SRC_FILES := \
 
 LOCAL_C_INCLUDES := \
 	external/zlib \
-	external/safe-iop/include
+	external/safe-iop/include \
+	external/lzma/xz-embedded
 
 LOCAL_STATIC_LIBRARIES := libselinux
+LOCAL_STATIC_LIBRARIES += libxz
 
 LOCAL_MODULE := libminzip
 
diff --git a/minzip/Zip.c b/minzip/Zip.c
index bdb565c..7823c38 100644
--- a/minzip/Zip.c
+++ b/minzip/Zip.c
@@ -5,6 +5,7 @@
  */
 #include "safe_iop.h"
 #include "zlib.h"
+#include "xz_config.h"
 
 #include <errno.h>
 #include <fcntl.h>
@@ -501,6 +502,68 @@ static bool processStoredEntry(const ZipArchive *pArchive,
     return processFunction(pArchive->addr + pEntry->offset, pEntry->uncompLen, cookie);
 }
 
+static bool processXZEntry(const ZipArchive *pArchive,
+    const ZipEntry *pEntry, ProcessZipEntryContentsFunction processFunction,
+    void *cookie)
+{
+    unsigned char out[32*1024];
+    struct xz_buf b;
+    struct xz_dec *s;
+    enum xz_ret ret;
+
+    printf("ok!\n");
+    xz_crc32_init();
+    xz_crc64_init();
+    s = xz_dec_init(XZ_DYNALLOC, 1 << 26);
+    if (s == NULL) {
+        LOGE("XZ decompression alloc failed\n");
+        goto bail;
+    }
+
+    b.in = pArchive->addr + pEntry->offset;
+    b.in_pos = 0;
+    b.in_size = pEntry->compLen;
+    b.out = out;
+    b.out_pos = 0;
+    b.out_size = sizeof(out);
+
+    do {
+        ret = xz_dec_run(s, &b);
+
+        LOGVV("+++ b.in_pos = %zu b.out_pos = %zu ret=%d\n", b.in_pos, b.out_pos, ret);
+        if (b.out_pos == sizeof(out)) {
+            LOGVV("+++ processing %d bytes\n", b.out_pos);
+            bool err = processFunction(out, b.out_pos, cookie);
+            if (!err) {
+                LOGW("Process function elected to fail (in xz_dec)\n");
+                goto xz_bail;
+            }
+            b.out_pos = 0;
+        }
+
+    } while (ret == XZ_OK);
+
+    assert(ret == XZ_STREAM_END);
+
+    bool err = processFunction(out, b.out_pos, cookie);
+    if (!err) {
+        LOGW("Process function elected to fail (in xz_dec)\n");
+        goto xz_bail;
+    }
+
+
+xz_bail:
+    xz_dec_end(s);
+
+bail:
+    if (b.in_pos != (unsigned long)pEntry->compLen) {
+        LOGW("Size mismatch on file after xz_dec (%ld vs %zu)\n",
+                pEntry->compLen, b.in_pos);
+        //return false;
+    }
+    return true;
+}
+
 static bool processDeflatedEntry(const ZipArchive *pArchive,
     const ZipEntry *pEntry, ProcessZipEntryContentsFunction processFunction,
     void *cookie)
@@ -638,6 +701,26 @@ static bool copyProcessFunction(const unsigned char *data, int dataLen,
 }
 
 /*
+ * Similar to mzProcessZipEntryContents, but explicitly process the stream
+ * using XZ/LZMA before calling processFunction.
+ *
+ * This is a separate function for use by the updater. LZMA provides huge
+ * size reductions vs deflate, but isn't actually supported by the ZIP format.
+ * We need to process it using as little memory as possible.
+ */
+bool mzProcessZipEntryContentsXZ(const ZipArchive *pArchive,
+    const ZipEntry *pEntry, ProcessZipEntryContentsFunction processFunction,
+    void *cookie)
+{
+    if (pEntry->compression == STORED) {
+        return processXZEntry(pArchive, pEntry, processFunction, cookie);
+    }
+    LOGE("Explicit XZ decoding of entry '%s' unsupported for type %d",
+            pEntry->fileName, pEntry->compression);
+    return false;
+}
+
+/*
  * Read an entry into a buffer allocated by the caller.
  */
 bool mzReadZipEntry(const ZipArchive* pArchive, const ZipEntry* pEntry,
diff --git a/minzip/Zip.h b/minzip/Zip.h
index 86d8db5..92c20a0 100644
--- a/minzip/Zip.h
+++ b/minzip/Zip.h
@@ -114,6 +114,18 @@ bool mzProcessZipEntryContents(const ZipArchive *pArchive,
     void *cookie);
 
 /*
+ * Similar to mzProcessZipEntryContents, but explicitly process the stream
+ * using XZ/LZMA before calling processFunction.
+ *
+ * This is a separate function for use by the updater. LZMA provides huge
+ * size reductions vs deflate, but isn't actually supported by the ZIP format.
+ * We need to process it using as little memory as possible.
+ */
+bool mzProcessZipEntryContentsXZ(const ZipArchive *pArchive,
+    const ZipEntry *pEntry, ProcessZipEntryContentsFunction processFunction,
+    void *cookie);
+
+/*
  * Read an entry into a buffer allocated by the caller.
  */
 bool mzReadZipEntry(const ZipArchive* pArchive, const ZipEntry* pEntry,
diff --git a/updater/Android.mk b/updater/Android.mk
index 6e224fb..470eb5d 100644
--- a/updater/Android.mk
+++ b/updater/Android.mk
@@ -41,7 +41,7 @@ endif
 
 LOCAL_STATIC_LIBRARIES += $(TARGET_RECOVERY_UPDATER_LIBS) $(TARGET_RECOVERY_UPDATER_EXTRA_LIBS)
 LOCAL_STATIC_LIBRARIES += libapplypatch libedify libmtdutils libminzip libz
-LOCAL_STATIC_LIBRARIES += libmincrypt libbz
+LOCAL_STATIC_LIBRARIES += libmincrypt libbz libxz
 LOCAL_STATIC_LIBRARIES += libcutils liblog libc
 LOCAL_STATIC_LIBRARIES += libselinux
 tune2fs_static_libraries := \
diff --git a/updater/blockimg.c b/updater/blockimg.c
index a93bfcb..be56fbd 100644
--- a/updater/blockimg.c
+++ b/updater/blockimg.c
@@ -341,7 +341,12 @@ static bool receive_new_data(const unsigned char* data, int size, void* cookie)
 
 static void* unzip_new_data(void* cookie) {
     NewThreadInfo* nti = (NewThreadInfo*) cookie;
-    mzProcessZipEntryContents(nti->za, nti->entry, receive_new_data, nti);
+    if (strncmp(".xz", nti->entry->fileName + (nti->entry->fileNameLen - 3), 3) == 0) {
+        mzProcessZipEntryContentsXZ(nti->za, nti->entry, receive_new_data, nti);
+    } else {
+        mzProcessZipEntryContents(nti->za, nti->entry, receive_new_data, nti);
+    }
+
     return NULL;
 }