fastboot: add support for auto-resparsing large files

Add support to fastboot for automatically using libsparse to break large
files, whether they are in sparse or normal format, into multiple sparse
files that can each fit into the target's memory.  Allows flashing
images that are larger than the size of the available memory on the
target.

By default, any file over 512MB will be sparsed into 512MB chunks.  The
limit can be modified with the -m argument, or sparsing can be forced
with -S or avoided with -N.  If -m is not specified, the target can
override the default by implementing getvar:max-download-size

Change-Id: I6c59381c3d24475c4f2587ea877200b96971cbd7

6 files changed, 409 insertions, 45 deletions

diff --git a/fastboot/Android.mk b/fastboot/Android.mk
index 089b9bb..e3261a7 100644
--- a/fastboot/Android.mk
+++ b/fastboot/Android.mk
@@ -48,7 +48,7 @@ ifeq ($(HOST_OS),windows)
   LOCAL_C_INCLUDES += development/host/windows/usb/api
 endif
 
-LOCAL_STATIC_LIBRARIES := $(EXTRA_STATIC_LIBS) libzipfile libunz libext4_utils libz
+LOCAL_STATIC_LIBRARIES := $(EXTRA_STATIC_LIBS) libzipfile libunz libext4_utils libsparse libz
 
 ifneq ($(HOST_OS),windows)
 ifeq ($(HAVE_SELINUX), true)
diff --git a/fastboot/engine.c b/fastboot/engine.c
index 7dc29e4..93be3de 100644
--- a/fastboot/engine.c
+++ b/fastboot/engine.c
@@ -28,7 +28,6 @@
 
 #include "fastboot.h"
 #include "make_ext4fs.h"
-#include "ext4_utils.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -77,6 +76,7 @@ char *mkmsg(const char *fmt, ...)
 #define OP_QUERY      3
 #define OP_NOTICE     4
 #define OP_FORMAT     5
+#define OP_DOWNLOAD_SPARSE 6
 
 typedef struct Action Action;
 
@@ -382,6 +382,19 @@ void fb_queue_flash(const char *ptn, void *data, unsigned sz)
     a->msg = mkmsg("writing '%s'", ptn);
 }
 
+void fb_queue_flash_sparse(const char *ptn, struct sparse_file *s, unsigned sz)
+{
+    Action *a;
+
+    a = queue_action(OP_DOWNLOAD_SPARSE, "");
+    a->data = s;
+    a->size = 0;
+    a->msg = mkmsg("sending sparse '%s' (%d KB)", ptn, sz / 1024);
+
+    a = queue_action(OP_COMMAND, "flash:%s", ptn);
+    a->msg = mkmsg("writing '%s'", ptn);
+}
+
 static int match(char *str, const char **value, unsigned count)
 {
     const char *val;
@@ -580,6 +593,10 @@ int fb_execute_queue(usb_handle *usb)
             status = fb_format(a, usb, (int)a->data);
             status = a->func(a, status, status ? fb_get_error() : "");
             if (status) break;
+        } else if (a->op == OP_DOWNLOAD_SPARSE) {
+            status = fb_download_data_sparse(usb, a->data);
+            status = a->func(a, status, status ? fb_get_error() : "");
+            if (status) break;
         } else {
             die("bogus action");
         }
diff --git a/fastboot/fastboot.c b/fastboot/fastboot.c
index 2dc79e9..ff99173 100644
--- a/fastboot/fastboot.c
+++ b/fastboot/fastboot.c
@@ -26,9 +26,13 @@
  * SUCH DAMAGE.
  */
 
+#define _LARGEFILE64_SOURCE
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
 #include <string.h>
 #include <errno.h>
 #include <fcntl.h>
@@ -38,11 +42,20 @@
 #include <getopt.h>
 
 #include <sys/time.h>
+#include <sys/types.h>
+
 #include <bootimg.h>
+#include <sparse/sparse.h>
 #include <zipfile/zipfile.h>
 
 #include "fastboot.h"
 
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+
+#define DEFAULT_SPARSE_LIMIT (256 * 1024 * 1024)
+
 char cur_product[FB_RESPONSE_SZ + 1];
 
 void bootimg_set_cmdline(boot_img_hdr *h, const char *cmdline);
@@ -59,6 +72,8 @@ static const char *product = 0;
 static const char *cmdline = 0;
 static int wipe_data = 0;
 static unsigned short vendor_id = 0;
+static int64_t sparse_limit = -1;
+static int64_t target_sparse_limit = -1;
 
 static unsigned base_addr = 0x10000000;
 
@@ -117,7 +132,27 @@ char *find_item(const char *item, const char *product)
 
 #ifdef _WIN32
 void *load_file(const char *fn, unsigned *_sz);
+int64_t file_size(const char *fn);
 #else
+#if defined(__APPLE__) && defined(__MACH__)
+#define lseek64 lseek
+#define off64_t off_t
+#endif
+
+int64_t file_size(const char *fn)
+{
+    off64_t off;
+    int fd;
+
+    fd = open(fn, O_RDONLY);
+    if (fd < 0) return -1;
+
+    off = lseek64(fd, 0, SEEK_END);
+    close(fd);
+
+    return off;
+}
+
 void *load_file(const char *fn, unsigned *_sz)
 {
     char *data;
@@ -245,6 +280,8 @@ void usage(void)
             "  -i <vendor id>                           specify a custom USB vendor id\n"
             "  -b <base_addr>                           specify a custom kernel base address\n"
             "  -n <page size>                           specify the nand page size. default: 2048\n"
+            "  -S <size>[K|M|G]                         automatically sparse files greater than\n"
+            "                                           size. default: 256M, 0 to disable\n"
         );
 }
 
@@ -430,6 +467,110 @@ void queue_info_dump(void)
     fb_queue_notice("--------------------------------------------");
 }
 
+
+struct sparse_file **load_sparse_files(const char *fname, int max_size)
+{
+    int fd;
+    struct sparse_file *s;
+    int files;
+    struct sparse_file **out_s;
+
+    fd = open(fname, O_RDONLY | O_BINARY);
+    if (fd < 0) {
+        die("cannot open '%s'\n", fname);
+    }
+
+    s = sparse_file_import_auto(fd, false);
+    if (!s) {
+        die("cannot sparse read file '%s'\n", fname);
+    }
+
+    files = sparse_file_resparse(s, max_size, NULL, 0);
+    if (files < 0) {
+        die("Failed to resparse '%s'\n", fname);
+    }
+
+    out_s = calloc(sizeof(struct sparse_file *), files + 1);
+    if (!out_s) {
+        die("Failed to allocate sparse file array\n");
+    }
+
+    files = sparse_file_resparse(s, max_size, out_s, files);
+    if (files < 0) {
+        die("Failed to resparse '%s'\n", fname);
+    }
+
+    return out_s;
+}
+
+static int64_t get_target_sparse_limit(struct usb_handle *usb)
+{
+    int64_t limit = 0;
+    char response[FB_RESPONSE_SZ + 1];
+    int status = fb_getvar(usb, response, "max-download-size");
+
+    if (!status) {
+        limit = strtoul(response, NULL, 0);
+        if (limit > 0) {
+            fprintf(stderr, "target reported max download size of %lld bytes\n",
+                    limit);
+        }
+    }
+
+    return limit;
+}
+
+static int64_t get_sparse_limit(struct usb_handle *usb, int64_t size)
+{
+    int64_t limit;
+
+    if (sparse_limit == 0) {
+        return 0;
+    } else if (sparse_limit > 0) {
+        limit = sparse_limit;
+    } else {
+        if (target_sparse_limit == -1) {
+            target_sparse_limit = get_target_sparse_limit(usb);
+        }
+        if (target_sparse_limit > 0) {
+            limit = target_sparse_limit;
+        } else {
+            limit = DEFAULT_SPARSE_LIMIT;
+        }
+    }
+
+    if (size > limit) {
+        return limit;
+    }
+
+    return 0;
+}
+
+void do_flash(usb_handle *usb, const char *pname, const char *fname)
+{
+    int64_t sz64;
+    void *data;
+    int64_t limit;
+
+    sz64 = file_size(fname);
+    limit = get_sparse_limit(usb, sz64);
+    if (limit) {
+        struct sparse_file **s = load_sparse_files(fname, limit);
+        if (s == NULL) {
+            die("cannot sparse load '%s'\n", fname);
+        }
+        while (*s) {
+            sz64 = sparse_file_len(*s, true, false);
+            fb_queue_flash_sparse(pname, *s++, sz64);
+        }
+    } else {
+        unsigned int sz;
+        data = load_file(fname, &sz);
+        if (data == 0) die("cannot load '%s'\n", fname);
+        fb_queue_flash(pname, data, sz);
+    }
+}
+
 void do_update_signature(zipfile_t zip, char *fn)
 {
     void *data;
@@ -567,6 +708,47 @@ int do_oem_command(int argc, char **argv)
     return 0;
 }
 
+static int64_t parse_num(const char *arg)
+{
+    char *endptr;
+    unsigned long long num;
+
+    num = strtoull(arg, &endptr, 0);
+    if (endptr == arg) {
+        return -1;
+    }
+
+    if (*endptr == 'k' || *endptr == 'K') {
+        if (num >= (-1ULL) / 1024) {
+            return -1;
+        }
+        num *= 1024LL;
+        endptr++;
+    } else if (*endptr == 'm' || *endptr == 'M') {
+        if (num >= (-1ULL) / (1024 * 1024)) {
+            return -1;
+        }
+        num *= 1024LL * 1024LL;
+        endptr++;
+    } else if (*endptr == 'g' || *endptr == 'G') {
+        if (num >= (-1ULL) / (1024 * 1024 * 1024)) {
+            return -1;
+        }
+        num *= 1024LL * 1024LL * 1024LL;
+        endptr++;
+    }
+
+    if (*endptr != '\0') {
+        return -1;
+    }
+
+    if (num > INT64_MAX) {
+        return -1;
+    }
+
+    return num;
+}
+
 int main(int argc, char **argv)
 {
     int wants_wipe = 0;
@@ -577,13 +759,14 @@ int main(int argc, char **argv)
     unsigned page_size = 2048;
     int status;
     int c;
+    int r;
 
-    struct option longopts = { 0, 0, 0, 0 };
+    const struct option longopts = { 0, 0, 0, 0 };
 
     serial = getenv("ANDROID_SERIAL");
 
     while (1) {
-        c = getopt_long(argc, argv, "wb:n:s:p:c:i:h", &longopts, NULL);
+        c = getopt_long(argc, argv, "wb:n:s:S:p:c:i:m:h", &longopts, NULL);
         if (c < 0) {
             break;
         }
@@ -602,6 +785,12 @@ int main(int argc, char **argv)
         case 's':
             serial = optarg;
             break;
+        case 'S':
+            sparse_limit = parse_num(optarg);
+            if (sparse_limit < 0) {
+                    die("invalid sparse limit");
+            }
+            break;
         case 'p':
             product = optarg;
             break;
@@ -702,9 +891,7 @@ int main(int argc, char **argv)
                 skip(2);
             }
             if (fname == 0) die("cannot determine image filename for '%s'", pname);
-            data = load_file(fname, &sz);
-            if (data == 0) die("cannot load '%s'\n", fname);
-            fb_queue_flash(pname, data, sz);
+            do_flash(usb, pname, fname);
         } else if(!strcmp(*argv, "flash:raw")) {
             char *pname = argv[1];
             char *kname = argv[2];
diff --git a/fastboot/fastboot.h b/fastboot/fastboot.h
index a84b0be..9177932 100644
--- a/fastboot/fastboot.h
+++ b/fastboot/fastboot.h
@@ -31,10 +31,13 @@
 
 #include "usb.h"
 
+struct sparse_file;
+
 /* protocol.c - fastboot protocol */
 int fb_command(usb_handle *usb, const char *cmd);
 int fb_command_response(usb_handle *usb, const char *cmd, char *response);
 int fb_download_data(usb_handle *usb, const void *data, unsigned size);
+int fb_download_data_sparse(usb_handle *usb, struct sparse_file *s);
 char *fb_get_error(void);
 
 #define FB_COMMAND_SZ 64
@@ -43,6 +46,7 @@ char *fb_get_error(void);
 /* engine.c - high level command queue engine */
 int fb_getvar(struct usb_handle *usb, char *response, const char *fmt, ...);
 void fb_queue_flash(const char *ptn, void *data, unsigned sz);;
+void fb_queue_flash_sparse(const char *ptn, struct sparse_file *s, unsigned sz);
 void fb_queue_erase(const char *ptn);
 void fb_queue_format(const char *ptn, int skip_if_not_supported);
 void fb_queue_require(const char *prod, const char *var, int invert,
diff --git a/fastboot/protocol.c b/fastboot/protocol.c
index e871113..a0e0fd4 100644
--- a/fastboot/protocol.c
+++ b/fastboot/protocol.c
@@ -26,11 +26,18 @@
  * SUCH DAMAGE.
  */
 
+#define min(a, b) \
+    ({ typeof(a) _a = (a); typeof(b) _b = (b); (_a < _b) ? _a : _b; })
+#define round_down(a, b) \
+    ({ typeof(a) _a = (a); typeof(b) _b = (b); _a - (_a % _b); })
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <errno.h>
 
+#include <sparse/sparse.h>
+
 #include "fastboot.h"
 
 static char ERROR[128];
@@ -40,8 +47,7 @@ char *fb_get_error(void)
     return ERROR;
 }
 
-static int check_response(usb_handle *usb, unsigned size,
-                          unsigned data_okay, char *response)
+static int check_response(usb_handle *usb, unsigned int size, char *response)
 {
     unsigned char status[65];
     int r;
@@ -82,7 +88,7 @@ static int check_response(usb_handle *usb, unsigned size,
             return -1;
         }
 
-        if(!memcmp(status, "DATA", 4) && data_okay){
+        if(!memcmp(status, "DATA", 4) && size > 0){
             unsigned dsize = strtoul((char*) status + 4, 0, 16);
             if(dsize > size) {
                 strcpy(ERROR, "data size too large");
@@ -100,9 +106,8 @@ static int check_response(usb_handle *usb, unsigned size,
     return -1;
 }
 
-static int _command_send(usb_handle *usb, const char *cmd,
-                         const void *data, unsigned size,
-                         char *response)
+static int _command_start(usb_handle *usb, const char *cmd, unsigned size,
+                          char *response)
 {
     int cmdsize = strlen(cmd);
     int r;
@@ -122,46 +127,81 @@ static int _command_send(usb_handle *usb, const char *cmd,
         return -1;
     }
 
-    if(data == 0) {
-        return check_response(usb, size, 0, response);
+    return check_response(usb, size, response);
+}
+
+static int _command_data(usb_handle *usb, const void *data, unsigned size)
+{
+    int r;
+
+    r = usb_write(usb, data, size);
+    if(r < 0) {
+        sprintf(ERROR, "data transfer failure (%s)", strerror(errno));
+        usb_close(usb);
+        return -1;
+    }
+    if(r != ((int) size)) {
+        sprintf(ERROR, "data transfer failure (short transfer)");
+        usb_close(usb);
+        return -1;
     }
 
-    r = check_response(usb, size, 1, 0);
+    return r;
+}
+
+static int _command_end(usb_handle *usb)
+{
+    int r;
+    r = check_response(usb, 0, 0);
     if(r < 0) {
         return -1;
     }
-    size = r;
+    return 0;
+}
 
-    if(size) {
-        r = usb_write(usb, data, size);
-        if(r < 0) {
-            sprintf(ERROR, "data transfer failure (%s)", strerror(errno));
-            usb_close(usb);
-            return -1;
-        }
-        if(r != ((int) size)) {
-            sprintf(ERROR, "data transfer failure (short transfer)");
-            usb_close(usb);
-            return -1;
-        }
+static int _command_send(usb_handle *usb, const char *cmd,
+                         const void *data, unsigned size,
+                         char *response)
+{
+    int r;
+    if (size == 0) {
+        return -1;
     }
 
-    r = check_response(usb, 0, 0, 0);
+    r = _command_start(usb, cmd, size, response);
+    if (r < 0) {
+        return -1;
+    }
+
+    r = _command_data(usb, data, size);
+    if (r < 0) {
+        return -1;
+    }
+
+    r = _command_end(usb);
     if(r < 0) {
         return -1;
-    } else {
-        return size;
     }
+
+    return size;
+}
+
+static int _command_send_no_data(usb_handle *usb, const char *cmd,
+                                 char *response)
+{
+    int r;
+
+    return _command_start(usb, cmd, 0, response);
 }
 
 int fb_command(usb_handle *usb, const char *cmd)
 {
-    return _command_send(usb, cmd, 0, 0, 0);
+    return _command_send_no_data(usb, cmd, 0);
 }
 
 int fb_command_response(usb_handle *usb, const char *cmd, char *response)
 {
-    return _command_send(usb, cmd, 0, 0, response);
+    return _command_send_no_data(usb, cmd, response);
 }
 
 int fb_download_data(usb_handle *usb, const void *data, unsigned size)
@@ -179,3 +219,96 @@ int fb_download_data(usb_handle *usb, const void *data, unsigned size)
     }
 }
 
+#define USB_BUF_SIZE 512
+static char usb_buf[USB_BUF_SIZE];
+static int usb_buf_len;
+
+static int fb_download_data_sparse_write(void *priv, const void *data, int len)
+{
+    int r;
+    usb_handle *usb = priv;
+    int to_write;
+    const char *ptr = data;
+
+    if (usb_buf_len) {
+        to_write = min(USB_BUF_SIZE - usb_buf_len, len);
+
+        memcpy(usb_buf + usb_buf_len, ptr, to_write);
+        usb_buf_len += to_write;
+        ptr += to_write;
+        len -= to_write;
+    }
+
+    if (usb_buf_len == USB_BUF_SIZE) {
+        r = _command_data(usb, usb_buf, USB_BUF_SIZE);
+        if (r != USB_BUF_SIZE) {
+            return -1;
+        }
+        usb_buf_len = 0;
+    }
+
+    if (len > USB_BUF_SIZE) {
+        if (usb_buf_len > 0) {
+            sprintf(ERROR, "internal error: usb_buf not empty\n");
+            return -1;
+        }
+        to_write = round_down(len, USB_BUF_SIZE);
+        r = _command_data(usb, ptr, to_write);
+        if (r != to_write) {
+            return -1;
+        }
+        ptr += to_write;
+        len -= to_write;
+    }
+
+    if (len > 0) {
+        if (len > USB_BUF_SIZE) {
+            sprintf(ERROR, "internal error: too much left for usb_buf\n");
+            return -1;
+        }
+        memcpy(usb_buf, ptr, len);
+        usb_buf_len = len;
+    }
+
+    return 0;
+}
+
+static int fb_download_data_sparse_flush(usb_handle *usb)
+{
+    int r;
+
+    if (usb_buf_len > 0) {
+        r = _command_data(usb, usb_buf, usb_buf_len);
+        if (r != usb_buf_len) {
+            return -1;
+        }
+        usb_buf_len = 0;
+    }
+
+    return 0;
+}
+
+int fb_download_data_sparse(usb_handle *usb, struct sparse_file *s)
+{
+    char cmd[64];
+    int r;
+    int size = sparse_file_len(s, true, false);
+    if (size <= 0) {
+        return -1;
+    }
+
+    sprintf(cmd, "download:%08x", size);
+    r = _command_start(usb, cmd, size, 0);
+    if (r < 0) {
+        return -1;
+    }
+
+    r = sparse_file_callback(s, true, false, fb_download_data_sparse_write, usb);
+    if (r < 0) {
+        return -1;
+    }
+
+    fb_download_data_sparse_flush(usb);
+
+    return _command_end(usb);
+}
diff --git a/fastboot/util_windows.c b/fastboot/util_windows.c
index c3d545c..9e029fd 100644
--- a/fastboot/util_windows.c
+++ b/fastboot/util_windows.c
@@ -36,6 +36,29 @@
 
 #include <windows.h>
 
+int64_t file_size(const char *fn)
+{
+    HANDLE    file;
+    char     *data;
+    DWORD     sz;
+
+    file = CreateFile( fn,
+                       GENERIC_READ,
+                       FILE_SHARE_READ,
+                       NULL,
+                       OPEN_EXISTING,
+                       0,
+                       NULL );
+
+    if (file == INVALID_HANDLE_VALUE)
+        return -1;
+
+    sz = GetFileSize( file, NULL );
+    CloseHandle( file );
+
+    return sz;
+}
+
 void get_my_path(char exe[PATH_MAX])
 {
 	char*  r;
@@ -52,7 +75,7 @@ void *load_file(const char *fn, unsigned *_sz)
 {
     HANDLE    file;
     char     *data;
-    DWORD     file_size;
+    DWORD     sz;
 
     file = CreateFile( fn,
                        GENERIC_READ,
@@ -65,29 +88,29 @@ void *load_file(const char *fn, unsigned *_sz)
     if (file == INVALID_HANDLE_VALUE)
         return NULL;
 
-    file_size = GetFileSize( file, NULL );
+    sz = GetFileSize( file, NULL );
     data      = NULL;
 
-    if (file_size > 0) {
-        data = (char*) malloc( file_size );
+    if (sz > 0) {
+        data = (char*) malloc( sz );
         if (data == NULL) {
-            fprintf(stderr, "load_file: could not allocate %ld bytes\n", file_size );
-            file_size = 0;
+            fprintf(stderr, "load_file: could not allocate %ld bytes\n", sz );
+            sz = 0;
         } else {
             DWORD  out_bytes;
 
-            if ( !ReadFile( file, data, file_size, &out_bytes, NULL ) ||
-                 out_bytes != file_size )
+            if ( !ReadFile( file, data, sz, &out_bytes, NULL ) ||
+                 out_bytes != sz )
             {
-                fprintf(stderr, "load_file: could not read %ld bytes from '%s'\n", file_size, fn);
+                fprintf(stderr, "load_file: could not read %ld bytes from '%s'\n", sz, fn);
                 free(data);
                 data      = NULL;
-                file_size = 0;
+                sz = 0;
             }
         }
     }
     CloseHandle( file );
 
-    *_sz = (unsigned) file_size;
+    *_sz = (unsigned) sz;
     return  data;
 }