From 90221205a3e58f2a198faa838088dc7bc7c9c752 Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Tue, 9 Dec 2014 16:39:47 +0000 Subject: Support resuming block based OTAs Add support for transfer list version 3, which allows us to verify the status of each command and resume an interrupted block based OTA update. Notes on the changes: - Move the previous BlockImageUpdateFn to a shorter and reusable PerformBlockImageUpdate, which can be used also in BlockImageVerifyFn for verification. - Split individual transfer list commands into separate functions with unified parameters for clarity, and use a hash table to locate them during execution. - Move common block reading and writing to ReadBlocks and WriteBlocks to reduce code duplication, and rename the readblock and writeblock to less confusing read_all and write_all. The coding style of the new functions follows the existing style in the updater/edify code. Needs matching changes from Ia5c56379f570047f10f0aa7373a1025439495c98 Bug: 18262110 Change-Id: I1e752464134aeb2d396946348e6041acabe13942 --- updater/blockimg.c | 1988 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 1551 insertions(+), 437 deletions(-) (limited to 'updater') diff --git a/updater/blockimg.c b/updater/blockimg.c index 3026893..a0f81e9 100644 --- a/updater/blockimg.c +++ b/updater/blockimg.c @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -32,7 +34,7 @@ #include "applypatch/applypatch.h" #include "edify/expr.h" #include "mincrypt/sha.h" -#include "minzip/DirUtil.h" +#include "minzip/Hash.h" #include "updater.h" #define BLOCKSIZE 4096 @@ -46,6 +48,10 @@ #define BLKDISCARD _IO(0x12,119) #endif +#define STASH_DIRECTORY_BASE "/cache/recovery" +#define STASH_DIRECTORY_MODE 0700 +#define STASH_FILE_MODE 0600 + char* PrintSha1(const uint8_t* digest); typedef struct { @@ -80,44 +86,77 @@ static RangeSet* parse_range(char* text) { return out; } -static void readblock(int fd, uint8_t* data, size_t size) { +static int range_overlaps(RangeSet* r1, RangeSet* r2) { + int i, j, r1_0, r1_1, r2_0, r2_1; + + if (!r1 || !r2) { + return 0; + } + + for (i = 0; i < r1->count; ++i) { + r1_0 = r1->pos[i * 2]; + r1_1 = r1->pos[i * 2 + 1]; + + for (j = 0; j < r2->count; ++j) { + r2_0 = r2->pos[j * 2]; + r2_1 = r2->pos[j * 2 + 1]; + + if (!(r2_0 > r1_1 || r1_0 > r2_1)) { + return 1; + } + } + } + + return 0; +} + +static int read_all(int fd, uint8_t* data, size_t size) { size_t so_far = 0; while (so_far < size) { ssize_t r = read(fd, data+so_far, size-so_far); if (r < 0 && errno != EINTR) { fprintf(stderr, "read failed: %s\n", strerror(errno)); - return; + return -1; } else { so_far += r; } } + return 0; } -static void writeblock(int fd, const uint8_t* data, size_t size) { +static int write_all(int fd, const uint8_t* data, size_t size) { size_t written = 0; while (written < size) { ssize_t w = write(fd, data+written, size-written); if (w < 0 && errno != EINTR) { fprintf(stderr, "write failed: %s\n", strerror(errno)); - return; + return -1; } else { written += w; } } + + if (fsync(fd) == -1) { + fprintf(stderr, "fsync failed: %s\n", strerror(errno)); + return -1; + } + + return 0; } -static void check_lseek(int fd, off64_t offset, int whence) { +static int check_lseek(int fd, off64_t offset, int whence) { while (true) { off64_t ret = lseek64(fd, offset, whence); if (ret < 0) { if (errno != EINTR) { fprintf(stderr, "lseek64 failed: %s\n", strerror(errno)); - exit(1); + return -1; } } else { break; } } + return 0; } static void allocate(size_t size, uint8_t** buffer, size_t* buffer_alloc) { @@ -146,14 +185,21 @@ static ssize_t RangeSinkWrite(const uint8_t* data, ssize_t size, void* token) { if (rss->p_remain <= 0) { fprintf(stderr, "range sink write overrun"); - exit(1); + return 0; } ssize_t written = 0; while (size > 0) { size_t write_now = size; - if (rss->p_remain < write_now) write_now = rss->p_remain; - writeblock(rss->fd, data, write_now); + + if (rss->p_remain < write_now) { + write_now = rss->p_remain; + } + + if (write_all(rss->fd, data, write_now) == -1) { + break; + } + data += write_now; size -= write_now; @@ -164,12 +210,17 @@ static ssize_t RangeSinkWrite(const uint8_t* data, ssize_t size, void* token) { // move to the next block ++rss->p_block; if (rss->p_block < rss->tgt->count) { - rss->p_remain = (rss->tgt->pos[rss->p_block*2+1] - rss->tgt->pos[rss->p_block*2]) * BLOCKSIZE; - check_lseek(rss->fd, (off64_t)rss->tgt->pos[rss->p_block*2] * BLOCKSIZE, SEEK_SET); + rss->p_remain = (rss->tgt->pos[rss->p_block * 2 + 1] - + rss->tgt->pos[rss->p_block * 2]) * BLOCKSIZE; + + if (check_lseek(rss->fd, (off64_t)rss->tgt->pos[rss->p_block*2] * BLOCKSIZE, + SEEK_SET) == -1) { + break; + } } else { // we can't write any more; return how many bytes have // been written so far. - return written; + break; } } } @@ -245,6 +296,58 @@ static void* unzip_new_data(void* cookie) { return NULL; } +static int ReadBlocks(RangeSet* src, uint8_t* buffer, int fd) { + int i; + size_t p = 0; + size_t size; + + if (!src || !buffer) { + return -1; + } + + for (i = 0; i < src->count; ++i) { + if (check_lseek(fd, (off64_t) src->pos[i * 2] * BLOCKSIZE, SEEK_SET) == -1) { + return -1; + } + + size = (src->pos[i * 2 + 1] - src->pos[i * 2]) * BLOCKSIZE; + + if (read_all(fd, buffer + p, size) == -1) { + return -1; + } + + p += size; + } + + return 0; +} + +static int WriteBlocks(RangeSet* tgt, uint8_t* buffer, int fd) { + int i; + size_t p = 0; + size_t size; + + if (!tgt || !buffer) { + return -1; + } + + for (i = 0; i < tgt->count; ++i) { + if (check_lseek(fd, (off64_t) tgt->pos[i * 2] * BLOCKSIZE, SEEK_SET) == -1) { + return -1; + } + + size = (tgt->pos[i * 2 + 1] - tgt->pos[i * 2]) * BLOCKSIZE; + + if (write_all(fd, buffer + p, size) == -1) { + return -1; + } + + p += size; + } + + return 0; +} + // Do a source/target load for move/bsdiff/imgdiff in version 1. // 'wordsave' is the save_ptr of a strtok_r()-in-progress. We expect // to parse the remainder of the string as: @@ -255,522 +358,1522 @@ static void* unzip_new_data(void* cookie) { // it to make it larger if necessary. The target ranges are returned // in *tgt, if tgt is non-NULL. -static void LoadSrcTgtVersion1(char* wordsave, RangeSet** tgt, int* src_blocks, +static int LoadSrcTgtVersion1(char** wordsave, RangeSet** tgt, int* src_blocks, uint8_t** buffer, size_t* buffer_alloc, int fd) { char* word; + int rc; - word = strtok_r(NULL, " ", &wordsave); + word = strtok_r(NULL, " ", wordsave); RangeSet* src = parse_range(word); if (tgt != NULL) { - word = strtok_r(NULL, " ", &wordsave); + word = strtok_r(NULL, " ", wordsave); *tgt = parse_range(word); } allocate(src->size * BLOCKSIZE, buffer, buffer_alloc); - size_t p = 0; - int i; - for (i = 0; i < src->count; ++i) { - check_lseek(fd, (off64_t)src->pos[i*2] * BLOCKSIZE, SEEK_SET); - size_t sz = (src->pos[i*2+1] - src->pos[i*2]) * BLOCKSIZE; - readblock(fd, *buffer+p, sz); - p += sz; - } - + rc = ReadBlocks(src, *buffer, fd); *src_blocks = src->size; + free(src); + return rc; } -static void MoveRange(uint8_t* dest, RangeSet* locs, const uint8_t* source) { - // source contains packed data, which we want to move to the - // locations given in *locs in the dest buffer. source and dest - // may be the same buffer. +static int VerifyBlocks(const char *expected, const uint8_t *buffer, + size_t blocks, int printerror) { + char* hexdigest = NULL; + int rc = -1; + uint8_t digest[SHA_DIGEST_SIZE]; - int start = locs->size; - int i; - for (i = locs->count-1; i >= 0; --i) { - int blocks = locs->pos[i*2+1] - locs->pos[i*2]; - start -= blocks; - memmove(dest + (locs->pos[i*2] * BLOCKSIZE), source + (start * BLOCKSIZE), - blocks * BLOCKSIZE); + if (!expected || !buffer) { + return rc; } -} -// Do a source/target load for move/bsdiff/imgdiff in version 2. -// 'wordsave' is the save_ptr of a strtok_r()-in-progress. We expect -// to parse the remainder of the string as one of: -// -// -// (loads data from source image only) -// -// - <[stash_id:stash_range] ...> -// (loads data from stashes only) -// -// <[stash_id:stash_range] ...> -// (loads data from both source image and stashes) -// -// On return, buffer is filled with the loaded source data (rearranged -// and combined with stashed data as necessary). buffer may be -// reallocated if needed to accommodate the source data. *tgt is the -// target RangeSet. Any stashes required are taken from stash_table -// and free()'d after being used. + SHA_hash(buffer, blocks * BLOCKSIZE, digest); + hexdigest = PrintSha1(digest); -static void LoadSrcTgtVersion2(char* wordsave, RangeSet** tgt, int* src_blocks, - uint8_t** buffer, size_t* buffer_alloc, int fd, - uint8_t** stash_table) { - char* word; + if (hexdigest != NULL) { + rc = strcmp(expected, hexdigest); - if (tgt != NULL) { - word = strtok_r(NULL, " ", &wordsave); - *tgt = parse_range(word); + if (rc != 0 && printerror) { + fprintf(stderr, "failed to verify blocks (expected %s, read %s)\n", + expected, hexdigest); + } + + free(hexdigest); } - word = strtok_r(NULL, " ", &wordsave); - *src_blocks = strtol(word, NULL, 0); + return rc; +} - allocate(*src_blocks * BLOCKSIZE, buffer, buffer_alloc); +static char* GetStashFileName(const char* base, const char* id, const char* postfix) { + char* fn; + int len; + int res; - word = strtok_r(NULL, " ", &wordsave); - if (word[0] == '-' && word[1] == '\0') { - // no source ranges, only stashes - } else { - RangeSet* src = parse_range(word); + if (base == NULL) { + return NULL; + } - size_t p = 0; - int i; - for (i = 0; i < src->count; ++i) { - check_lseek(fd, (off64_t)src->pos[i*2] * BLOCKSIZE, SEEK_SET); - size_t sz = (src->pos[i*2+1] - src->pos[i*2]) * BLOCKSIZE; - readblock(fd, *buffer+p, sz); - p += sz; - } - free(src); + if (id == NULL) { + id = ""; + } - word = strtok_r(NULL, " ", &wordsave); - if (word == NULL) { - // no stashes, only source range - return; - } + if (postfix == NULL) { + postfix = ""; + } - RangeSet* locs = parse_range(word); - MoveRange(*buffer, locs, *buffer); + len = strlen(STASH_DIRECTORY_BASE) + 1 + strlen(base) + 1 + strlen(id) + strlen(postfix) + 1; + fn = malloc(len); + + if (fn == NULL) { + fprintf(stderr, "failed to malloc %d bytes for fn\n", len); + return NULL; } - while ((word = strtok_r(NULL, " ", &wordsave)) != NULL) { - // Each word is a an index into the stash table, a colon, and - // then a rangeset describing where in the source block that - // stashed data should go. - char* colonsave = NULL; - char* colon = strtok_r(word, ":", &colonsave); - int stash_id = strtol(colon, NULL, 0); - colon = strtok_r(NULL, ":", &colonsave); - RangeSet* locs = parse_range(colon); - MoveRange(*buffer, locs, stash_table[stash_id]); - free(stash_table[stash_id]); - stash_table[stash_id] = NULL; - free(locs); + res = snprintf(fn, len, STASH_DIRECTORY_BASE "/%s/%s%s", base, id, postfix); + + if (res < 0 || res >= len) { + fprintf(stderr, "failed to format file name (return value %d)\n", res); + free(fn); + return NULL; } + + return fn; } -// args: -// - block device (or file) to modify in-place -// - transfer list (blob) -// - new data stream (filename within package.zip) -// - patch stream (filename within package.zip, must be uncompressed) +typedef void (*StashCallback)(const char*, void*); -Value* BlockImageUpdateFn(const char* name, State* state, int argc, Expr* argv[]) { - Value* blockdev_filename; - Value* transfer_list_value; - char* transfer_list = NULL; - Value* new_data_fn; - Value* patch_data_fn; - bool success = false; +// Does a best effort enumeration of stash files. Ignores possible non-file +// items in the stash directory and continues despite of errors. Calls the +// 'callback' function for each file and passes 'data' to the function as a +// parameter. - if (ReadValueArgs(state, argv, 4, &blockdev_filename, &transfer_list_value, - &new_data_fn, &patch_data_fn) < 0) { - return NULL; - } +static void EnumerateStash(const char* dirname, StashCallback callback, void* data) { + char* fn; + DIR* directory; + int len; + int res; + struct dirent* item; - if (blockdev_filename->type != VAL_STRING) { - ErrorAbort(state, "blockdev_filename argument to %s must be string", name); - goto done; + if (dirname == NULL || callback == NULL) { + return; } - if (transfer_list_value->type != VAL_BLOB) { - ErrorAbort(state, "transfer_list argument to %s must be blob", name); - goto done; + + directory = opendir(dirname); + + if (directory == NULL) { + if (errno != ENOENT) { + fprintf(stderr, "failed to opendir %s (errno %d)\n", dirname, errno); + } + return; } - if (new_data_fn->type != VAL_STRING) { - ErrorAbort(state, "new_data_fn argument to %s must be string", name); - goto done; + + while ((item = readdir(directory)) != NULL) { + if (item->d_type != DT_REG) { + continue; + } + + len = strlen(dirname) + 1 + strlen(item->d_name) + 1; + fn = malloc(len); + + if (fn == NULL) { + fprintf(stderr, "failed to malloc %d bytes for fn\n", len); + continue; + } + + res = snprintf(fn, len, "%s/%s", dirname, item->d_name); + + if (res < 0 || res >= len) { + fprintf(stderr, "failed to format file name (return value %d)\n", res); + free(fn); + continue; + } + + callback(fn, data); + free(fn); } - if (patch_data_fn->type != VAL_STRING) { - ErrorAbort(state, "patch_data_fn argument to %s must be string", name); - goto done; + + if (closedir(directory) == -1) { + fprintf(stderr, "failed to closedir %s (errno %d)\n", dirname, errno); } +} - UpdaterInfo* ui = (UpdaterInfo*)(state->cookie); - FILE* cmd_pipe = ui->cmd_pipe; +static void UpdateFileSize(const char* fn, void* data) { + int* size = (int*) data; + struct stat st; - ZipArchive* za = ((UpdaterInfo*)(state->cookie))->package_zip; + if (!fn || !data) { + return; + } - const ZipEntry* patch_entry = mzFindZipEntry(za, patch_data_fn->data); - if (patch_entry == NULL) { - ErrorAbort(state, "%s(): no file \"%s\" in package", name, patch_data_fn->data); - goto done; + if (stat(fn, &st) == -1) { + fprintf(stderr, "failed to stat %s (errno %d)\n", fn, errno); + return; } - uint8_t* patch_start = ((UpdaterInfo*)(state->cookie))->package_zip_addr + - mzGetZipEntryOffset(patch_entry); + *size += st.st_size; +} - const ZipEntry* new_entry = mzFindZipEntry(za, new_data_fn->data); - if (new_entry == NULL) { - ErrorAbort(state, "%s(): no file \"%s\" in package", name, new_data_fn->data); - goto done; +// Deletes the stash directory and all files in it. Assumes that it only +// contains files. There is nothing we can do about unlikely, but possible +// errors, so they are merely logged. + +static void DeleteFile(const char* fn, void* data) { + if (fn) { + fprintf(stderr, "deleting %s\n", fn); + + if (unlink(fn) == -1 && errno != ENOENT) { + fprintf(stderr, "failed to unlink %s (errno %d)\n", fn, errno); + } } +} - // The transfer list is a text file containing commands to - // transfer data from one place to another on the target - // partition. We parse it and execute the commands in order: - // - // zero [rangeset] - // - fill the indicated blocks with zeros - // - // new [rangeset] - // - fill the blocks with data read from the new_data file - // - // erase [rangeset] - // - mark the given blocks as empty - // - // move <...> - // bsdiff <...> - // imgdiff <...> - // - read the source blocks, apply a patch (or not in the - // case of move), write result to target blocks. bsdiff or - // imgdiff specifies the type of patch; move means no patch - // at all. - // - // The format of <...> differs between versions 1 and 2; - // see the LoadSrcTgtVersion{1,2}() functions for a - // description of what's expected. - // - // stash - // - (version 2 only) load the given source range and stash - // the data in the given slot of the stash table. - // - // The creator of the transfer list will guarantee that no block - // is read (ie, used as the source for a patch or move) after it - // has been written. - // - // In version 2, the creator will guarantee that a given stash is - // loaded (with a stash command) before it's used in a - // move/bsdiff/imgdiff command. - // - // Within one command the source and target ranges may overlap so - // in general we need to read the entire source into memory before - // writing anything to the target blocks. - // - // All the patch data is concatenated into one patch_data file in - // the update package. It must be stored uncompressed because we - // memory-map it in directly from the archive. (Since patches are - // already compressed, we lose very little by not compressing - // their concatenation.) - - pthread_t new_data_thread; - NewThreadInfo nti; - nti.za = za; - nti.entry = new_entry; - nti.rss = NULL; - pthread_mutex_init(&nti.mu, NULL); - pthread_cond_init(&nti.cv, NULL); +static void DeletePartial(const char* fn, void* data) { + if (fn && strstr(fn, ".partial") != NULL) { + DeleteFile(fn, data); + } +} - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - pthread_create(&new_data_thread, &attr, unzip_new_data, &nti); +static void DeleteStash(const char* base) { + char* dirname; - int i, j; + if (base == NULL) { + return; + } - char* linesave; - char* wordsave; + dirname = GetStashFileName(base, NULL, NULL); - int fd = open(blockdev_filename->data, O_RDWR); - if (fd < 0) { - ErrorAbort(state, "failed to open %s: %s", blockdev_filename->data, strerror(errno)); - goto done; + if (dirname == NULL) { + return; } - char* line; - char* word; + fprintf(stderr, "deleting stash %s\n", base); + EnumerateStash(dirname, DeleteFile, NULL); - // The data in transfer_list_value is not necessarily - // null-terminated, so we need to copy it to a new buffer and add - // the null that strtok_r will need. - transfer_list = malloc(transfer_list_value->size+1); - if (transfer_list == NULL) { - fprintf(stderr, "failed to allocate %zd bytes for transfer list\n", - transfer_list_value->size+1); - exit(1); + if (rmdir(dirname) == -1) { + if (errno != ENOENT && errno != ENOTDIR) { + fprintf(stderr, "failed to rmdir %s (errno %d)\n", dirname, errno); + } } - memcpy(transfer_list, transfer_list_value->data, transfer_list_value->size); - transfer_list[transfer_list_value->size] = '\0'; - line = strtok_r(transfer_list, "\n", &linesave); + free(dirname); +} - int version; - // first line in transfer list is the version number; currently - // there's only version 1. - if (strcmp(line, "1") == 0) { - version = 1; - } else if (strcmp(line, "2") == 0) { - version = 2; - } else { - ErrorAbort(state, "unexpected transfer list version [%s]\n", line); - goto done; +static int LoadStash(const char* base, const char* id, int verify, int* blocks, uint8_t** buffer, + size_t* buffer_alloc, int printnoent) { + char *fn = NULL; + int blockcount = 0; + int fd = -1; + int rc = -1; + int res; + struct stat st; + + if (!base || !id || !buffer || !buffer_alloc) { + goto lsout; } - printf("blockimg version is %d\n", version); - // second line in transfer list is the total number of blocks we - // expect to write. - line = strtok_r(NULL, "\n", &linesave); - int total_blocks = strtol(line, NULL, 0); - // shouldn't happen, but avoid divide by zero. - if (total_blocks == 0) ++total_blocks; - int blocks_so_far = 0; - - uint8_t** stash_table = NULL; - if (version >= 2) { - // Next line is how many stash entries are needed simultaneously. - line = strtok_r(NULL, "\n", &linesave); - int stash_entries = strtol(line, NULL, 0); + if (!blocks) { + blocks = &blockcount; + } - stash_table = (uint8_t**) calloc(stash_entries, sizeof(uint8_t*)); - if (stash_table == NULL) { - fprintf(stderr, "failed to allocate %d-entry stash table\n", stash_entries); - exit(1); - } + fn = GetStashFileName(base, id, NULL); - // Next line is the maximum number of blocks that will be - // stashed simultaneously. This could be used to verify that - // enough memory or scratch disk space is available. - line = strtok_r(NULL, "\n", &linesave); - int stash_max_blocks = strtol(line, NULL, 0); + if (fn == NULL) { + goto lsout; } - uint8_t* buffer = NULL; - size_t buffer_alloc = 0; + res = stat(fn, &st); - // third and subsequent lines are all individual transfer commands. - for (line = strtok_r(NULL, "\n", &linesave); line; - line = strtok_r(NULL, "\n", &linesave)) { + if (res == -1) { + if (errno != ENOENT || printnoent) { + fprintf(stderr, "failed to stat %s (errno %d)\n", fn, errno); + } + goto lsout; + } - char* style; - style = strtok_r(line, " ", &wordsave); - - if (strcmp("move", style) == 0) { - RangeSet* tgt; - int src_blocks; - if (version == 1) { - LoadSrcTgtVersion1(wordsave, &tgt, &src_blocks, - &buffer, &buffer_alloc, fd); - } else if (version == 2) { - LoadSrcTgtVersion2(wordsave, &tgt, &src_blocks, - &buffer, &buffer_alloc, fd, stash_table); - } + fprintf(stderr, " loading %s\n", fn); - printf(" moving %d blocks\n", src_blocks); + if ((st.st_size % BLOCKSIZE) != 0) { + fprintf(stderr, "%s size %zd not multiple of block size %d", fn, st.st_size, BLOCKSIZE); + goto lsout; + } - size_t p = 0; - for (i = 0; i < tgt->count; ++i) { - check_lseek(fd, (off64_t)tgt->pos[i*2] * BLOCKSIZE, SEEK_SET); - size_t sz = (tgt->pos[i*2+1] - tgt->pos[i*2]) * BLOCKSIZE; - writeblock(fd, buffer+p, sz); - p += sz; - } + fd = TEMP_FAILURE_RETRY(open(fn, O_RDONLY)); - blocks_so_far += tgt->size; - fprintf(cmd_pipe, "set_progress %.4f\n", (double)blocks_so_far / total_blocks); - fflush(cmd_pipe); + if (fd == -1) { + fprintf(stderr, "failed to open %s (errno %d)\n", fn, errno); + goto lsout; + } - free(tgt); - - } else if (strcmp("stash", style) == 0) { - word = strtok_r(NULL, " ", &wordsave); - int stash_id = strtol(word, NULL, 0); - int src_blocks; - size_t stash_alloc = 0; - - // Even though the "stash" style only appears in version - // 2, the version 1 source loader happens to do exactly - // what we want to read data into the stash_table. - LoadSrcTgtVersion1(wordsave, NULL, &src_blocks, - stash_table + stash_id, &stash_alloc, fd); - - } else if (strcmp("zero", style) == 0 || - (DEBUG_ERASE && strcmp("erase", style) == 0)) { - word = strtok_r(NULL, " ", &wordsave); - RangeSet* tgt = parse_range(word); - - printf(" zeroing %d blocks\n", tgt->size); - - allocate(BLOCKSIZE, &buffer, &buffer_alloc); - memset(buffer, 0, BLOCKSIZE); - for (i = 0; i < tgt->count; ++i) { - check_lseek(fd, (off64_t)tgt->pos[i*2] * BLOCKSIZE, SEEK_SET); - for (j = tgt->pos[i*2]; j < tgt->pos[i*2+1]; ++j) { - writeblock(fd, buffer, BLOCKSIZE); - } - } + allocate(st.st_size, buffer, buffer_alloc); - if (style[0] == 'z') { // "zero" but not "erase" - blocks_so_far += tgt->size; - fprintf(cmd_pipe, "set_progress %.4f\n", (double)blocks_so_far / total_blocks); - fflush(cmd_pipe); - } + if (read_all(fd, *buffer, st.st_size) == -1) { + goto lsout; + } - free(tgt); - } else if (strcmp("new", style) == 0) { + *blocks = st.st_size / BLOCKSIZE; - word = strtok_r(NULL, " ", &wordsave); - RangeSet* tgt = parse_range(word); + if (verify && VerifyBlocks(id, *buffer, *blocks, 1) != 0) { + fprintf(stderr, "unexpected contents in %s\n", fn); + DeleteFile(fn, NULL); + goto lsout; + } - printf(" writing %d blocks of new data\n", tgt->size); + rc = 0; - RangeSinkState rss; - rss.fd = fd; - rss.tgt = tgt; - rss.p_block = 0; - rss.p_remain = (tgt->pos[1] - tgt->pos[0]) * BLOCKSIZE; - check_lseek(fd, (off64_t)tgt->pos[0] * BLOCKSIZE, SEEK_SET); +lsout: + if (fd != -1) { + TEMP_FAILURE_RETRY(close(fd)); + } - pthread_mutex_lock(&nti.mu); - nti.rss = &rss; - pthread_cond_broadcast(&nti.cv); - while (nti.rss) { - pthread_cond_wait(&nti.cv, &nti.mu); - } - pthread_mutex_unlock(&nti.mu); + if (fn) { + free(fn); + } - blocks_so_far += tgt->size; - fprintf(cmd_pipe, "set_progress %.4f\n", (double)blocks_so_far / total_blocks); - fflush(cmd_pipe); + return rc; +} - free(tgt); - - } else if (strcmp("bsdiff", style) == 0 || - strcmp("imgdiff", style) == 0) { - word = strtok_r(NULL, " ", &wordsave); - size_t patch_offset = strtoul(word, NULL, 0); - word = strtok_r(NULL, " ", &wordsave); - size_t patch_len = strtoul(word, NULL, 0); - - RangeSet* tgt; - int src_blocks; - if (version == 1) { - LoadSrcTgtVersion1(wordsave, &tgt, &src_blocks, - &buffer, &buffer_alloc, fd); - } else if (version == 2) { - LoadSrcTgtVersion2(wordsave, &tgt, &src_blocks, - &buffer, &buffer_alloc, fd, stash_table); - } +static int WriteStash(const char* base, const char* id, int blocks, uint8_t* buffer, + int checkspace) { + char *fn = NULL; + char *cn = NULL; + int fd = -1; + int rc = -1; + int res; - printf(" patching %d blocks to %d\n", src_blocks, tgt->size); + if (base == NULL || buffer == NULL) { + goto wsout; + } - Value patch_value; - patch_value.type = VAL_BLOB; - patch_value.size = patch_len; - patch_value.data = (char*)(patch_start + patch_offset); + if (checkspace && CacheSizeCheck(blocks * BLOCKSIZE) != 0) { + fprintf(stderr, "not enough space to write stash\n"); + goto wsout; + } - RangeSinkState rss; - rss.fd = fd; - rss.tgt = tgt; - rss.p_block = 0; - rss.p_remain = (tgt->pos[1] - tgt->pos[0]) * BLOCKSIZE; - check_lseek(fd, (off64_t)tgt->pos[0] * BLOCKSIZE, SEEK_SET); + fn = GetStashFileName(base, id, ".partial"); + cn = GetStashFileName(base, id, NULL); - if (style[0] == 'i') { // imgdiff - ApplyImagePatch(buffer, src_blocks * BLOCKSIZE, - &patch_value, - &RangeSinkWrite, &rss, NULL, NULL); - } else { - ApplyBSDiffPatch(buffer, src_blocks * BLOCKSIZE, - &patch_value, 0, - &RangeSinkWrite, &rss, NULL); - } + if (fn == NULL || cn == NULL) { + goto wsout; + } - // We expect the output of the patcher to fill the tgt ranges exactly. - if (rss.p_block != tgt->count || rss.p_remain != 0) { - fprintf(stderr, "range sink underrun?\n"); - } + fprintf(stderr, " writing %d blocks to %s\n", blocks, cn); - blocks_so_far += tgt->size; - fprintf(cmd_pipe, "set_progress %.4f\n", (double)blocks_so_far / total_blocks); - fflush(cmd_pipe); + fd = TEMP_FAILURE_RETRY(open(fn, O_WRONLY | O_CREAT | O_TRUNC | O_SYNC, STASH_FILE_MODE)); - free(tgt); - } else if (!DEBUG_ERASE && strcmp("erase", style) == 0) { - struct stat st; - if (fstat(fd, &st) == 0 && S_ISBLK(st.st_mode)) { - word = strtok_r(NULL, " ", &wordsave); - RangeSet* tgt = parse_range(word); - - printf(" erasing %d blocks\n", tgt->size); - - for (i = 0; i < tgt->count; ++i) { - uint64_t range[2]; - // offset in bytes - range[0] = tgt->pos[i*2] * (uint64_t)BLOCKSIZE; - // len in bytes - range[1] = (tgt->pos[i*2+1] - tgt->pos[i*2]) * (uint64_t)BLOCKSIZE; - - if (ioctl(fd, BLKDISCARD, &range) < 0) { - printf(" blkdiscard failed: %s\n", strerror(errno)); - } - } + if (fd == -1) { + fprintf(stderr, "failed to create %s (errno %d)\n", fn, errno); + goto wsout; + } - free(tgt); - } else { - printf(" ignoring erase (not block device)\n"); - } - } else { - fprintf(stderr, "unknown transfer style \"%s\"\n", style); - exit(1); - } + if (write_all(fd, buffer, blocks * BLOCKSIZE) == -1) { + goto wsout; } - pthread_join(new_data_thread, NULL); - success = true; + if (fsync(fd) == -1) { + fprintf(stderr, "failed to fsync %s (errno %d)\n", fn, errno); + goto wsout; + } - free(buffer); - printf("wrote %d blocks; expected %d\n", blocks_so_far, total_blocks); - printf("max alloc needed was %zu\n", buffer_alloc); + if (rename(fn, cn) == -1) { + fprintf(stderr, "failed to rename %s to %s (errno %d)\n", fn, cn, errno); + goto wsout; + } - done: - free(transfer_list); - FreeValue(blockdev_filename); - FreeValue(transfer_list_value); - FreeValue(new_data_fn); - FreeValue(patch_data_fn); - return StringValue(success ? strdup("t") : strdup("")); -} + rc = 0; -Value* RangeSha1Fn(const char* name, State* state, int argc, Expr* argv[]) { - Value* blockdev_filename; - Value* ranges; - const uint8_t* digest = NULL; - if (ReadValueArgs(state, argv, 2, &blockdev_filename, &ranges) < 0) { - return NULL; +wsout: + if (fd != -1) { + TEMP_FAILURE_RETRY(close(fd)); } - if (blockdev_filename->type != VAL_STRING) { - ErrorAbort(state, "blockdev_filename argument to %s must be string", name); - goto done; + if (fn) { + free(fn); } - if (ranges->type != VAL_STRING) { - ErrorAbort(state, "ranges argument to %s must be string", name); - goto done; + + if (cn) { + free(cn); } - int fd = open(blockdev_filename->data, O_RDWR); + return rc; +} + +// Creates a directory for storing stash files and checks if the /cache partition +// hash enough space for the expected amount of blocks we need to store. Returns +// >0 if we created the directory, zero if it existed already, and <0 of failure. + +static int CreateStash(State* state, int maxblocks, const char* blockdev, char** base) { + char* dirname = NULL; + const uint8_t* digest; + int rc = -1; + int res; + int size = 0; + SHA_CTX ctx; + struct stat st; + + if (blockdev == NULL || base == NULL) { + goto csout; + } + + // Stash directory should be different for each partition to avoid conflicts + // when updating multiple partitions at the same time, so we use the hash of + // the block device name as the base directory + SHA_init(&ctx); + SHA_update(&ctx, blockdev, strlen(blockdev)); + digest = SHA_final(&ctx); + *base = PrintSha1(digest); + + if (*base == NULL) { + goto csout; + } + + dirname = GetStashFileName(*base, NULL, NULL); + + if (dirname == NULL) { + goto csout; + } + + res = stat(dirname, &st); + + if (res == -1 && errno != ENOENT) { + ErrorAbort(state, "failed to stat %s (errno %d)\n", dirname, errno); + goto csout; + } else if (res != 0) { + fprintf(stderr, "creating stash %s\n", dirname); + res = mkdir(dirname, STASH_DIRECTORY_MODE); + + if (res != 0) { + ErrorAbort(state, "failed to create %s (errno %d)\n", dirname, errno); + goto csout; + } + + if (CacheSizeCheck(maxblocks * BLOCKSIZE) != 0) { + ErrorAbort(state, "not enough space for stash\n"); + goto csout; + } + + rc = 1; // Created directory + goto csout; + } + + fprintf(stderr, "using existing stash %s\n", dirname); + + // If the directory already exists, calculate the space already allocated to + // stash files and check if there's enough for all required blocks. Delete any + // partially completed stash files first. + + EnumerateStash(dirname, DeletePartial, NULL); + EnumerateStash(dirname, UpdateFileSize, &size); + + size = (maxblocks * BLOCKSIZE) - size; + + if (size > 0 && CacheSizeCheck(size) != 0) { + ErrorAbort(state, "not enough space for stash (%d more needed)\n", size); + goto csout; + } + + rc = 0; // Using existing directory + +csout: + if (dirname) { + free(dirname); + } + + return rc; +} + +static int SaveStash(const char* base, char** wordsave, uint8_t** buffer, size_t* buffer_alloc, + int fd, int usehash, int* isunresumable) { + char *id = NULL; + int res = -1; + int blocks = 0; + + if (!wordsave || !buffer || !buffer_alloc || !isunresumable) { + return -1; + } + + id = strtok_r(NULL, " ", wordsave); + + if (id == NULL) { + fprintf(stderr, "missing id field in stash command\n"); + return -1; + } + + if (usehash && LoadStash(base, id, 1, &blocks, buffer, buffer_alloc, 0) == 0) { + // Stash file already exists and has expected contents. Do not + // read from source again, as the source may have been already + // overwritten during a previous attempt. + return 0; + } + + if (LoadSrcTgtVersion1(wordsave, NULL, &blocks, buffer, buffer_alloc, fd) == -1) { + return -1; + } + + if (usehash && VerifyBlocks(id, *buffer, blocks, 1) != 0) { + // Source blocks have unexpected contents. If we actually need this + // data later, this is an unrecoverable error. However, the command + // that uses the data may have already completed previously, so the + // possible failure will occur during source block verification. + fprintf(stderr, "failed to load source blocks for stash %s\n", id); + return 0; + } + + fprintf(stderr, "stashing %d blocks to %s\n", blocks, id); + return WriteStash(base, id, blocks, *buffer, 0); +} + +static int FreeStash(const char* base, const char* id) { + char *fn = NULL; + + if (base == NULL || id == NULL) { + return -1; + } + + fn = GetStashFileName(base, id, NULL); + + if (fn == NULL) { + return -1; + } + + DeleteFile(fn, NULL); + free(fn); + + return 0; +} + +static void MoveRange(uint8_t* dest, RangeSet* locs, const uint8_t* source) { + // source contains packed data, which we want to move to the + // locations given in *locs in the dest buffer. source and dest + // may be the same buffer. + + int start = locs->size; + int i; + for (i = locs->count-1; i >= 0; --i) { + int blocks = locs->pos[i*2+1] - locs->pos[i*2]; + start -= blocks; + memmove(dest + (locs->pos[i*2] * BLOCKSIZE), source + (start * BLOCKSIZE), + blocks * BLOCKSIZE); + } +} + +// Do a source/target load for move/bsdiff/imgdiff in version 2. +// 'wordsave' is the save_ptr of a strtok_r()-in-progress. We expect +// to parse the remainder of the string as one of: +// +// +// (loads data from source image only) +// +// - <[stash_id:stash_range] ...> +// (loads data from stashes only) +// +// <[stash_id:stash_range] ...> +// (loads data from both source image and stashes) +// +// On return, buffer is filled with the loaded source data (rearranged +// and combined with stashed data as necessary). buffer may be +// reallocated if needed to accommodate the source data. *tgt is the +// target RangeSet. Any stashes required are loaded using LoadStash. + +static int LoadSrcTgtVersion2(char** wordsave, RangeSet** tgt, int* src_blocks, + uint8_t** buffer, size_t* buffer_alloc, int fd, + const char* stashbase, int* overlap) { + char* word; + char* colonsave; + char* colon; + int id; + int res; + RangeSet* locs; + size_t stashalloc = 0; + uint8_t* stash = NULL; + + if (tgt != NULL) { + word = strtok_r(NULL, " ", wordsave); + *tgt = parse_range(word); + } + + word = strtok_r(NULL, " ", wordsave); + *src_blocks = strtol(word, NULL, 0); + + allocate(*src_blocks * BLOCKSIZE, buffer, buffer_alloc); + + word = strtok_r(NULL, " ", wordsave); + if (word[0] == '-' && word[1] == '\0') { + // no source ranges, only stashes + } else { + RangeSet* src = parse_range(word); + res = ReadBlocks(src, *buffer, fd); + + if (overlap && tgt) { + *overlap = range_overlaps(src, *tgt); + } + + free(src); + + if (res == -1) { + return -1; + } + + word = strtok_r(NULL, " ", wordsave); + if (word == NULL) { + // no stashes, only source range + return 0; + } + + locs = parse_range(word); + MoveRange(*buffer, locs, *buffer); + free(locs); + } + + while ((word = strtok_r(NULL, " ", wordsave)) != NULL) { + // Each word is a an index into the stash table, a colon, and + // then a rangeset describing where in the source block that + // stashed data should go. + colonsave = NULL; + colon = strtok_r(word, ":", &colonsave); + + res = LoadStash(stashbase, colon, 0, NULL, &stash, &stashalloc, 1); + + if (res == -1) { + // These source blocks will fail verification if used later, but we + // will let the caller decide if this is a fatal failure + fprintf(stderr, "failed to load stash %s\n", colon); + continue; + } + + colon = strtok_r(NULL, ":", &colonsave); + locs = parse_range(colon); + + MoveRange(*buffer, locs, stash); + free(locs); + } + + if (stash) { + free(stash); + } + + return 0; +} + +// Parameters for transfer list command functions +typedef struct { + char* cmdname; + char* cpos; + char* freestash; + char* stashbase; + int canwrite; + int createdstash; + int fd; + int foundwrites; + int isunresumable; + int version; + int written; + NewThreadInfo nti; + pthread_t thread; + size_t bufsize; + uint8_t* buffer; + uint8_t* patch_start; +} CommandParameters; + +// Do a source/target load for move/bsdiff/imgdiff in version 3. +// +// Parameters are the same as for LoadSrcTgtVersion2, except for 'onehash', which +// tells the function whether to expect separate source and targe block hashes, or +// if they are both the same and only one hash should be expected, and +// 'isunresumable', which receives a non-zero value if block verification fails in +// a way that the update cannot be resumed anymore. +// +// If the function is unable to load the necessary blocks or their contents don't +// match the hashes, the return value is -1 and the command should be aborted. +// +// If the return value is 1, the command has already been completed according to +// the contents of the target blocks, and should not be performed again. +// +// If the return value is 0, source blocks have expected content and the command +// can be performed. + +static int LoadSrcTgtVersion3(CommandParameters* params, RangeSet** tgt, int* src_blocks, + int onehash, int* overlap) { + char* srchash = NULL; + char* tgthash = NULL; + int overlap_blocks = 0; + int rc = -1; + uint8_t* tgtbuffer = NULL; + + if (!params|| !tgt || !src_blocks || !overlap) { + goto v3out; + } + + srchash = strtok_r(NULL, " ", ¶ms->cpos); + + if (srchash == NULL) { + fprintf(stderr, "missing source hash\n"); + goto v3out; + } + + if (onehash) { + tgthash = srchash; + } else { + tgthash = strtok_r(NULL, " ", ¶ms->cpos); + + if (tgthash == NULL) { + fprintf(stderr, "missing target hash\n"); + goto v3out; + } + } + + if (LoadSrcTgtVersion2(¶ms->cpos, tgt, src_blocks, ¶ms->buffer, ¶ms->bufsize, + params->fd, params->stashbase, overlap) == -1) { + goto v3out; + } + + tgtbuffer = (uint8_t*) malloc((*tgt)->size * BLOCKSIZE); + + if (tgtbuffer == NULL) { + fprintf(stderr, "failed to allocate %d bytes\n", (*tgt)->size * BLOCKSIZE); + goto v3out; + } + + if (ReadBlocks(*tgt, tgtbuffer, params->fd) == -1) { + goto v3out; + } + + if (VerifyBlocks(tgthash, tgtbuffer, (*tgt)->size, 0) == 0) { + // Target blocks already have expected content, command should be skipped + rc = 1; + goto v3out; + } + + if (VerifyBlocks(srchash, params->buffer, *src_blocks, 1) == 0) { + // If source and target blocks overlap, stash the source blocks so we can + // resume from possible write errors + if (*overlap) { + fprintf(stderr, "stashing %d overlapping blocks to %s\n", *src_blocks, + srchash); + + if (WriteStash(params->stashbase, srchash, *src_blocks, params->buffer, 1) != 0) { + fprintf(stderr, "failed to stash overlapping source blocks\n"); + goto v3out; + } + + // Can be deleted when the write has completed + params->freestash = srchash; + } + + // Source blocks have expected content, command can proceed + rc = 0; + goto v3out; + } + + if (*overlap && LoadStash(params->stashbase, srchash, 1, NULL, ¶ms->buffer, + ¶ms->bufsize, 1) == 0) { + // Overlapping source blocks were previously stashed, command can proceed + if (params->canwrite) { + // We didn't create the stash, so delete after write only if we will + // actually perform the write + params->freestash = srchash; + } + rc = 0; + goto v3out; + } + + // Valid source data not available, update cannot be resumed + fprintf(stderr, "partition has unexpected contents\n"); + params->isunresumable = 1; + +v3out: + if (tgtbuffer) { + free(tgtbuffer); + } + + return rc; +} + +static int PerformCommandMove(CommandParameters* params) { + int blocks = 0; + int overlap = 0; + int rc = -1; + int status = 0; + RangeSet* tgt = NULL; + + if (!params) { + goto pcmout; + } + + if (params->version == 1) { + status = LoadSrcTgtVersion1(¶ms->cpos, &tgt, &blocks, ¶ms->buffer, + ¶ms->bufsize, params->fd); + } else if (params->version == 2) { + status = LoadSrcTgtVersion2(¶ms->cpos, &tgt, &blocks, ¶ms->buffer, + ¶ms->bufsize, params->fd, params->stashbase, NULL); + } else if (params->version >= 3) { + status = LoadSrcTgtVersion3(params, &tgt, &blocks, 1, &overlap); + } + + if (status == -1) { + fprintf(stderr, "failed to read blocks for move\n"); + goto pcmout; + } + + if (status == 0) { + params->foundwrites = 1; + } else if (params->foundwrites) { + fprintf(stderr, "warning: commands executed out of order [%s]\n", params->cmdname); + } + + if (params->canwrite) { + if (status == 0) { + fprintf(stderr, " moving %d blocks\n", blocks); + + if (WriteBlocks(tgt, params->buffer, params->fd) == -1) { + goto pcmout; + } + } else { + fprintf(stderr, "skipping %d already moved blocks\n", blocks); + } + + } + + if (params->freestash) { + FreeStash(params->stashbase, params->freestash); + params->freestash = NULL; + } + + params->written += tgt->size; + rc = 0; + +pcmout: + if (tgt) { + free(tgt); + } + + return rc; +} + +static int PerformCommandStash(CommandParameters* params) { + if (!params) { + return -1; + } + + return SaveStash(params->stashbase, ¶ms->cpos, ¶ms->buffer, ¶ms->bufsize, + params->fd, (params->version >= 3), ¶ms->isunresumable); +} + +static int PerformCommandFree(CommandParameters* params) { + if (!params) { + return -1; + } + + if (params->createdstash || params->canwrite) { + return FreeStash(params->stashbase, params->cpos); + } + + return 0; +} + +static int PerformCommandZero(CommandParameters* params) { + char* range = NULL; + int i; + int j; + int rc = -1; + RangeSet* tgt = NULL; + + if (!params) { + goto pczout; + } + + range = strtok_r(NULL, " ", ¶ms->cpos); + + if (range == NULL) { + fprintf(stderr, "missing target blocks for zero\n"); + goto pczout; + } + + tgt = parse_range(range); + + fprintf(stderr, " zeroing %d blocks\n", tgt->size); + + allocate(BLOCKSIZE, ¶ms->buffer, ¶ms->bufsize); + memset(params->buffer, 0, BLOCKSIZE); + + if (params->canwrite) { + for (i = 0; i < tgt->count; ++i) { + if (check_lseek(params->fd, (off64_t) tgt->pos[i * 2] * BLOCKSIZE, SEEK_SET) == -1) { + goto pczout; + } + + for (j = tgt->pos[i * 2]; j < tgt->pos[i * 2 + 1]; ++j) { + if (write_all(params->fd, params->buffer, BLOCKSIZE) == -1) { + goto pczout; + } + } + } + } + + if (params->cmdname[0] == 'z') { + // Update only for the zero command, as the erase command will call + // this if DEBUG_ERASE is defined. + params->written += tgt->size; + } + + rc = 0; + +pczout: + if (tgt) { + free(tgt); + } + + return rc; +} + +static int PerformCommandNew(CommandParameters* params) { + char* range = NULL; + int rc = -1; + RangeSet* tgt = NULL; + RangeSinkState rss; + + if (!params) { + goto pcnout; + } + + range = strtok_r(NULL, " ", ¶ms->cpos); + + if (range == NULL) { + goto pcnout; + } + + tgt = parse_range(range); + + if (params->canwrite) { + fprintf(stderr, " writing %d blocks of new data\n", tgt->size); + + rss.fd = params->fd; + rss.tgt = tgt; + rss.p_block = 0; + rss.p_remain = (tgt->pos[1] - tgt->pos[0]) * BLOCKSIZE; + + if (check_lseek(params->fd, (off64_t) tgt->pos[0] * BLOCKSIZE, SEEK_SET) == -1) { + goto pcnout; + } + + pthread_mutex_lock(¶ms->nti.mu); + params->nti.rss = &rss; + pthread_cond_broadcast(¶ms->nti.cv); + + while (params->nti.rss) { + pthread_cond_wait(¶ms->nti.cv, ¶ms->nti.mu); + } + + pthread_mutex_unlock(¶ms->nti.mu); + } + + params->written += tgt->size; + rc = 0; + +pcnout: + if (tgt) { + free(tgt); + } + + return rc; +} + +static int PerformCommandDiff(CommandParameters* params) { + char* logparams = NULL; + char* value = NULL; + int blocks = 0; + int overlap = 0; + int rc = -1; + int status = 0; + RangeSet* tgt = NULL; + RangeSinkState rss; + size_t len = 0; + size_t offset = 0; + Value patch_value; + + if (!params) { + goto pcdout; + } + + logparams = strdup(params->cpos); + value = strtok_r(NULL, " ", ¶ms->cpos); + + if (value == NULL) { + fprintf(stderr, "missing patch offset for %s\n", params->cmdname); + goto pcdout; + } + + offset = strtoul(value, NULL, 0); + + value = strtok_r(NULL, " ", ¶ms->cpos); + + if (value == NULL) { + fprintf(stderr, "missing patch length for %s\n", params->cmdname); + goto pcdout; + } + + len = strtoul(value, NULL, 0); + + if (params->version == 1) { + status = LoadSrcTgtVersion1(¶ms->cpos, &tgt, &blocks, ¶ms->buffer, + ¶ms->bufsize, params->fd); + } else if (params->version == 2) { + status = LoadSrcTgtVersion2(¶ms->cpos, &tgt, &blocks, ¶ms->buffer, + ¶ms->bufsize, params->fd, params->stashbase, NULL); + } else if (params->version >= 3) { + status = LoadSrcTgtVersion3(params, &tgt, &blocks, 0, &overlap); + } + + if (status == -1) { + fprintf(stderr, "failed to read blocks for diff\n"); + goto pcdout; + } + + if (status == 0) { + params->foundwrites = 1; + } else if (params->foundwrites) { + fprintf(stderr, "warning: commands executed out of order [%s]\n", params->cmdname); + } + + if (params->canwrite) { + if (status == 0) { + fprintf(stderr, "patching %d blocks to %d\n", blocks, tgt->size); + + patch_value.type = VAL_BLOB; + patch_value.size = len; + patch_value.data = (char*) (params->patch_start + offset); + + rss.fd = params->fd; + rss.tgt = tgt; + rss.p_block = 0; + rss.p_remain = (tgt->pos[1] - tgt->pos[0]) * BLOCKSIZE; + + if (check_lseek(params->fd, (off64_t) tgt->pos[0] * BLOCKSIZE, SEEK_SET) == -1) { + goto pcdout; + } + + if (params->cmdname[0] == 'i') { // imgdiff + ApplyImagePatch(params->buffer, blocks * BLOCKSIZE, &patch_value, + &RangeSinkWrite, &rss, NULL, NULL); + } else { + ApplyBSDiffPatch(params->buffer, blocks * BLOCKSIZE, &patch_value, + 0, &RangeSinkWrite, &rss, NULL); + } + + // We expect the output of the patcher to fill the tgt ranges exactly. + if (rss.p_block != tgt->count || rss.p_remain != 0) { + fprintf(stderr, "range sink underrun?\n"); + } + } else { + fprintf(stderr, "skipping %d blocks already patched to %d [%s]\n", + blocks, tgt->size, logparams); + } + } + + if (params->freestash) { + FreeStash(params->stashbase, params->freestash); + params->freestash = NULL; + } + + params->written += tgt->size; + rc = 0; + +pcdout: + if (logparams) { + free(logparams); + } + + if (tgt) { + free(tgt); + } + + return rc; +} + +static int PerformCommandErase(CommandParameters* params) { + char* range = NULL; + int i; + int rc = -1; + RangeSet* tgt = NULL; + struct stat st; + uint64_t blocks[2]; + + if (DEBUG_ERASE) { + return PerformCommandZero(params); + } + + if (!params) { + goto pceout; + } + + if (fstat(params->fd, &st) == -1) { + fprintf(stderr, "failed to fstat device to erase (errno %d)\n", errno); + goto pceout; + } + + if (!S_ISBLK(st.st_mode)) { + fprintf(stderr, "not a block device; skipping erase\n"); + rc = 0; + goto pceout; + } + + range = strtok_r(NULL, " ", ¶ms->cpos); + + if (range == NULL) { + fprintf(stderr, "missing target blocks for zero\n"); + goto pceout; + } + + tgt = parse_range(range); + + if (params->canwrite) { + fprintf(stderr, " erasing %d blocks\n", tgt->size); + + for (i = 0; i < tgt->count; ++i) { + // offset in bytes + blocks[0] = tgt->pos[i * 2] * (uint64_t) BLOCKSIZE; + // length in bytes + blocks[1] = (tgt->pos[i * 2 + 1] - tgt->pos[i * 2]) * (uint64_t) BLOCKSIZE; + + if (ioctl(params->fd, BLKDISCARD, &blocks) == -1) { + fprintf(stderr, "failed to blkdiscard (errno %d)\n", errno); + // Continue anyway, nothing we can do + } + } + } + + rc = 0; + +pceout: + if (tgt) { + free(tgt); + } + + return rc; +} + +// Definitions for transfer list command functions +typedef int (*CommandFunction)(CommandParameters*); + +typedef struct { + const char* name; + CommandFunction f; +} Command; + +// CompareCommands and CompareCommandNames are for the hash table + +static int CompareCommands(const void* c1, const void* c2) { + return strcmp(((const Command*) c1)->name, ((const Command*) c2)->name); +} + +static int CompareCommandNames(const void* c1, const void* c2) { + return strcmp(((const Command*) c1)->name, (const char*) c2); +} + +// HashString is used to hash command names for the hash table + +static unsigned int HashString(const char *s) { + unsigned int hash = 0; + if (s) { + while (*s) { + hash = hash * 33 + *s++; + } + } + return hash; +} + +// args: +// - block device (or file) to modify in-place +// - transfer list (blob) +// - new data stream (filename within package.zip) +// - patch stream (filename within package.zip, must be uncompressed) + +static Value* PerformBlockImageUpdate(const char* name, State* state, int argc, Expr* argv[], + const Command* commands, int cmdcount, int dryrun) { + + char* line = NULL; + char* linesave = NULL; + char* logcmd = NULL; + char* transfer_list = NULL; + CommandParameters params; + const Command* cmd = NULL; + const ZipEntry* new_entry = NULL; + const ZipEntry* patch_entry = NULL; + FILE* cmd_pipe = NULL; + HashTable* cmdht = NULL; + int i; + int res; + int rc = -1; + int stash_max_blocks = 0; + int total_blocks = 0; + pthread_attr_t attr; + unsigned int cmdhash; + UpdaterInfo* ui = NULL; + Value* blockdev_filename = NULL; + Value* new_data_fn = NULL; + Value* patch_data_fn = NULL; + Value* transfer_list_value = NULL; + ZipArchive* za = NULL; + + memset(¶ms, 0, sizeof(params)); + params.canwrite = !dryrun; + + fprintf(stderr, "performing %s\n", dryrun ? "verification" : "update"); + + if (ReadValueArgs(state, argv, 4, &blockdev_filename, &transfer_list_value, + &new_data_fn, &patch_data_fn) < 0) { + goto pbiudone; + } + + if (blockdev_filename->type != VAL_STRING) { + ErrorAbort(state, "blockdev_filename argument to %s must be string", name); + goto pbiudone; + } + if (transfer_list_value->type != VAL_BLOB) { + ErrorAbort(state, "transfer_list argument to %s must be blob", name); + goto pbiudone; + } + if (new_data_fn->type != VAL_STRING) { + ErrorAbort(state, "new_data_fn argument to %s must be string", name); + goto pbiudone; + } + if (patch_data_fn->type != VAL_STRING) { + ErrorAbort(state, "patch_data_fn argument to %s must be string", name); + goto pbiudone; + } + + ui = (UpdaterInfo*) state->cookie; + + if (ui == NULL) { + goto pbiudone; + } + + cmd_pipe = ui->cmd_pipe; + za = ui->package_zip; + + if (cmd_pipe == NULL || za == NULL) { + goto pbiudone; + } + + patch_entry = mzFindZipEntry(za, patch_data_fn->data); + + if (patch_entry == NULL) { + fprintf(stderr, "%s(): no file \"%s\" in package", name, patch_data_fn->data); + goto pbiudone; + } + + params.patch_start = ui->package_zip_addr + mzGetZipEntryOffset(patch_entry); + new_entry = mzFindZipEntry(za, new_data_fn->data); + + if (new_entry == NULL) { + fprintf(stderr, "%s(): no file \"%s\" in package", name, new_data_fn->data); + goto pbiudone; + } + + params.fd = TEMP_FAILURE_RETRY(open(blockdev_filename->data, O_RDWR)); + + if (params.fd == -1) { + fprintf(stderr, "failed to open %s: %s", blockdev_filename->data, strerror(errno)); + goto pbiudone; + } + + if (params.canwrite) { + params.nti.za = za; + params.nti.entry = new_entry; + + pthread_mutex_init(¶ms.nti.mu, NULL); + pthread_cond_init(¶ms.nti.cv, NULL); + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(¶ms.thread, &attr, unzip_new_data, ¶ms.nti) != 0) { + fprintf(stderr, "failed to create a thread (errno %d)\n", errno); + goto pbiudone; + } + } + + // The data in transfer_list_value is not necessarily null-terminated, so we need + // to copy it to a new buffer and add the null that strtok_r will need. + transfer_list = malloc(transfer_list_value->size + 1); + + if (transfer_list == NULL) { + fprintf(stderr, "failed to allocate %zd bytes for transfer list\n", + transfer_list_value->size + 1); + goto pbiudone; + } + + memcpy(transfer_list, transfer_list_value->data, transfer_list_value->size); + transfer_list[transfer_list_value->size] = '\0'; + + // First line in transfer list is the version number + line = strtok_r(transfer_list, "\n", &linesave); + params.version = strtol(line, NULL, 0); + + if (params.version < 1 || params.version > 3) { + fprintf(stderr, "unexpected transfer list version [%s]\n", line); + goto pbiudone; + } + + fprintf(stderr, "blockimg version is %d\n", params.version); + + // Second line in transfer list is the total number of blocks we expect to write + line = strtok_r(NULL, "\n", &linesave); + total_blocks = strtol(line, NULL, 0); + + if (total_blocks < 0) { + ErrorAbort(state, "unexpected block count [%s]\n", line); + goto pbiudone; + } else if (total_blocks == 0) { + rc = 0; + goto pbiudone; + } + + if (params.version >= 2) { + // Third line is how many stash entries are needed simultaneously + line = strtok_r(NULL, "\n", &linesave); + fprintf(stderr, "maximum stash entries %s\n", line); + + // Fourth line is the maximum number of blocks that will be stashed simultaneously + line = strtok_r(NULL, "\n", &linesave); + stash_max_blocks = strtol(line, NULL, 0); + + if (stash_max_blocks < 0) { + ErrorAbort(state, "unexpected maximum stash blocks [%s]\n", line); + goto pbiudone; + } + + if (stash_max_blocks > 0) { + res = CreateStash(state, stash_max_blocks, blockdev_filename->data, + ¶ms.stashbase); + + if (res == -1) { + goto pbiudone; + } + + params.createdstash = res; + } + } + + // Build a hash table of the available commands + cmdht = mzHashTableCreate(cmdcount, NULL); + + for (i = 0; i < cmdcount; ++i) { + cmdhash = HashString(commands[i].name); + mzHashTableLookup(cmdht, cmdhash, (void*) &commands[i], CompareCommands, true); + } + + // Subsequent lines are all individual transfer commands + for (line = strtok_r(NULL, "\n", &linesave); line; + line = strtok_r(NULL, "\n", &linesave)) { + + logcmd = strdup(line); + params.cmdname = strtok_r(line, " ", ¶ms.cpos); + + if (params.cmdname == NULL) { + fprintf(stderr, "missing command [%s]\n", line); + goto pbiudone; + } + + cmdhash = HashString(params.cmdname); + cmd = (const Command*) mzHashTableLookup(cmdht, cmdhash, params.cmdname, + CompareCommandNames, false); + + if (cmd == NULL) { + fprintf(stderr, "unexpected command [%s]\n", params.cmdname); + goto pbiudone; + } + + if (cmd->f != NULL && cmd->f(¶ms) == -1) { + fprintf(stderr, "failed to execute command [%s]\n", + logcmd ? logcmd : params.cmdname); + goto pbiudone; + } + + if (logcmd) { + free(logcmd); + logcmd = NULL; + } + + if (params.canwrite) { + fprintf(cmd_pipe, "set_progress %.4f\n", (double) params.written / total_blocks); + fflush(cmd_pipe); + } + } + + if (params.canwrite) { + pthread_join(params.thread, NULL); + + fprintf(stderr, "wrote %d blocks; expected %d\n", params.written, total_blocks); + fprintf(stderr, "max alloc needed was %zu\n", params.bufsize); + + // Delete stash only after successfully completing the update, as it + // may contain blocks needed to complete the update later. + DeleteStash(params.stashbase); + } else { + fprintf(stderr, "verified partition contents; update may be resumed\n"); + } + + rc = 0; + +pbiudone: + if (params.fd != -1) { + if (fsync(params.fd) == -1) { + fprintf(stderr, "failed to fsync device (errno %d)\n", errno); + } + TEMP_FAILURE_RETRY(close(params.fd)); + } + + if (logcmd) { + free(logcmd); + } + + if (cmdht) { + mzHashTableFree(cmdht); + } + + if (params.buffer) { + free(params.buffer); + } + + if (transfer_list) { + free(transfer_list); + } + + if (blockdev_filename) { + FreeValue(blockdev_filename); + } + + if (transfer_list_value) { + FreeValue(transfer_list_value); + } + + if (new_data_fn) { + FreeValue(new_data_fn); + } + + if (patch_data_fn) { + FreeValue(patch_data_fn); + } + + // Only delete the stash if the update cannot be resumed, or it's + // a verification run and we created the stash. + if (params.isunresumable || (!params.canwrite && params.createdstash)) { + DeleteStash(params.stashbase); + } + + if (params.stashbase) { + free(params.stashbase); + } + + return StringValue(rc == 0 ? strdup("t") : strdup("")); +} + +// The transfer list is a text file containing commands to +// transfer data from one place to another on the target +// partition. We parse it and execute the commands in order: +// +// zero [rangeset] +// - fill the indicated blocks with zeros +// +// new [rangeset] +// - fill the blocks with data read from the new_data file +// +// erase [rangeset] +// - mark the given blocks as empty +// +// move <...> +// bsdiff <...> +// imgdiff <...> +// - read the source blocks, apply a patch (or not in the +// case of move), write result to target blocks. bsdiff or +// imgdiff specifies the type of patch; move means no patch +// at all. +// +// The format of <...> differs between versions 1 and 2; +// see the LoadSrcTgtVersion{1,2}() functions for a +// description of what's expected. +// +// stash +// - (version 2+ only) load the given source range and stash +// the data in the given slot of the stash table. +// +// The creator of the transfer list will guarantee that no block +// is read (ie, used as the source for a patch or move) after it +// has been written. +// +// In version 2, the creator will guarantee that a given stash is +// loaded (with a stash command) before it's used in a +// move/bsdiff/imgdiff command. +// +// Within one command the source and target ranges may overlap so +// in general we need to read the entire source into memory before +// writing anything to the target blocks. +// +// All the patch data is concatenated into one patch_data file in +// the update package. It must be stored uncompressed because we +// memory-map it in directly from the archive. (Since patches are +// already compressed, we lose very little by not compressing +// their concatenation.) +// +// In version 3, commands that read data from the partition (i.e. +// move/bsdiff/imgdiff/stash) have one or more additional hashes +// before the range parameters, which are used to check if the +// command has already been completed and verify the integrity of +// the source data. + +Value* BlockImageVerifyFn(const char* name, State* state, int argc, Expr* argv[]) { + // Commands which are not tested are set to NULL to skip them completely + const Command commands[] = { + { "bsdiff", PerformCommandDiff }, + { "erase", NULL }, + { "free", PerformCommandFree }, + { "imgdiff", PerformCommandDiff }, + { "move", PerformCommandMove }, + { "new", NULL }, + { "stash", PerformCommandStash }, + { "zero", NULL } + }; + + // Perform a dry run without writing to test if an update can proceed + return PerformBlockImageUpdate(name, state, argc, argv, commands, + sizeof(commands) / sizeof(commands[0]), 1); +} + +Value* BlockImageUpdateFn(const char* name, State* state, int argc, Expr* argv[]) { + const Command commands[] = { + { "bsdiff", PerformCommandDiff }, + { "erase", PerformCommandErase }, + { "free", PerformCommandFree }, + { "imgdiff", PerformCommandDiff }, + { "move", PerformCommandMove }, + { "new", PerformCommandNew }, + { "stash", PerformCommandStash }, + { "zero", PerformCommandZero } + }; + + return PerformBlockImageUpdate(name, state, argc, argv, commands, + sizeof(commands) / sizeof(commands[0]), 0); +} + +Value* RangeSha1Fn(const char* name, State* state, int argc, Expr* argv[]) { + Value* blockdev_filename; + Value* ranges; + const uint8_t* digest = NULL; + if (ReadValueArgs(state, argv, 2, &blockdev_filename, &ranges) < 0) { + return NULL; + } + + if (blockdev_filename->type != VAL_STRING) { + ErrorAbort(state, "blockdev_filename argument to %s must be string", name); + goto done; + } + if (ranges->type != VAL_STRING) { + ErrorAbort(state, "ranges argument to %s must be string", name); + goto done; + } + + int fd = open(blockdev_filename->data, O_RDWR); if (fd < 0) { ErrorAbort(state, "failed to open %s: %s", blockdev_filename->data, strerror(errno)); goto done; @@ -784,9 +1887,19 @@ Value* RangeSha1Fn(const char* name, State* state, int argc, Expr* argv[]) { int i, j; for (i = 0; i < rs->count; ++i) { - check_lseek(fd, (off64_t)rs->pos[i*2] * BLOCKSIZE, SEEK_SET); + if (check_lseek(fd, (off64_t)rs->pos[i*2] * BLOCKSIZE, SEEK_SET) == -1) { + ErrorAbort(state, "failed to seek %s: %s", blockdev_filename->data, + strerror(errno)); + goto done; + } + for (j = rs->pos[i*2]; j < rs->pos[i*2+1]; ++j) { - readblock(fd, buffer, BLOCKSIZE); + if (read_all(fd, buffer, BLOCKSIZE) == -1) { + ErrorAbort(state, "failed to read %s: %s", blockdev_filename->data, + strerror(errno)); + goto done; + } + SHA_update(&ctx, buffer, BLOCKSIZE); } } @@ -804,6 +1917,7 @@ Value* RangeSha1Fn(const char* name, State* state, int argc, Expr* argv[]) { } void RegisterBlockImageFunctions() { + RegisterFunction("block_image_verify", BlockImageVerifyFn); RegisterFunction("block_image_update", BlockImageUpdateFn); RegisterFunction("range_sha1", RangeSha1Fn); } -- cgit v1.1