Hook up the backup data writer, and add a utility to read the backup data files.

2 files changed, 149 insertions, 56 deletions

diff --git a/libs/utils/backup_data.cpp b/libs/utils/backup_data.cpp
index dd04449..95c05b7 100644
--- a/libs/utils/backup_data.cpp
+++ b/libs/utils/backup_data.cpp
@@ -39,10 +39,6 @@ namespace android {
  *      - The value, padded to 4 byte boundary
  */
 
-#define APP_MAGIC_V1 0x31707041 // App1 (little endian)
-#define ENTITY_MAGIC_V1 0x61746144 // Data (little endian)
-#define FOOTER_MAGIC_V1 0x746f6f46 // Foot (little endian)
-
 const static int ROUND_UP[4] = { 0, 3, 2, 1 };
 
 static inline size_t
@@ -108,7 +104,7 @@ BackupDataWriter::WriteAppHeader(const String8& packageName, int cookie)
 
     nameLen = packageName.length();
 
-    header.type = tolel(APP_MAGIC_V1);
+    header.type = tolel(BACKUP_HEADER_APP_V1);
     header.packageLen = tolel(nameLen);
     header.cookie = cookie;
 
@@ -148,7 +144,7 @@ BackupDataWriter::WriteEntityHeader(const String8& key, size_t dataSize)
 
     keyLen = key.length();
 
-    header.type = tolel(ENTITY_MAGIC_V1);
+    header.type = tolel(BACKUP_HEADER_ENTITY_V1);
     header.keyLen = tolel(keyLen);
     header.dataSize = tolel(dataSize);
 
@@ -209,7 +205,7 @@ BackupDataWriter::WriteAppFooter(int cookie)
     app_footer_v1 footer;
     ssize_t nameLen;
 
-    footer.type = tolel(FOOTER_MAGIC_V1);
+    footer.type = tolel(BACKUP_FOOTER_APP_V1);
     footer.entityCount = tolel(m_entityCount);
     footer.cookie = cookie;
 
@@ -264,7 +260,7 @@ BackupDataReader::Status()
     } while(0)
 
 status_t
-BackupDataReader::ReadNextHeader()
+BackupDataReader::ReadNextHeader(int* type)
 {
     if (m_status != NO_ERROR) {
         return m_status;
@@ -280,7 +276,7 @@ BackupDataReader::ReadNextHeader()
     m_header.type = fromlel(m_header.type);
     switch (m_header.type)
     {
-        case APP_MAGIC_V1:
+        case BACKUP_HEADER_APP_V1:
             m_header.app.packageLen = fromlel(m_header.app.packageLen);
             if (m_header.app.packageLen < 0) {
                 LOGD("App header at %d has packageLen<0: 0x%08x\n", (int)m_pos,
@@ -289,7 +285,7 @@ BackupDataReader::ReadNextHeader()
             }
             m_header.app.cookie = m_header.app.cookie;
             break;
-        case ENTITY_MAGIC_V1:
+        case BACKUP_HEADER_ENTITY_V1:
             m_header.entity.keyLen = fromlel(m_header.entity.keyLen);
             if (m_header.entity.keyLen <= 0) {
                 LOGD("Entity header at %d has keyLen<=0: 0x%08x\n", (int)m_pos,
@@ -297,14 +293,9 @@ BackupDataReader::ReadNextHeader()
                 m_status = EINVAL;
             }
             m_header.entity.dataSize = fromlel(m_header.entity.dataSize);
-            if (m_header.entity.dataSize < 0) {
-                LOGD("Entity header at %d has dataSize<0: 0x%08x\n", (int)m_pos,
-                        (int)m_header.entity.dataSize);
-                m_status = EINVAL;
-            }
             m_entityCount++;
             break;
-        case FOOTER_MAGIC_V1:
+        case BACKUP_FOOTER_APP_V1:
             m_header.footer.entityCount = fromlel(m_header.footer.entityCount);
             if (m_header.footer.entityCount < 0) {
                 LOGD("Entity header at %d has entityCount<0: 0x%08x\n", (int)m_pos,
@@ -318,6 +309,9 @@ BackupDataReader::ReadNextHeader()
             m_status = EINVAL;
     }
     m_pos += sizeof(m_header);
+    if (type) {
+        *type = m_header.type;
+    }
     
     return m_status;
 }
@@ -328,7 +322,7 @@ BackupDataReader::ReadAppHeader(String8* packageName, int* cookie)
     if (m_status != NO_ERROR) {
         return m_status;
     }
-    if (m_header.type != APP_MAGIC_V1) {
+    if (m_header.type != BACKUP_HEADER_APP_V1) {
         return EINVAL;
     }
     size_t size = m_header.app.packageLen;
@@ -349,7 +343,7 @@ BackupDataReader::ReadAppHeader(String8* packageName, int* cookie)
 bool
 BackupDataReader::HasEntities()
 {
-    return m_status == NO_ERROR && m_header.type == ENTITY_MAGIC_V1;
+    return m_status == NO_ERROR && m_header.type == BACKUP_HEADER_ENTITY_V1;
 }
 
 status_t
@@ -358,10 +352,10 @@ BackupDataReader::ReadEntityHeader(String8* key, size_t* dataSize)
     if (m_status != NO_ERROR) {
         return m_status;
     }
-    if (m_header.type != ENTITY_MAGIC_V1) {
+    if (m_header.type != BACKUP_HEADER_ENTITY_V1) {
         return EINVAL;
     }
-    size_t size = m_header.app.packageLen;
+    size_t size = m_header.entity.keyLen;
     char* buf = key->lockBuffer(size);
     if (key == NULL) {
         key->unlockBuffer();
@@ -378,6 +372,23 @@ BackupDataReader::ReadEntityHeader(String8* key, size_t* dataSize)
 }
 
 status_t
+BackupDataReader::SkipEntityData()
+{
+    if (m_status != NO_ERROR) {
+        return m_status;
+    }
+    if (m_header.type != BACKUP_HEADER_ENTITY_V1) {
+        return EINVAL;
+    }
+    if (m_header.entity.dataSize > 0) {
+        int pos = lseek(m_fd, m_header.entity.dataSize, SEEK_CUR);
+        return pos == -1 ? (int)errno : (int)NO_ERROR;
+    } else {
+        return NO_ERROR;
+    }
+}
+
+status_t
 BackupDataReader::ReadEntityData(void* data, size_t size)
 {
     if (m_status != NO_ERROR) {
@@ -395,7 +406,7 @@ BackupDataReader::ReadAppFooter(int* cookie)
     if (m_status != NO_ERROR) {
         return m_status;
     }
-    if (m_header.type != FOOTER_MAGIC_V1) {
+    if (m_header.type != BACKUP_FOOTER_APP_V1) {
         return EINVAL;
     }
     if (m_header.footer.entityCount != m_entityCount) {
diff --git a/libs/utils/backup_helper_file.cpp b/libs/utils/backup_helper_file.cpp
index bf56945..8efb3eb 100644
--- a/libs/utils/backup_helper_file.cpp
+++ b/libs/utils/backup_helper_file.cpp
@@ -40,7 +40,7 @@ namespace android {
 #define MAGIC0 0x70616e53 // Snap
 #define MAGIC1 0x656c6946 // File
 
-#if TEST_BACKUP_HELPERS
+#if 0 // TEST_BACKUP_HELPERS
 #define LOGP(x...) printf(x)
 #else
 #define LOGP(x...) LOGD(x)
@@ -181,45 +181,105 @@ write_snapshot_file(int fd, const KeyedVector<String8,FileState>& snapshot)
 }
 
 static int
-write_delete_file(const String8& key)
+write_delete_file(BackupDataWriter* dataStream, const String8& key)
 {
     LOGP("write_delete_file %s\n", key.string());
-    return 0;
+    return dataStream->WriteEntityHeader(key, -1);
 }
 
 static int
-write_update_file(const String8& realFilename, const String8& key)
+write_update_file(BackupDataWriter* dataStream, int fd, const String8& key,
+        const String8& realFilename)
 {
     LOGP("write_update_file %s (%s)\n", realFilename.string(), key.string());
-    return 0;
-}
 
-static int
-compute_crc32(const String8& filename)
-{
     const int bufsize = 4*1024;
+    int err;
     int amt;
+    int fileSize;
+    int bytesLeft;
+
+    char* buf = (char*)malloc(bufsize);
+    int crc = crc32(0L, Z_NULL, 0);
+
+
+    bytesLeft = fileSize = lseek(fd, 0, SEEK_END);
+    lseek(fd, 0, SEEK_SET);
+
+    err = dataStream->WriteEntityHeader(key, bytesLeft);
+    if (err != 0) {
+        return err;
+    }
+
+    while ((amt = read(fd, buf, bufsize)) != 0 && bytesLeft > 0) {
+        bytesLeft -= amt;
+        if (bytesLeft < 0) {
+            amt += bytesLeft; // Plus a negative is minus.  Don't write more than we promised.
+        }
+        err = dataStream->WriteEntityData(buf, amt);
+        if (err != 0) {
+            return err;
+        }
+    }
+    if (bytesLeft != 0) {
+        if (bytesLeft > 0) {
+            // Pad out the space we promised in the buffer.  We can't corrupt the buffer,
+            // even though the data we're sending is probably bad.
+            memset(buf, 0, bufsize);
+            while (bytesLeft > 0) {
+                amt = bytesLeft < bufsize ? bytesLeft : bufsize;
+                bytesLeft -= amt;
+                err = dataStream->WriteEntityData(buf, amt);
+                if (err != 0) {
+                    return err;
+                }
+            }
+        }
+        LOGE("write_update_file size mismatch for %s. expected=%d actual=%d."
+                " You aren't doing proper locking!",
+                realFilename.string(), fileSize, fileSize-bytesLeft);
+    }
+
+    free(buf);
 
-    int fd = open(filename.string(), O_RDONLY);
+    return NO_ERROR;
+}
+
+static int
+write_update_file(BackupDataWriter* dataStream, const String8& key, const String8& realFilename)
+{
+    int err;
+    int fd = open(realFilename.string(), O_RDONLY);
     if (fd == -1) {
-        return -1;
+        return errno;
     }
+    err = write_update_file(dataStream, fd, key, realFilename);
+    close(fd);
+    return err;
+}
+
+static int
+compute_crc32(int fd)
+{
+    const int bufsize = 4*1024;
+    int amt;
 
     char* buf = (char*)malloc(bufsize);
     int crc = crc32(0L, Z_NULL, 0);
 
+    lseek(fd, 0, SEEK_SET);
+
     while ((amt = read(fd, buf, bufsize)) != 0) {
         crc = crc32(crc, (Bytef*)buf, amt);
     }
 
-    close(fd);
     free(buf);
 
     return crc;
 }
 
 int
-back_up_files(int oldSnapshotFD, int oldDataStream, int newSnapshotFD,
+back_up_files(int oldSnapshotFD, BackupDataWriter* dataStream, int newSnapshotFD,
         char const* fileBase, char const* const* files, int fileCount)
 {
     int err;
@@ -252,7 +312,8 @@ back_up_files(int oldSnapshotFD, int oldDataStream, int newSnapshotFD,
         s.modTime_nsec = 0; // workaround sim breakage
         //s.modTime_nsec = st.st_mtime_nsec;
         s.size = st.st_size;
-        s.crc32 = compute_crc32(realFilename);
+
+        // we compute the crc32 later down below, when we already have the file open.
 
         newSnapshot.add(name, s);
     }
@@ -270,30 +331,42 @@ back_up_files(int oldSnapshotFD, int oldDataStream, int newSnapshotFD,
             String8 realFilename(base);
             realFilename.appendPath(q);
             LOGP("file added: %s\n", realFilename.string());
-            write_update_file(realFilename, q);
+            write_update_file(dataStream, q, realFilename);
             m++;
         }
         else if (cmp < 0) {
             // file removed
             LOGP("file removed: %s\n", p.string());
-            write_delete_file(p);
+            dataStream->WriteEntityHeader(p, -1);
             n++;
         }
         else {
+
             // both files exist, check them
             String8 realFilename(base);
             realFilename.appendPath(q);
             const FileState& f = oldSnapshot.valueAt(n);
-            const FileState& g = newSnapshot.valueAt(m);
-
-            LOGP("%s\n", q.string());
-            LOGP("  new: modTime=%d,%d size=%-3d crc32=0x%08x\n",
-                    f.modTime_sec, f.modTime_nsec, f.size, f.crc32);
-            LOGP("  old: modTime=%d,%d size=%-3d crc32=0x%08x\n",
-                    g.modTime_sec, g.modTime_nsec, g.size, g.crc32);
-            if (f.modTime_sec != g.modTime_sec || f.modTime_nsec != g.modTime_nsec
-                    || f.size != g.size || f.crc32 != g.crc32) {
-                write_update_file(realFilename, p);
+            FileState& g = newSnapshot.editValueAt(m);
+
+            int fd = open(realFilename.string(), O_RDONLY);
+            if (fd != -1) {
+                // We can't open the file.  Don't report it as a delete either.  Let the
+                // server keep the old version.  Maybe they'll be able to deal with it
+                // on restore.
+            } else {
+                g.crc32 = compute_crc32(fd);
+
+                LOGP("%s\n", q.string());
+                LOGP("  new: modTime=%d,%d size=%-3d crc32=0x%08x\n",
+                        f.modTime_sec, f.modTime_nsec, f.size, f.crc32);
+                LOGP("  old: modTime=%d,%d size=%-3d crc32=0x%08x\n",
+                        g.modTime_sec, g.modTime_nsec, g.size, g.crc32);
+                if (f.modTime_sec != g.modTime_sec || f.modTime_nsec != g.modTime_nsec
+                        || f.size != g.size || f.crc32 != g.crc32) {
+                    write_update_file(dataStream, fd, p, realFilename);
+                }
+
+                close(fd);
             }
             n++;
             m++;
@@ -302,7 +375,7 @@ back_up_files(int oldSnapshotFD, int oldDataStream, int newSnapshotFD,
 
     // these were deleted
     while (n<N) {
-        write_delete_file(oldSnapshot.keyAt(n));
+        dataStream->WriteEntityHeader(oldSnapshot.keyAt(n), -1);
         n++;
     }
 
@@ -311,7 +384,7 @@ back_up_files(int oldSnapshotFD, int oldDataStream, int newSnapshotFD,
         const String8& q = newSnapshot.keyAt(m);
         String8 realFilename(base);
         realFilename.appendPath(q);
-        write_update_file(realFilename, q);
+        write_update_file(dataStream, q, realFilename);
         m++;
     }
 
@@ -911,10 +984,14 @@ backup_helper_test_files()
         fprintf(stderr, "error creating: %s\n", strerror(errno));
         return errno;
     }
+
+    {
+        BackupDataWriter dataStream(dataStreamFD);
     
-    err = back_up_files(-1, dataStreamFD, newSnapshotFD, SCRATCH_DIR, files_before, 5);
-    if (err != 0) {
-        return err;
+        err = back_up_files(-1, &dataStream, newSnapshotFD, SCRATCH_DIR, files_before, 5);
+        if (err != 0) {
+            return err;
+        }
     }
 
     close(dataStreamFD);
@@ -968,10 +1045,15 @@ backup_helper_test_files()
         return errno;
     }
 
-    err = back_up_files(oldSnapshotFD, dataStreamFD, newSnapshotFD, SCRATCH_DIR, files_after, 6);
-    if (err != 0) {
-        return err;
-    }
+    {
+        BackupDataWriter dataStream(dataStreamFD);
+    
+        err = back_up_files(oldSnapshotFD, &dataStream, newSnapshotFD, SCRATCH_DIR,
+                files_after, 6);
+        if (err != 0) {
+            return err;
+        }
+}
 
     close(oldSnapshotFD);
     close(dataStreamFD);