Add support for block incremental OTAs

Change-Id: Ie72015e34ed8d7595a5c74c8df41cba73275afab

4 files changed, 265 insertions, 13 deletions

diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py
index 6b8bf15..4be67c9 100644
--- a/tools/releasetools/common.py
+++ b/tools/releasetools/common.py
@@ -740,11 +740,14 @@ class PasswordManager(object):
     return result
 
 
-def ZipWriteStr(zip, filename, data, perms=0644):
+def ZipWriteStr(zip, filename, data, perms=0644, compression=None):
   # use a fixed timestamp so the output is repeatable.
   zinfo = zipfile.ZipInfo(filename=filename,
                           date_time=(2009, 1, 1, 0, 0, 0))
-  zinfo.compress_type = zip.compression
+  if compression is None:
+    zinfo.compress_type = zip.compression
+  else:
+    zinfo.compress_type = compression
   zinfo.external_attr = perms << 16
   zip.writestr(zinfo, data)
 
@@ -850,8 +853,8 @@ class File(object):
     t.flush()
     return t
 
-  def AddToZip(self, z):
-    ZipWriteStr(z, self.name, self.data)
+  def AddToZip(self, z, compression=None):
+    ZipWriteStr(z, self.name, self.data, compression=compression)
 
 DIFF_PROGRAM_BY_EXT = {
     ".gz" : "imgdiff",
@@ -989,6 +992,30 @@ def ParseCertificate(data):
   cert = "".join(cert).decode('base64')
   return cert
 
+def XDelta3(source_path, target_path, output_path):
+  diff_program = ["xdelta3", "-0", "-B", str(64<<20), "-e", "-f", "-s"]
+  diff_program.append(source_path)
+  diff_program.append(target_path)
+  diff_program.append(output_path)
+  p = Run(diff_program, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+  p.communicate()
+  assert p.returncode == 0, "Couldn't produce patch"
+
+def XZ(path):
+  compress_program = ["xz", "-zk", "-9", "--check=crc32"]
+  compress_program.append(path)
+  p = Run(compress_program, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+  p.communicate()
+  assert p.returncode == 0, "Couldn't compress patch"
+
+def MakeSystemPatch(source_file, target_file):
+  with tempfile.NamedTemporaryFile() as output_file:
+    XDelta3(source_file.name, target_file.name, output_file.name)
+    XZ(output_file.name)
+    with open(output_file.name + ".xz") as patch_file:
+      patch_data = patch_file.read()
+      os.unlink(patch_file.name)
+      return File("system.img.p", patch_data)
 
 def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img):
   """Generate a binary patch that creates the recovery image starting
diff --git a/tools/releasetools/edify_generator.py b/tools/releasetools/edify_generator.py
index 426b713..a5340a0 100644
--- a/tools/releasetools/edify_generator.py
+++ b/tools/releasetools/edify_generator.py
@@ -81,6 +81,18 @@ class EdifyGenerator(object):
            ) % (" or ".join(fp),)
     self.script.append(cmd)
 
+  def AssertRecoveryFingerprint(self, *fp):
+    """Assert that the current recovery build fingerprint is one of *fp."""
+    if not fp:
+      raise ValueError("must specify some fingerprints")
+    cmd = (
+           ' ||\n    '.join([('getprop("ro.build.fingerprint") == "%s"')
+                        % i for i in fp]) +
+           ' ||\n    abort("Package expects build fingerprint of %s; this '
+           'device has " + getprop("ro.build.fingerprint") + ".");'
+           ) % (" or ".join(fp),)
+    self.script.append(cmd)
+
   def AssertOlderBuild(self, timestamp, timestamp_text):
     """Assert that the build on the device is older (or the same as)
     the given timestamp."""
@@ -296,3 +308,8 @@ class EdifyGenerator(object):
       data = open(os.path.join(input_path, "updater")).read()
     common.ZipWriteStr(output_zip, "META-INF/com/google/android/update-binary",
                        data, perms=0755)
+
+  def Syspatch(self, filename, size, target_sha, source_sha, patchfile):
+    """Applies a compressed binary patch to a block device."""
+    call = 'syspatch("%s", "%s", "%s", "%s", "%s");'
+    self.script.append(call % (filename, size, target_sha, source_sha, patchfile))
diff --git a/tools/releasetools/img_from_target_files.py b/tools/releasetools/img_from_target_files.py
index ab505b1..a7aca13 100755
--- a/tools/releasetools/img_from_target_files.py
+++ b/tools/releasetools/img_from_target_files.py
@@ -56,7 +56,11 @@ OPTIONS = common.OPTIONS
 def AddSystem(output_zip, sparse=True):
   """Turn the contents of SYSTEM into a system image and store it in
   output_zip."""
+  data = BuildSystem(OPTIONS.input_tmp, OPTIONS.info_dict, sparse=sparse)
+  common.ZipWriteStr(output_zip, "system.img", data)
+
 
+def BuildSystem(input_dir, info_dict, sparse=True):
   print "creating system.img..."
 
   img = tempfile.NamedTemporaryFile()
@@ -65,8 +69,8 @@ def AddSystem(output_zip, sparse=True):
   # mkyaffs2image.  It wants "system" but we have a directory named
   # "SYSTEM", so create a symlink.
   try:
-    os.symlink(os.path.join(OPTIONS.input_tmp, "SYSTEM"),
-               os.path.join(OPTIONS.input_tmp, "system"))
+    os.symlink(os.path.join(input_dir, "SYSTEM"),
+               os.path.join(input_dir, "system"))
   except OSError, e:
       # bogus error on my mac version?
       #   File "./build/tools/releasetools/img_from_target_files", line 86, in AddSystem
@@ -75,12 +79,11 @@ def AddSystem(output_zip, sparse=True):
     if (e.errno == errno.EEXIST):
       pass
 
-  image_props = build_image.ImagePropFromGlobalDict(OPTIONS.info_dict,
-                                                    "system")
-  fstab = OPTIONS.info_dict["fstab"]
+  image_props = build_image.ImagePropFromGlobalDict(info_dict, "system")
+  fstab = info_dict["fstab"]
   if fstab:
     image_props["fs_type" ] = fstab["/system"].fs_type
-  succ = build_image.BuildImage(os.path.join(OPTIONS.input_tmp, "system"),
+  succ = build_image.BuildImage(os.path.join(input_dir, "system"),
                                 image_props, img.name)
   assert succ, "build system.img image failed"
 
@@ -98,7 +101,7 @@ def AddSystem(output_zip, sparse=True):
     finally:
       os.unlink(name)
 
-  common.ZipWriteStr(output_zip, "system.img", data)
+  return data
 
 
 def AddVendor(output_zip):
diff --git a/tools/releasetools/ota_from_target_files b/tools/releasetools/ota_from_target_files
index dcfbf83..7c9f80c 100755
--- a/tools/releasetools/ota_from_target_files
+++ b/tools/releasetools/ota_from_target_files
@@ -82,6 +82,7 @@ except ImportError:
 import common
 import img_from_target_files
 import edify_generator
+import build_image
 
 OPTIONS = common.OPTIONS
 OPTIONS.package_key = None
@@ -544,7 +545,213 @@ def AddToKnownPaths(filename, known_paths):
     known_paths.add(path)
     dirs.pop()
 
+def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip):
+  source_version = OPTIONS.source_info_dict["recovery_api_version"]
+  target_version = OPTIONS.target_info_dict["recovery_api_version"]
+
+  if source_version == 0:
+    print ("WARNING: generating edify script for a source that "
+           "can't install it.")
+  script = edify_generator.EdifyGenerator(source_version,
+                                          OPTIONS.target_info_dict)
+
+  metadata = {"pre-device": GetBuildProp("ro.product.device",
+                                         OPTIONS.source_info_dict),
+              "post-timestamp": GetBuildProp("ro.build.date.utc",
+                                             OPTIONS.target_info_dict),
+              }
+
+  device_specific = common.DeviceSpecificParams(
+      source_zip=source_zip,
+      source_version=source_version,
+      target_zip=target_zip,
+      target_version=target_version,
+      output_zip=output_zip,
+      script=script,
+      metadata=metadata,
+      info_dict=OPTIONS.info_dict)
+
+  source_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.source_info_dict)
+  target_fp = GetBuildProp("ro.build.fingerprint", OPTIONS.target_info_dict)
+  metadata["pre-build"] = source_fp
+  metadata["post-build"] = target_fp
+
+  source_boot = common.GetBootableImage(
+      "/tmp/boot.img", "boot.img", OPTIONS.source_tmp, "BOOT",
+      OPTIONS.source_info_dict)
+  target_boot = common.GetBootableImage(
+      "/tmp/boot.img", "boot.img", OPTIONS.target_tmp, "BOOT")
+  updating_boot = (not OPTIONS.two_step and
+                   (source_boot.data != target_boot.data))
+
+  source_recovery = common.GetBootableImage(
+      "/tmp/recovery.img", "recovery.img", OPTIONS.source_tmp, "RECOVERY",
+      OPTIONS.source_info_dict)
+  target_recovery = common.GetBootableImage(
+      "/tmp/recovery.img", "recovery.img", OPTIONS.target_tmp, "RECOVERY")
+  updating_recovery = (source_recovery.data != target_recovery.data)
+
+  with tempfile.NamedTemporaryFile() as src_file:
+    with tempfile.NamedTemporaryFile() as tgt_file:
+      print "building source system image..."
+      src_file = tempfile.NamedTemporaryFile()
+      src_data = img_from_target_files.BuildSystem(
+          OPTIONS.source_tmp, OPTIONS.source_info_dict, sparse=False)
+      src_sys_sha1 = sha1(src_data).hexdigest()
+      print "source system sha1:", src_sys_sha1
+      src_file.write(src_data)
+      src_data = None
+
+      print "building target system image..."
+      tgt_file = tempfile.NamedTemporaryFile()
+      tgt_data = img_from_target_files.BuildSystem(
+          OPTIONS.target_tmp, OPTIONS.target_info_dict, sparse=False)
+      tgt_sys_sha1 = sha1(tgt_data).hexdigest()
+      print "target system sha1:", tgt_sys_sha1
+      tgt_sys_len = len(tgt_data)
+      tgt_file.write(tgt_data)
+      tgt_data = None
+
+      system_type, system_device = common.GetTypeAndDevice("/system", OPTIONS.info_dict)
+      system_patch = common.MakeSystemPatch(src_file, tgt_file)
+      system_patch.AddToZip(output_zip, compression=zipfile.ZIP_STORED)
+
+  AppendAssertions(script, OPTIONS.target_info_dict)
+  device_specific.IncrementalOTA_Assertions()
+
+  # Two-step incremental package strategy (in chronological order,
+  # which is *not* the order in which the generated script has
+  # things):
+  #
+  # if stage is not "2/3" or "3/3":
+  #    do verification on current system
+  #    write recovery image to boot partition
+  #    set stage to "2/3"
+  #    reboot to boot partition and restart recovery
+  # else if stage is "2/3":
+  #    write recovery image to recovery partition
+  #    set stage to "3/3"
+  #    reboot to recovery partition and restart recovery
+  # else:
+  #    (stage must be "3/3")
+  #    perform update:
+  #       patch system files, etc.
+  #       force full install of new boot image
+  #       set up system to update recovery partition on first boot
+  #    complete script normally (allow recovery to mark itself finished and reboot)
+
+  if OPTIONS.two_step:
+    if not OPTIONS.info_dict.get("multistage_support", None):
+      assert False, "two-step packages not supported by this build"
+    fs = OPTIONS.info_dict["fstab"]["/misc"]
+    assert fs.fs_type.upper() == "EMMC", \
+        "two-step packages only supported on devices with EMMC /misc partitions"
+    bcb_dev = {"bcb_dev": fs.device}
+    common.ZipWriteStr(output_zip, "recovery.img", target_recovery.data)
+    script.AppendExtra("""
+if get_stage("%(bcb_dev)s", "stage") == "2/3" then
+""" % bcb_dev)
+    script.AppendExtra("sleep(20);\n");
+    script.WriteRawImage("/recovery", "recovery.img")
+    script.AppendExtra("""
+set_stage("%(bcb_dev)s", "3/3");
+reboot_now("%(bcb_dev)s", "recovery");
+else if get_stage("%(bcb_dev)s", "stage") != "3/3" then
+""" % bcb_dev)
+
+  script.Print("Verifying current system...")
+
+  device_specific.IncrementalOTA_VerifyBegin()
+
+  script.AssertRecoveryFingerprint(source_fp, target_fp)
+
+  if updating_boot:
+    total_verify_size += OPTIONS.info_dict["boot_size"]
+    d = common.Difference(target_boot, source_boot)
+    _, _, d = d.ComputePatch()
+    print "boot      target: %d  source: %d  diff: %d" % (
+        target_boot.size, source_boot.size, len(d))
+
+    common.ZipWriteStr(output_zip, "patch/boot.img.p", d)
+
+    boot_type, boot_device = common.GetTypeAndDevice("/boot", OPTIONS.info_dict)
+
+    script.PatchCheck("%s:%s:%d:%s:%d:%s" %
+                      (boot_type, boot_device,
+                       source_boot.size, source_boot.sha1,
+                       target_boot.size, target_boot.sha1))
+
+  device_specific.IncrementalOTA_VerifyEnd()
+
+  if OPTIONS.two_step:
+    script.WriteRawImage("/boot", "recovery.img")
+    script.AppendExtra("""
+set_stage("%(bcb_dev)s", "2/3");
+reboot_now("%(bcb_dev)s", "");
+else
+""" % bcb_dev)
+
+  script.Comment("---- start making changes here ----")
+
+  device_specific.IncrementalOTA_InstallBegin()
+
+  if OPTIONS.wipe_user_data:
+    script.Print("Erasing user data...")
+    script.FormatPartition("/data")
+
+  script.Print("Patching system image...")
+  script.Syspatch(system_device,
+                  OPTIONS.info_dict["system_size"],
+                  tgt_sys_sha1,
+                  src_sys_sha1,
+                  system_patch.name)
+
+  if OPTIONS.two_step:
+    common.ZipWriteStr(output_zip, "boot.img", target_boot.data)
+    script.WriteRawImage("/boot", "boot.img")
+    print "writing full boot image (forced by two-step mode)"
+
+  if not OPTIONS.two_step:
+    if updating_boot:
+      # Produce the boot image by applying a patch to the current
+      # contents of the boot partition, and write it back to the
+      # partition.
+      script.Print("Patching boot image...")
+      script.ApplyPatch("%s:%s:%d:%s:%d:%s"
+                        % (boot_type, boot_device,
+                           source_boot.size, source_boot.sha1,
+                           target_boot.size, target_boot.sha1),
+                        "-",
+                        target_boot.size, target_boot.sha1,
+                        source_boot.sha1, "patch/boot.img.p")
+      print "boot image changed; including."
+    else:
+      print "boot image unchanged; skipping."
+
+  # Do device-specific installation (eg, write radio image).
+  device_specific.IncrementalOTA_InstallEnd()
+
+  if OPTIONS.extra_script is not None:
+    script.AppendExtra(OPTIONS.extra_script)
+
+  if OPTIONS.two_step:
+    script.AppendExtra("""
+set_stage("%(bcb_dev)s", "");
+endif;
+endif;
+""" % bcb_dev)
+
+  script.SetProgress(1)
+  script.AddToZip(target_zip, output_zip)
+  WriteMetadata(metadata, output_zip)
+
 def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
+  target_has_recovery_patch = HasRecoveryPatch(target_zip)
+  source_has_recovery_patch = HasRecoveryPatch(source_zip)
+
+  if target_has_recovery_patch and source_has_recovery_patch:
+    return WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip)
+
   source_version = OPTIONS.source_info_dict["recovery_api_version"]
   target_version = OPTIONS.target_info_dict["recovery_api_version"]
 
@@ -575,8 +782,6 @@ def WriteIncrementalOTAPackage(target_zip, source_zip, output_zip):
   print "Loading source..."
   source_data = LoadSystemFiles(source_zip)
 
-  target_has_recovery_patch = HasRecoveryPatch(target_zip)
-
   verbatim_targets = []
   patch_list = []
   diffs = []