SDK: primitive implementation of download cache.

It supports:
- A local binary cache + a few http headers are saved
- If ETag is present, generates a GET with If-None-Match
- If Last-Modified is present, generates a GET with If-Modified-Since
- Ability to configure the cache to be direct (don't cache),
  or serve without checkout or serve with a server check.
- Doesn't check cached files if newer than 10 minutes.
- For servers with no ETag/LastModified support, check files
  every 4 hours (no pref to change this yet.)

Change-Id: I515e77291fb6810453e82e73f6508cfc60b2f422

8 files changed, 762 insertions, 102 deletions

diff --git a/sdkmanager/libs/sdklib/.classpath b/sdkmanager/libs/sdklib/.classpath
index a82525e..3e6a435 100644
--- a/sdkmanager/libs/sdklib/.classpath
+++ b/sdkmanager/libs/sdklib/.classpath
@@ -5,13 +5,13 @@
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
 	<classpathentry combineaccessrules="false" kind="src" path="/AndroidPrefs"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/3"/>
-	<classpathentry kind="var" path="ANDROID_SRC/prebuilt/common/commons-compress/commons-compress-1.0.jar"/>
 	<classpathentry combineaccessrules="false" kind="src" path="/common"/>
+	<classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/commons-compress/commons-compress-1.0.jar"/>
 	<classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/http-client/commons-codec-1.4.jar"/>
 	<classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/http-client/commons-logging-1.1.1.jar"/>
-	<classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/http-client/httpclient-4.1.1.jar"/>
-	<classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/http-client/httpcore-4.1.jar"/>
-	<classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/http-client/httpmime-4.1.1.jar"/>
+	<classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/http-client/httpclient-4.1.1.jar" sourcepath="/ANDROID_SRC/prebuilts/tools/common/http-client/src/httpcomponents-client-4.1.1-src.zip"/>
+	<classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/http-client/httpcore-4.1.jar" sourcepath="/ANDROID_SRC/prebuilts/tools/common/http-client/src/httpcomponents-core-4.1-src.zip"/>
+	<classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/http-client/httpmime-4.1.1.jar" sourcepath="/ANDROID_SRC/prebuilts/tools/common/http-client/src/httpcomponents-client-4.1.1-src.zip"/>
 	<classpathentry kind="var" path="ANDROID_SRC/prebuilt/common/mkidentity/mkidentity-prebuilt.jar"/>
 	<classpathentry kind="output" path="bin"/>
 </classpath>
diff --git a/sdkmanager/libs/sdklib/src/com/android/sdklib/SdkConstants.java b/sdkmanager/libs/sdklib/src/com/android/sdklib/SdkConstants.java
index ebdc9c8..6e6c657 100644
--- a/sdkmanager/libs/sdklib/src/com/android/sdklib/SdkConstants.java
+++ b/sdkmanager/libs/sdklib/src/com/android/sdklib/SdkConstants.java
@@ -260,6 +260,9 @@ public final class SdkConstants {
     /** Name of the addon libs folder. */
     public final static String FD_ADDON_LIBS = "libs";                  //$NON-NLS-1$
 
+    /** Name of the cache folder in the $HOME/.android. */
+    public final static String FD_CACHE = "cache";                      //$NON-NLS-1$
+
     /** Namespace for the resource XML, i.e. "http://schemas.android.com/apk/res/android" */
     public final static String NS_RESOURCES =
         "http://schemas.android.com/apk/res/android";                   //$NON-NLS-1$
diff --git a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/AddonsListFetcher.java b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/AddonsListFetcher.java
index 919a30e..216e309 100755
--- a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/AddonsListFetcher.java
+++ b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/AddonsListFetcher.java
@@ -29,7 +29,6 @@ import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;

 import org.xml.sax.SAXParseException;

 

-import java.io.ByteArrayInputStream;

 import java.io.FileNotFoundException;

 import java.io.IOException;

 import java.io.InputStream;

@@ -82,13 +81,14 @@ public class AddonsListFetcher {
     /**

      * Fetches the addons list from the given URL.

      *

-     * @param monitor A monitor to report errors. Cannot be null.

      * @param url The URL of an XML file resource that conforms to the latest sdk-addons-list-N.xsd.

      *   For the default operation, use {@link SdkAddonsListConstants#URL_ADDON_LIST}.

      *   Cannot be null.

+     * @param cache The {@link DownloadCache} instance to use. Cannot be null.

+     * @param monitor A monitor to report errors. Cannot be null.

      * @return An array of {@link Site} on success (possibly empty), or null on error.

      */

-    public Site[] fetch(ITaskMonitor monitor, String url) {

+    public Site[] fetch(String url, DownloadCache cache, ITaskMonitor monitor) {

 

         url = url == null ? "" : url.trim();

 

@@ -102,7 +102,7 @@ public class AddonsListFetcher {
         Document validatedDoc = null;

         String validatedUri = null;

 

-        ByteArrayInputStream xml = fetchUrl(url, monitor.createSubMonitor(1), exception);

+        InputStream xml = fetchUrl(url, cache, monitor.createSubMonitor(1), exception);

 

         if (xml != null) {

             monitor.setDescription("Validate XML");

@@ -187,41 +187,12 @@ public class AddonsListFetcher {
      *            happens during the fetch.

      * @see UrlOpener UrlOpener, which handles all URL logic.

      */

-    private ByteArrayInputStream fetchUrl(String urlString, ITaskMonitor monitor,

+    private InputStream fetchUrl(String urlString,

+            DownloadCache cache,

+            ITaskMonitor monitor,

             Exception[] outException) {

         try {

-

-            InputStream is = null;

-

-            int inc = 65536;

-            int curr = 0;

-            byte[] result = new byte[inc];

-

-            try {

-                is = UrlOpener.openUrl(urlString, monitor);

-

-                int n;

-                while ((n = is.read(result, curr, result.length - curr)) != -1) {

-                    curr += n;

-                    if (curr == result.length) {

-                        byte[] temp = new byte[curr + inc];

-                        System.arraycopy(result, 0, temp, 0, curr);

-                        result = temp;

-                    }

-                }

-

-                return new ByteArrayInputStream(result, 0, curr);

-

-            } finally {

-                if (is != null) {

-                    try {

-                        is.close();

-                    } catch (IOException e) {

-                        // pass

-                    }

-                }

-            }

-

+            return cache.openCachedUrl(urlString, monitor);

         } catch (Exception e) {

             if (outException != null) {

                 outException[0] = e;

diff --git a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/ArchiveInstaller.java b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/ArchiveInstaller.java
index 2e2396f..53fc140 100755
--- a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/ArchiveInstaller.java
+++ b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/ArchiveInstaller.java
@@ -96,6 +96,7 @@ public class ArchiveInstaller {
             String osSdkRoot,

             boolean forceHttp,

             SdkManager sdkManager,

+            DownloadCache cache,

             ITaskMonitor monitor) {

 

         Archive newArchive = archiveInfo.getNewArchive();

@@ -122,7 +123,7 @@ public class ArchiveInstaller {
             return false;

         }

 

-        archiveFile = downloadFile(newArchive, osSdkRoot, monitor, forceHttp);

+        archiveFile = downloadFile(newArchive, osSdkRoot, cache, monitor, forceHttp);

         if (archiveFile != null) {

             // Unarchive calls the pre/postInstallHook methods.

             if (unarchive(archiveInfo, osSdkRoot, archiveFile, sdkManager, monitor)) {

@@ -143,6 +144,7 @@ public class ArchiveInstaller {
     @VisibleForTesting(visibility=Visibility.PRIVATE)

     protected File downloadFile(Archive archive,

             String osSdkRoot,

+            DownloadCache cache,

             ITaskMonitor monitor,

             boolean forceHttp) {

 

@@ -219,7 +221,7 @@ public class ArchiveInstaller {
             mFileOp.deleteFileOrFolder(tmpFile);

         }

 

-        if (fetchUrl(archive, tmpFile, link, pkgName, monitor)) {

+        if (fetchUrl(archive, tmpFile, link, pkgName, cache, monitor)) {

             // Fetching was successful, let's use this file.

             return tmpFile;

         } else {

@@ -303,12 +305,13 @@ public class ArchiveInstaller {
             File tmpFile,

             String urlString,

             String pkgName,

+            DownloadCache cache,

             ITaskMonitor monitor) {

 

         FileOutputStream os = null;

         InputStream is = null;

         try {

-            is = UrlOpener.openUrl(urlString, monitor);

+            is = cache.openDirectUrl(urlString, monitor);

             os = new FileOutputStream(tmpFile);

 

             MessageDigest digester = archive.getChecksumType().getMessageDigest();

diff --git a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/DownloadCache.java b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/DownloadCache.java
new file mode 100755
index 0000000..039e165
--- /dev/null
+++ b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/DownloadCache.java
@@ -0,0 +1,667 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.sdklib.internal.repository;
+
+import com.android.annotations.Nullable;
+import com.android.annotations.VisibleForTesting;
+import com.android.annotations.VisibleForTesting.Visibility;
+import com.android.prefs.AndroidLocation;
+import com.android.prefs.AndroidLocation.AndroidLocationException;
+import com.android.sdklib.SdkConstants;
+import com.android.sdklib.internal.repository.UrlOpener.CanceledByUserException;
+import com.android.util.Pair;
+
+import org.apache.http.Header;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.message.BasicHeader;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.Properties;
+import java.util.concurrent.atomic.AtomicInteger;
+
+
+/**
+ * A simple cache for the XML resources handled by the SDK Manager.
+ * <p/>
+ * Callers should use {@link #openDirectUrl(String, ITaskMonitor)} to download "large files"
+ * that should not be cached (like actual installation packages which are several MBs big)
+ * and call {@link #openCachedUrl(String, ITaskMonitor)} to download small XML files.
+ * <p/>
+ * The cache can work in 3 different strategies (direct is a pass-through, fresh-cache is the
+ * default and tries to update resources if they are older than 10 minutes by respecting
+ * either ETag or Last-Modified, and finally server-cache is a strategy to always serve
+ * cached entries if present.)
+ */
+public class DownloadCache {
+
+    /*
+     * HTTP/1.1 references:
+     * - Possible headers:
+     *     http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
+     * - Rules about conditional requests:
+     *     http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.3.4
+     * - Error codes:
+     *     http://www.w3.org/Protocols/rfc2616/rfc2616-sec6.html#sec6.1.1
+     */
+
+    private static final boolean DEBUG = System.getenv("SDKMAN_DEBUG_CACHE") != null;
+
+    /** Key for the Status-Code in the info properties. */
+    private static final String KEY_STATUS_CODE = "Status-Code";        //$NON-NLS-1$
+    /** Key for the URL in the info properties. */
+    private static final String KEY_URL = "URL";                        //$NON-NLS-1$
+
+    /** Prefix of binary files stored in the {@link SdkConstants#FD_CACHE} directory. */
+    private final static String BIN_FILE_PREFIX = "sdkbin-";            //$NON-NLS-1$
+    /** Prefix of meta info files stored in the {@link SdkConstants#FD_CACHE} directory. */
+    private final static String INFO_FILE_PREFIX = "sdkinf-";           //$NON-NLS-1$
+
+    /**
+     * Minimum time before we consider a cached entry is potentially stale.
+     * Expressed in milliseconds.
+     * <p/>
+     * When using the {@link Strategy#FRESH_CACHE}, the cache will not try to refresh
+     * a cached file if it's has been saved more recently than this time.
+     * When using the direct mode or the serve mode, the cache either doesn't serve
+     * cached files or always serves caches files so this expiration delay is not used.
+     * <p/>
+     * Default is 10 minutes.
+     * <p/>
+     * TODO: change for a dynamic preference later.
+     */
+    private final static long MIN_TIME_EXPIRED_MS =  10*60*1000;
+    /**
+     * Maximum time before we consider a cache entry to be stale.
+     * Expressed in milliseconds.
+     * <p/>
+     * When using the {@link Strategy#FRESH_CACHE}, entries that have no ETag
+     * or Last-Modified will be refreshed if their file timestamp is older than
+     * this value.
+     * <p/>
+     * Default is 4 hours.
+     * <p/>
+     * TODO: change for a dynamic preference later.
+     */
+    private final static long MAX_TIME_EXPIRED_MS = 4*60*60*1000;
+
+    /**
+     * The maximum file size we'll cache for "small" files.
+     * 640KB is more than enough and is already a stretch since these are read in memory.
+     * (The actual typical size of the files handled here is in the 4-64KB range.)
+     */
+    private final static int MAX_SMALL_FILE_SIZE = 640 * 1024;
+
+    /**
+     * HTTP Headers that are saved in an info file.
+     * For HTTP/1.1 header names, see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
+     */
+    private final static String[] INFO_HTTP_HEADERS = {
+        HttpHeaders.LAST_MODIFIED,
+        HttpHeaders.ETAG,
+        HttpHeaders.CONTENT_LENGTH,
+        HttpHeaders.DATE
+    };
+
+    private final Strategy mStrategy;
+    private final File mCacheRoot;
+
+    public enum Strategy {
+        /**
+         * If the files are available in the cache, serve them as-is, otherwise
+         * download them and return the cached version. No expiration or refresh
+         * is attempted if a file is in the cache.
+         */
+        SERVE_CACHE,
+        /**
+         * If the files are available in the cache, check if there's an update
+         * (either using an e-tag check or comparing to the default time expiration).
+         * If files have expired or are not in the cache then download them and return
+         * the cached version.
+         */
+        FRESH_CACHE,
+        /**
+         * Disables caching. URLs are always downloaded and returned directly.
+         * Downloaded streams aren't cached locally.
+         */
+        DIRECT
+    }
+
+    /** Creates a default instance of the URL cache */
+    public DownloadCache(Strategy strategy) {
+        mCacheRoot = initCacheRoot();
+        mStrategy = mCacheRoot == null ? Strategy.DIRECT : strategy;
+    }
+
+    public Strategy getStrategy() {
+        return mStrategy;
+    }
+
+    /**
+     * Returns the directory to be used as a cache.
+     * Creates it if necessary.
+     * Makes it possible to disable or override the cache location in unit tests.
+     *
+     * @return An existing directory to use as a cache root dir,
+     *   or null in case of error in which case the cache will be disabled.
+     */
+    @VisibleForTesting(visibility=Visibility.PRIVATE)
+    protected File initCacheRoot() {
+        try {
+            File root = new File(AndroidLocation.getFolder());
+            root = new File(root, SdkConstants.FD_CACHE);
+            if (!root.exists()) {
+                root.mkdirs();
+            }
+            return root;
+        } catch (AndroidLocationException e) {
+            // No root? Disable the cache.
+            return null;
+        }
+    }
+
+    /**
+     * Does a direct download of the given URL using {@link UrlOpener}.
+     * This does not check the download cache and does not attempt to cache the file.
+     * Instead the HttpClient library returns a progressive download stream.
+     * <p/>
+     * For details on realm authentication and user/password handling,
+     * check the underlying {@link UrlOpener#openUrl(String, ITaskMonitor, Header[])}
+     * documentation.
+     *
+     * @param urlString the URL string to be opened.
+     * @param monitor {@link ITaskMonitor} which is related to this URL
+     *            fetching.
+     * @return Returns an {@link InputStream} holding the URL content.
+     * @throws IOException Exception thrown when there are problems retrieving
+     *             the URL or its content.
+     * @throws CanceledByUserException Exception thrown if the user cancels the
+     *              authentication dialog.
+     */
+    public InputStream openDirectUrl(String urlString, ITaskMonitor monitor)
+            throws IOException, CanceledByUserException {
+        if (DEBUG) {
+            System.out.println(String.format("%s : Direct download", urlString)); //$NON-NLS-1$
+        }
+        Pair<InputStream, HttpResponse> result =
+            UrlOpener.openUrl(urlString, monitor, null /*headers*/);
+        return result.getFirst();
+    }
+
+    /**
+     * Downloads a small file, typically XML manifests.
+     * The current {@link Strategy} governs whether the file is served as-is
+     * from the cache, potentially updated first or directly downloaded.
+     * <p/>
+     * For large downloads (e.g. installable archives) please do not invoke the
+     * cache and instead use the {@link #openDirectUrl(String, ITaskMonitor)}
+     * method.
+     * <p/>
+     * For details on realm authentication and user/password handling,
+     * check the underlying {@link UrlOpener#openUrl(String, ITaskMonitor, Header[])}
+     * documentation.
+     *
+     * @param urlString the URL string to be opened.
+     * @param monitor {@link ITaskMonitor} which is related to this URL
+     *            fetching.
+     * @return Returns an {@link InputStream} holding the URL content.
+     * @throws IOException Exception thrown when there are problems retrieving
+     *             the URL or its content.
+     * @throws CanceledByUserException Exception thrown if the user cancels the
+     *              authentication dialog.
+     */
+    public InputStream openCachedUrl(String urlString, ITaskMonitor monitor)
+            throws IOException, CanceledByUserException {
+        // Don't cache in direct mode. Don't try to cache non-http URLs.
+        if (mStrategy == Strategy.DIRECT || !urlString.startsWith("http")) {        //$NON-NLS-1$
+            return openDirectUrl(urlString, monitor);
+        }
+
+        File cached = new File(mCacheRoot, getCacheFilename(urlString));
+        File info   = new File(mCacheRoot, getInfoFilename(cached.getName()));
+
+        boolean useCached = cached.exists();
+
+        if (useCached && mStrategy == Strategy.FRESH_CACHE) {
+            // Check whether the file should be served from the cache or
+            // refreshed first.
+
+            long cacheModifiedMs = cached.lastModified(); /* last mod time in epoch/millis */
+            boolean checkCache = true;
+
+            Properties props = readInfo(info);
+            if (props == null) {
+                // No properties, no chocolate for you.
+                useCached = false;
+            } else {
+                long minExpiration = System.currentTimeMillis() - MIN_TIME_EXPIRED_MS;
+                checkCache = cacheModifiedMs < minExpiration;
+
+                if (!checkCache && DEBUG) {
+                    System.out.println(String.format(
+                            "%s : Too fresh [%,d ms], not checking yet.",    //$NON-NLS-1$
+                            urlString, cacheModifiedMs - minExpiration));
+                }
+            }
+
+            if (useCached && checkCache) {
+                assert props != null;
+
+                // Right now we only support 200 codes and will requery all 404s.
+                String code = props.getProperty(KEY_STATUS_CODE, "");   //$NON-NLS-1$
+                useCached = Integer.toString(HttpStatus.SC_OK).equals(code);
+
+                if (!useCached && DEBUG) {
+                    System.out.println(String.format(
+                            "%s : cache disabled by code %s",           //$NON-NLS-1$
+                            urlString, code));
+                }
+
+                if (useCached) {
+                    // Do we have a valid Content-Length? If so, it should match the file size.
+                    try {
+                        long length = Long.parseLong(props.getProperty(HttpHeaders.CONTENT_LENGTH,
+                                                        "-1")); //$NON-NLS-1$
+                        if (length >= 0) {
+                            useCached = length == cached.length();
+
+                            if (!useCached && DEBUG) {
+                                System.out.println(String.format(
+                                    "%s : cache disabled by length mismatch %d, expected %d", //$NON-NLS-1$
+                                    urlString, length, cached.length()));
+                            }
+                        }
+                    } catch (NumberFormatException ignore) {}
+                }
+
+                if (useCached) {
+                    // Do we have an ETag and/or a Last-Modified?
+                    String etag = props.getProperty(HttpHeaders.ETAG);
+                    String lastMod = props.getProperty(HttpHeaders.LAST_MODIFIED);
+
+                    if (etag != null || lastMod != null) {
+                        // Details on how to use them is defined at
+                        // http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.3.4
+                        // Bottom line:
+                        // - if there's an ETag, it should be used first with an
+                        //   If-None-Match header. That's a strong comparison for HTTP/1.1 servers.
+                        // - otherwise use a Last-Modified if an If-Modified-Since header exists.
+                        // In this case, we place both and the rules indicates a spec-abiding
+                        // server should strongly match ETag and weakly the Modified-Since.
+
+                        // TODO there are some servers out there which report ETag/Last-Mod
+                        // yet don't honor them when presented with a precondition. In this
+                        // case we should identify it in the reply and invalidate ETag support
+                        // for these servers and instead fallback on the pure-timeout case below.
+
+                        AtomicInteger statusCode = new AtomicInteger(0);
+                        InputStream is = null;
+                        List<Header> headers = new ArrayList<Header>(2);
+
+                        if (etag != null) {
+                            headers.add(new BasicHeader(HttpHeaders.IF_NONE_MATCH, etag));
+                        }
+
+                        if (lastMod != null) {
+                            headers.add(new BasicHeader(HttpHeaders.IF_MODIFIED_SINCE, lastMod));
+                        }
+
+                        if (!headers.isEmpty()) {
+                            is = downloadAndCache(urlString, monitor, cached, info,
+                                    headers.toArray(new Header[headers.size()]),
+                                    statusCode);
+                        }
+
+                        if (is != null && statusCode.get() == HttpStatus.SC_OK) {
+                            // The resource was modified, the server said there was something
+                            // new, which has been cached. We can return that to the caller.
+                            return is;
+                        }
+
+                        // If we get here, we should have is == null and code
+                        // could be:
+                        // - 304 for not-modified -- same resource, still available, in
+                        //       which case we'll use the cached one.
+                        // - 404 -- resource doesn't exist anymore in which case there's
+                        //       no point in retrying.
+                        // - For any other code, just retry a download.
+
+                        if (is != null) {
+                            try {
+                                is.close();
+                            } catch (Exception ignore) {}
+                            is = null;
+                        }
+
+                        if (statusCode.get() == HttpStatus.SC_NOT_MODIFIED) {
+                            // Cached file was not modified.
+                            // Change its timestamp for the next MIN_TIME_EXPIRED_MS check.
+                            cached.setLastModified(System.currentTimeMillis());
+
+                            // At this point useCached==true so we'll return
+                            // the cached file below.
+                        } else {
+                            // URL fetch returned something other than 200 or 304.
+                            // For 404, we're done, no need to check the server again.
+                            // For all other codes, we'll retry a download below.
+                            useCached = false;
+                            if (statusCode.get() == HttpStatus.SC_NOT_FOUND) {
+                                return null;
+                            }
+                        }
+                    } else {
+                        // If we don't have an Etag nor Last-Modified, let's use a
+                        // basic file timestamp and compare to a 1 hour threshold.
+
+                        long maxExpiration = System.currentTimeMillis() - MAX_TIME_EXPIRED_MS;
+                        useCached = cacheModifiedMs >= maxExpiration;
+
+                        if (!useCached && DEBUG) {
+                            System.out.println(String.format(
+                                "[%1$s] cache disabled by timestamp %2$tD %2$tT < %3$tD %3$tT", //$NON-NLS-1$
+                                urlString, cacheModifiedMs, maxExpiration));
+                        }
+                    }
+                }
+            }
+        }
+
+        if (useCached) {
+            // The caller needs an InputStream that supports the reset() operation.
+            // The default FileInputStream does not, so load the file into a byte
+            // array and return that.
+            try {
+                InputStream is = readCachedFile(cached);
+                if (is != null) {
+                    if (DEBUG) {
+                        System.out.println(String.format("%s : Use cached file", urlString)); //$NON-NLS-1$
+                    }
+
+                    return is;
+                }
+            } catch (IOException ignore) {}
+        }
+
+        // If we're not using the cache, try to remove the cache and download again.
+        try {
+            cached.delete();
+            info.delete();
+        } catch (SecurityException ignore) {}
+
+        return downloadAndCache(urlString, monitor, cached, info,
+                null /*headers*/, null /*statusCode*/);
+    }
+
+    // --------------
+
+    private InputStream readCachedFile(File cached) throws IOException {
+        InputStream is = null;
+
+        int inc = 65536;
+        int curr = 0;
+        long len = cached.length();
+        assert len < Integer.MAX_VALUE;
+        if (len >= MAX_SMALL_FILE_SIZE) {
+            // This is supposed to cache small files, not 2+ GB files.
+            return null;
+        }
+        byte[] result = new byte[(int) (len > 0 ? len : inc)];
+
+        try {
+            is = new FileInputStream(cached);
+
+            int n;
+            while ((n = is.read(result, curr, result.length - curr)) != -1) {
+                curr += n;
+                if (curr == result.length) {
+                    byte[] temp = new byte[curr + inc];
+                    System.arraycopy(result, 0, temp, 0, curr);
+                    result = temp;
+                }
+            }
+
+            return new ByteArrayInputStream(result, 0, curr);
+
+        } finally {
+            if (is != null) {
+                try {
+                    is.close();
+                } catch (IOException ignore) {}
+            }
+        }
+    }
+
+    /**
+     * Download, cache and return as an in-memory byte stream.
+     * The download is only done if the server returns 200/OK.
+     * On success, store an info file next to the download with
+     * a few headers.
+     * <p/>
+     * This method deletes the cached file and the info file ONLY if it
+     * attempted a download and it failed to complete. It doesn't erase
+     * anything if there's no download because the server returned a 404
+     * or 304 or similar.
+     *
+     * @return An in-memory byte buffer input stream for the downloaded
+     *   and locally cached file, or null if nothing was downloaded
+     *   (including if it was a 304 Not-Modified status code.)
+     */
+    private InputStream downloadAndCache(
+            String urlString,
+            ITaskMonitor monitor,
+            File cached,
+            File info,
+            @Nullable Header[] headers,
+            @Nullable AtomicInteger outStatusCode)
+                throws FileNotFoundException, IOException, CanceledByUserException {
+        InputStream is = null;
+        OutputStream os = null;
+
+        int inc = 65536;
+        int curr = 0;
+        byte[] result = new byte[inc];
+
+        try {
+            Pair<InputStream, HttpResponse> r = UrlOpener.openUrl(urlString, monitor, headers);
+
+            is = r.getFirst();
+            HttpResponse response = r.getSecond();
+
+            if (DEBUG) {
+                System.out.println(String.format("%s : fetch: %s => %s",  //$NON-NLS-1$
+                        urlString,
+                        headers == null ? "" : Arrays.toString(headers),    //$NON-NLS-1$
+                        response.getStatusLine()));
+            }
+
+            int code = response.getStatusLine().getStatusCode();
+
+            if (outStatusCode != null) {
+                outStatusCode.set(code);
+            }
+
+            if (code != HttpStatus.SC_OK) {
+                // Only a 200 response code makes sense here.
+                // Even the other 20x codes should not apply, e.g. no content or partial
+                // content are not statuses we want to handle and should never happen.
+                // (see http://www.w3.org/Protocols/rfc2616/rfc2616-sec6.html#sec6.1.1 for list)
+                return null;
+            }
+
+            os = new FileOutputStream(cached);
+
+            int n;
+            while ((n = is.read(result, curr, result.length - curr)) != -1) {
+                if (os != null && n > 0) {
+                    os.write(result, curr, n);
+                }
+
+                curr += n;
+
+                if (os != null && curr > MAX_SMALL_FILE_SIZE) {
+                    // If the file size exceeds our "small file size" threshold,
+                    // stop caching. We don't want to fill the disk.
+                    try {
+                        os.close();
+                    } catch (IOException ignore) {}
+                    try {
+                        cached.delete();
+                        info.delete();
+                    } catch (SecurityException ignore) {}
+                    os = null;
+                }
+                if (curr == result.length) {
+                    byte[] temp = new byte[curr + inc];
+                    System.arraycopy(result, 0, temp, 0, curr);
+                    result = temp;
+                }
+            }
+
+            // Close the output stream, signaling it was stored properly.
+            if (os != null) {
+                try {
+                    os.close();
+                    os = null;
+
+                    saveInfo(urlString, response, info);
+                } catch (IOException ignore) {}
+            }
+
+            return new ByteArrayInputStream(result, 0, curr);
+
+        } finally {
+            if (is != null) {
+                try {
+                    is.close();
+                } catch (IOException ignore) {}
+            }
+            if (os != null) {
+                try {
+                    os.close();
+                } catch (IOException ignore) {}
+                // If we get here with the output stream not null, it means there
+                // was an issue and we don't want to keep that file. We'll try to
+                // delete it.
+                try {
+                    cached.delete();
+                    info.delete();
+                } catch (SecurityException ignore) {}
+            }
+        }
+    }
+
+    /**
+     * Saves part of the HTTP Response to the info file.
+     */
+    private void saveInfo(String urlString, HttpResponse response, File info) throws IOException {
+        Properties props = new Properties();
+
+        // we don't need the status code & URL right now.
+        // Save it in case we want to have it later (e.g. to differentiate 200 and 404.)
+        props.setProperty(KEY_URL, urlString);
+        props.setProperty(KEY_STATUS_CODE,
+                Integer.toString(response.getStatusLine().getStatusCode()));
+
+        for (String name : INFO_HTTP_HEADERS) {
+            Header h = response.getFirstHeader(name);
+            if (h != null) {
+                props.setProperty(name, h.getValue());
+            }
+        }
+
+        FileOutputStream os = null;
+        try {
+            os = new FileOutputStream(info);
+            props.store(os, "## Meta data for SDK Manager cache. Do not modify."); //$NON-NLS-1$
+        } finally {
+            if (os != null) {
+                os.close();
+            }
+        }
+    }
+
+    /**
+     * Reads the info properties file.
+     * @return The properties found or null if there's no file or it can't be read.
+     */
+    private Properties readInfo(File info) {
+        if (info.exists()) {
+            Properties props = new Properties();
+
+            InputStream is = null;
+            try {
+                is = new FileInputStream(info);
+                props.load(is);
+                return props;
+            } catch (IOException ignore) {
+            } finally {
+                if (is != null) {
+                    try {
+                        is.close();
+                    } catch (IOException ignore) {}
+                }
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Computes the cache filename for the given URL.
+     * The filename uses the {@link #BIN_FILE_PREFIX}, the full URL string's hashcode and
+     * a sanitized portion of the URL filename. The returned filename is never
+     * more than 64 characters to ensure maximum file system compatibility.
+     *
+     * @param urlString The download URL.
+     * @return A leaf filename for the cached download file.
+     */
+    private String getCacheFilename(String urlString) {
+        String hash = String.format("%08x", urlString.hashCode());
+
+        String leaf = urlString.toLowerCase(Locale.US);
+        if (leaf.length() >= 2) {
+            int index = urlString.lastIndexOf('/', leaf.length() - 2);
+            leaf = urlString.substring(index + 1);
+        }
+
+        leaf = leaf.replaceAll("[^a-z0-9_-]+", "_");
+        leaf = leaf.replaceAll("__+", "_");
+
+        leaf = hash + '-' + leaf;
+        int n = 64 - BIN_FILE_PREFIX.length();
+        if (leaf.length() > n) {
+            leaf = leaf.substring(0, n);
+        }
+
+        return BIN_FILE_PREFIX + leaf;
+    }
+
+    private String getInfoFilename(String cacheFilename) {
+        return cacheFilename.replaceFirst(BIN_FILE_PREFIX, INFO_FILE_PREFIX);
+    }
+}
diff --git a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/SdkSource.java b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/SdkSource.java
index 38417f7..749bc01 100755
--- a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/SdkSource.java
+++ b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/SdkSource.java
@@ -32,7 +32,6 @@ import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;

 import org.xml.sax.SAXParseException;

 

-import java.io.ByteArrayInputStream;

 import java.io.FileNotFoundException;

 import java.io.IOException;

 import java.io.InputStream;

@@ -192,7 +191,7 @@ public abstract class SdkSource implements IDescription, Comparable<SdkSource> {
     /**

      * Returns the list of known packages found by the last call to load().

      * This is null when the source hasn't been loaded yet -- caller should

-     * then call {@link #load(ITaskMonitor, boolean)} to load the packages.

+     * then call {@link #load} to load the packages.

      */

     public Package[] getPackages() {

         return mPackages;

@@ -306,7 +305,7 @@ public abstract class SdkSource implements IDescription, Comparable<SdkSource> {
      * null in case of error, in which case {@link #getFetchError()} can be used to an

      * error message.

      */

-    public void load(ITaskMonitor monitor, boolean forceHttp) {

+    public void load(DownloadCache cache, ITaskMonitor monitor, boolean forceHttp) {

 

         setDefaultDescription();

         monitor.setProgressMax(7);

@@ -338,7 +337,7 @@ public abstract class SdkSource implements IDescription, Comparable<SdkSource> {
         String[] defaultNames = getDefaultXmlFileUrls();

         String firstDefaultName = defaultNames.length > 0 ? defaultNames[0] : "";

 

-        InputStream xml = fetchUrl(url, monitor.createSubMonitor(1), exception);

+        InputStream xml = fetchUrl(url, cache, monitor.createSubMonitor(1), exception);

         if (xml != null) {

             int version = getXmlSchemaVersion(xml);

             if (version == 0) {

@@ -365,7 +364,7 @@ public abstract class SdkSource implements IDescription, Comparable<SdkSource> {
                 if (newUrl.equals(url)) {

                     continue;

                 }

-                xml = fetchUrl(newUrl, subMonitor.createSubMonitor(1), exception);

+                xml = fetchUrl(newUrl, cache, subMonitor.createSubMonitor(1), exception);

                 if (xml != null) {

                     int version = getXmlSchemaVersion(xml);

                     if (version == 0) {

@@ -394,7 +393,7 @@ public abstract class SdkSource implements IDescription, Comparable<SdkSource> {
             }

             url += firstDefaultName;

 

-            xml = fetchUrl(url, monitor.createSubMonitor(1), exception);

+            xml = fetchUrl(url, cache, monitor.createSubMonitor(1), exception);

             usingAlternateUrl = true;

         } else {

             monitor.incProgress(1);

@@ -467,7 +466,8 @@ public abstract class SdkSource implements IDescription, Comparable<SdkSource> {
                         }

                         url += firstDefaultName;

 

-                        xml = fetchUrl(url, subMonitor.createSubMonitor(1), null /* outException */);

+                        xml = fetchUrl(url, cache, subMonitor.createSubMonitor(1),

+                                null /* outException */);

                         subMonitor.incProgress(1);

                         // Loop to try the alternative document

                         if (xml != null) {

@@ -596,40 +596,12 @@ public abstract class SdkSource implements IDescription, Comparable<SdkSource> {
      *            happens during the fetch.

      * @see UrlOpener UrlOpener, which handles all URL logic.

      */

-    private InputStream fetchUrl(String urlString, ITaskMonitor monitor, Exception[] outException) {

+    private InputStream fetchUrl(String urlString,

+            DownloadCache cache,

+            ITaskMonitor monitor,

+            Exception[] outException) {

         try {

-

-            InputStream is = null;

-

-            int inc = 65536;

-            int curr = 0;

-            byte[] result = new byte[inc];

-

-            try {

-                is = UrlOpener.openUrl(urlString, monitor);

-

-                int n;

-                while ((n = is.read(result, curr, result.length - curr)) != -1) {

-                    curr += n;

-                    if (curr == result.length) {

-                        byte[] temp = new byte[curr + inc];

-                        System.arraycopy(result, 0, temp, 0, curr);

-                        result = temp;

-                    }

-                }

-

-                return new ByteArrayInputStream(result, 0, curr);

-

-            } finally {

-                if (is != null) {

-                    try {

-                        is.close();

-                    } catch (IOException e) {

-                        // pass

-                    }

-                }

-            }

-

+            return cache.openCachedUrl(urlString, monitor);

         } catch (Exception e) {

             if (outException != null) {

                 outException[0] = e;

diff --git a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/UrlOpener.java b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/UrlOpener.java
index 3114fc0..69d232a 100644
--- a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/UrlOpener.java
+++ b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/UrlOpener.java
@@ -16,9 +16,15 @@
 
 package com.android.sdklib.internal.repository;
 
+import com.android.annotations.NonNull;
+import com.android.annotations.Nullable;
+import com.android.util.Pair;
+
+import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
+import org.apache.http.ProtocolVersion;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.AuthState;
 import org.apache.http.auth.Credentials;
@@ -30,6 +36,7 @@ import org.apache.http.client.params.AuthPolicy;
 import org.apache.http.client.protocol.ClientContext;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.impl.conn.ProxySelectorRoutePlanner;
+import org.apache.http.message.BasicHttpResponse;
 import org.apache.http.protocol.BasicHttpContext;
 import org.apache.http.protocol.HttpContext;
 
@@ -46,9 +53,9 @@ import java.util.Map;
 
 /**
  * This class holds methods for adding URLs management.
- * @see #openUrl(String, ITaskMonitor)
+ * @see #openUrl(String, ITaskMonitor, Header[])
  */
-public class UrlOpener {
+class UrlOpener {
 
     public static class CanceledByUserException extends Exception {
         private static final long serialVersionUID = -7669346110926032403L;
@@ -79,35 +86,50 @@ public class UrlOpener {
      * - {@code http://hc.apache.org/httpcomponents-client-ga/} <br/>
      * - {@code http://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/ProxySelectorRoutePlanner.html}
      * <p/>
-     * There's a very simple <b>cache</b> implementation.
+     * There's a very simple realm cache implementation.
      * Login/Password for each realm are stored in a static {@link Map}.
-     * Before asking the user the method verifies if the information is already available in cache.
+     * Before asking the user the method verifies if the information is already
+     * available in the memory cache.
      *
      * @param url the URL string to be opened.
      * @param monitor {@link ITaskMonitor} which is related to this URL
      *            fetching.
-     * @return Returns an {@link InputStream} holding the URL content.
+     * @param headers An optional array of HTTP headers to use in the GET request.
+     * @return Returns an {@link InputStream} holding the URL content and
+     *      the HttpResponse (locale, headers and an status line).
+     *      This never returns null; an exception is thrown instead in case of
+     *      error or if the user canceled an authentication dialog.
      * @throws IOException Exception thrown when there are problems retrieving
      *             the URL or its content.
      * @throws CanceledByUserException Exception thrown if the user cancels the
      *              authentication dialog.
      */
-    static InputStream openUrl(String url, ITaskMonitor monitor)
+    static @NonNull Pair<InputStream, HttpResponse> openUrl(
+            @NonNull String url,
+            @NonNull ITaskMonitor monitor,
+            @Nullable Header[] headers)
         throws IOException, CanceledByUserException {
 
         try {
-            return openWithHttpClient(url, monitor);
+            return openWithHttpClient(url, monitor, headers);
 
         } catch (ClientProtocolException e) {
             // If the protocol is not supported by HttpClient (e.g. file:///),
             // revert to the standard java.net.Url.open
 
             URL u = new URL(url);
-            return u.openStream();
+            InputStream is = u.openStream();
+            HttpResponse response = new BasicHttpResponse(
+                    new ProtocolVersion(u.getProtocol(), 1, 0),
+                    200, "");
+            return Pair.of(is, response);
         }
     }
 
-    private static InputStream openWithHttpClient(String url, ITaskMonitor monitor)
+    private static @NonNull Pair<InputStream, HttpResponse> openWithHttpClient(
+            @NonNull String url,
+            @NonNull ITaskMonitor monitor,
+            Header[] headers)
             throws IOException, ClientProtocolException, CanceledByUserException {
         UserCredentials result = null;
         String realm = null;
@@ -117,7 +139,12 @@ public class UrlOpener {
 
         // create local execution context
         HttpContext localContext = new BasicHttpContext();
-        HttpGet httpget = new HttpGet(url);
+        final HttpGet httpGet = new HttpGet(url);
+        if (headers != null) {
+            for (Header header : headers) {
+                httpGet.addHeader(header);
+            }
+        }
 
         // retrieve local java configured network in case there is the need to
         // authenticate a proxy
@@ -143,7 +170,7 @@ public class UrlOpener {
         // loop while the response is being fetched
         while (trying) {
             // connect and get status code
-            HttpResponse response = httpClient.execute(httpget, localContext);
+            HttpResponse response = httpClient.execute(httpGet, localContext);
             int statusCode = response.getStatusLine().getStatusCode();
 
             // check whether any authentication is required
@@ -158,7 +185,7 @@ public class UrlOpener {
                 authenticationState = (AuthState) localContext
                         .getAttribute(ClientContext.PROXY_AUTH_STATE);
             }
-            if (statusCode == HttpStatus.SC_OK) {
+            if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_NOT_MODIFIED) {
                 // in case the status is OK and there is a realm and result,
                 // cache it
                 if (realm != null && result != null) {
@@ -226,8 +253,7 @@ public class UrlOpener {
                     // Note: don't use something like a BufferedHttpEntity since it would consume
                     // all content and store it in memory, resulting in an OutOfMemory exception
                     // on a large download.
-
-                    return new FilterInputStream(entity.getContent()) {
+                    InputStream is = new FilterInputStream(entity.getContent()) {
                         @Override
                         public void close() throws IOException {
                             // Since Http Client is no longer needed, close it.
@@ -240,7 +266,20 @@ public class UrlOpener {
                             super.close();
                         }
                     };
+
+                    HttpResponse outResponse = new BasicHttpResponse(response.getStatusLine());
+                    outResponse.setHeaders(response.getAllHeaders());
+                    outResponse.setLocale(response.getLocale());
+
+                    return Pair.of(is, outResponse);
                 }
+            } else if (statusCode == HttpStatus.SC_NOT_MODIFIED) {
+                // It's ok to not have an entity (e.g. nothing to download) for a 304
+                HttpResponse outResponse = new BasicHttpResponse(response.getStatusLine());
+                outResponse.setHeaders(response.getAllHeaders());
+                outResponse.setLocale(response.getLocale());
+
+                return Pair.of(null, outResponse);
             }
         }
 
diff --git a/sdkmanager/libs/sdklib/tests/src/com/android/sdklib/internal/repository/ArchiveInstallerTest.java b/sdkmanager/libs/sdklib/tests/src/com/android/sdklib/internal/repository/ArchiveInstallerTest.java
index 46390b5..8f42f8a 100755
--- a/sdkmanager/libs/sdklib/tests/src/com/android/sdklib/internal/repository/ArchiveInstallerTest.java
+++ b/sdkmanager/libs/sdklib/tests/src/com/android/sdklib/internal/repository/ArchiveInstallerTest.java
@@ -58,6 +58,7 @@ public class ArchiveInstallerTest extends TestCase {
         protected File downloadFile(
                 Archive archive,
                 String osSdkRoot,
+                DownloadCache cache,
                 ITaskMonitor monitor,
                 boolean forceHttp) {
             File file = mDownloadMap.get(archive);
@@ -102,7 +103,8 @@ public class ArchiveInstallerTest extends TestCase {
         MockEmptyPackage p = new MockEmptyPackage("testPkg");
         ArchiveReplacement ar = new ArchiveReplacement(p.getArchives()[0], null /*replaced*/);
 
-        assertFalse(mArchInst.install(ar, mSdkRoot, false /*forceHttp*/, mSdkMan, mMon));
+        assertFalse(mArchInst.install(ar, mSdkRoot, false /*forceHttp*/, mSdkMan,
+                null /*UrlCache*/, mMon));
         assertTrue(mMon.getCapturedLog().indexOf("Skipping already installed archive") != -1);
     }
 
@@ -116,7 +118,8 @@ public class ArchiveInstallerTest extends TestCase {
         mArchInst.setDownloadResponse(
                 p.getArchives()[0], createFile("/sdk", "tmp", "download1.zip"));
 
-        assertTrue(mArchInst.install(ar, mSdkRoot, false /*forceHttp*/, mSdkMan, mMon));
+        assertTrue(mArchInst.install(ar, mSdkRoot, false /*forceHttp*/, mSdkMan,
+                null /*UrlCache*/, mMon));
 
         // check what was created
         assertEquals(
@@ -162,7 +165,8 @@ public class ArchiveInstallerTest extends TestCase {
         mArchInst.setDownloadResponse(
                 newPkg.getArchives()[0], createFile("/sdk", "tmp", "download1.zip"));
 
-        assertTrue(mArchInst.install(ar, mSdkRoot, false /*forceHttp*/, mSdkMan, mMon));
+        assertTrue(mArchInst.install(ar, mSdkRoot, false /*forceHttp*/, mSdkMan,
+                null /*UrlCache*/, mMon));
 
         // check what was created
         assertEquals(
@@ -237,7 +241,8 @@ public class ArchiveInstallerTest extends TestCase {
         mArchInst.setDownloadResponse(
                 newPkg.getArchives()[0], createFile("/sdk", "tmp", "download1.zip"));
 
-        assertTrue(mArchInst.install(ar, mSdkRoot, false /*forceHttp*/, mSdkMan, mMon));
+        assertTrue(mArchInst.install(ar, mSdkRoot, false /*forceHttp*/, mSdkMan,
+                null /*UrlCache*/, mMon));
 
         // check what was created
         assertEquals(