diff options
author | Raphael Moll <ralf@android.com> | 2012-04-12 14:41:25 -0700 |
---|---|---|
committer | Raphael Moll <ralf@android.com> | 2012-04-16 15:04:14 -0700 |
commit | af832e8eeb4eb7e24a292ed912ce8eb7cc2a8233 (patch) | |
tree | b28f218a8a409dbee35786ea68df82ee9eea3068 /sdkmanager/libs/sdklib | |
parent | 72d16d222e7205cab473d01f483c534747c9026b (diff) | |
download | sdk-af832e8eeb4eb7e24a292ed912ce8eb7cc2a8233.zip sdk-af832e8eeb4eb7e24a292ed912ce8eb7cc2a8233.tar.gz sdk-af832e8eeb4eb7e24a292ed912ce8eb7cc2a8233.tar.bz2 |
SDK: primitive implementation of download cache.
It supports:
- A local binary cache + a few http headers are saved
- If ETag is present, generates a GET with If-None-Match
- If Last-Modified is present, generates a GET with If-Modified-Since
- Ability to configure the cache to be direct (don't cache),
or serve without checkout or serve with a server check.
- Doesn't check cached files if newer than 10 minutes.
- For servers with no ETag/LastModified support, check files
every 4 hours (no pref to change this yet.)
Change-Id: I515e77291fb6810453e82e73f6508cfc60b2f422
Diffstat (limited to 'sdkmanager/libs/sdklib')
8 files changed, 762 insertions, 102 deletions
diff --git a/sdkmanager/libs/sdklib/.classpath b/sdkmanager/libs/sdklib/.classpath index a82525e..3e6a435 100644 --- a/sdkmanager/libs/sdklib/.classpath +++ b/sdkmanager/libs/sdklib/.classpath @@ -5,13 +5,13 @@ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/> <classpathentry combineaccessrules="false" kind="src" path="/AndroidPrefs"/> <classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/3"/> - <classpathentry kind="var" path="ANDROID_SRC/prebuilt/common/commons-compress/commons-compress-1.0.jar"/> <classpathentry combineaccessrules="false" kind="src" path="/common"/> + <classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/commons-compress/commons-compress-1.0.jar"/> <classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/http-client/commons-codec-1.4.jar"/> <classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/http-client/commons-logging-1.1.1.jar"/> - <classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/http-client/httpclient-4.1.1.jar"/> - <classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/http-client/httpcore-4.1.jar"/> - <classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/http-client/httpmime-4.1.1.jar"/> + <classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/http-client/httpclient-4.1.1.jar" sourcepath="/ANDROID_SRC/prebuilts/tools/common/http-client/src/httpcomponents-client-4.1.1-src.zip"/> + <classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/http-client/httpcore-4.1.jar" sourcepath="/ANDROID_SRC/prebuilts/tools/common/http-client/src/httpcomponents-core-4.1-src.zip"/> + <classpathentry kind="var" path="ANDROID_SRC/prebuilts/tools/common/http-client/httpmime-4.1.1.jar" sourcepath="/ANDROID_SRC/prebuilts/tools/common/http-client/src/httpcomponents-client-4.1.1-src.zip"/> <classpathentry kind="var" path="ANDROID_SRC/prebuilt/common/mkidentity/mkidentity-prebuilt.jar"/> <classpathentry kind="output" path="bin"/> </classpath> diff --git a/sdkmanager/libs/sdklib/src/com/android/sdklib/SdkConstants.java b/sdkmanager/libs/sdklib/src/com/android/sdklib/SdkConstants.java index ebdc9c8..6e6c657 100644 --- a/sdkmanager/libs/sdklib/src/com/android/sdklib/SdkConstants.java +++ b/sdkmanager/libs/sdklib/src/com/android/sdklib/SdkConstants.java @@ -260,6 +260,9 @@ public final class SdkConstants { /** Name of the addon libs folder. */ public final static String FD_ADDON_LIBS = "libs"; //$NON-NLS-1$ + /** Name of the cache folder in the $HOME/.android. */ + public final static String FD_CACHE = "cache"; //$NON-NLS-1$ + /** Namespace for the resource XML, i.e. "http://schemas.android.com/apk/res/android" */ public final static String NS_RESOURCES = "http://schemas.android.com/apk/res/android"; //$NON-NLS-1$ diff --git a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/AddonsListFetcher.java b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/AddonsListFetcher.java index 919a30e..216e309 100755 --- a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/AddonsListFetcher.java +++ b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/AddonsListFetcher.java @@ -29,7 +29,6 @@ import org.xml.sax.InputSource; import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
-import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
@@ -82,13 +81,14 @@ public class AddonsListFetcher { /**
* Fetches the addons list from the given URL.
*
- * @param monitor A monitor to report errors. Cannot be null.
* @param url The URL of an XML file resource that conforms to the latest sdk-addons-list-N.xsd.
* For the default operation, use {@link SdkAddonsListConstants#URL_ADDON_LIST}.
* Cannot be null.
+ * @param cache The {@link DownloadCache} instance to use. Cannot be null.
+ * @param monitor A monitor to report errors. Cannot be null.
* @return An array of {@link Site} on success (possibly empty), or null on error.
*/
- public Site[] fetch(ITaskMonitor monitor, String url) {
+ public Site[] fetch(String url, DownloadCache cache, ITaskMonitor monitor) {
url = url == null ? "" : url.trim();
@@ -102,7 +102,7 @@ public class AddonsListFetcher { Document validatedDoc = null;
String validatedUri = null;
- ByteArrayInputStream xml = fetchUrl(url, monitor.createSubMonitor(1), exception);
+ InputStream xml = fetchUrl(url, cache, monitor.createSubMonitor(1), exception);
if (xml != null) {
monitor.setDescription("Validate XML");
@@ -187,41 +187,12 @@ public class AddonsListFetcher { * happens during the fetch.
* @see UrlOpener UrlOpener, which handles all URL logic.
*/
- private ByteArrayInputStream fetchUrl(String urlString, ITaskMonitor monitor,
+ private InputStream fetchUrl(String urlString,
+ DownloadCache cache,
+ ITaskMonitor monitor,
Exception[] outException) {
try {
-
- InputStream is = null;
-
- int inc = 65536;
- int curr = 0;
- byte[] result = new byte[inc];
-
- try {
- is = UrlOpener.openUrl(urlString, monitor);
-
- int n;
- while ((n = is.read(result, curr, result.length - curr)) != -1) {
- curr += n;
- if (curr == result.length) {
- byte[] temp = new byte[curr + inc];
- System.arraycopy(result, 0, temp, 0, curr);
- result = temp;
- }
- }
-
- return new ByteArrayInputStream(result, 0, curr);
-
- } finally {
- if (is != null) {
- try {
- is.close();
- } catch (IOException e) {
- // pass
- }
- }
- }
-
+ return cache.openCachedUrl(urlString, monitor);
} catch (Exception e) {
if (outException != null) {
outException[0] = e;
diff --git a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/ArchiveInstaller.java b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/ArchiveInstaller.java index 2e2396f..53fc140 100755 --- a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/ArchiveInstaller.java +++ b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/ArchiveInstaller.java @@ -96,6 +96,7 @@ public class ArchiveInstaller { String osSdkRoot,
boolean forceHttp,
SdkManager sdkManager,
+ DownloadCache cache,
ITaskMonitor monitor) {
Archive newArchive = archiveInfo.getNewArchive();
@@ -122,7 +123,7 @@ public class ArchiveInstaller { return false;
}
- archiveFile = downloadFile(newArchive, osSdkRoot, monitor, forceHttp);
+ archiveFile = downloadFile(newArchive, osSdkRoot, cache, monitor, forceHttp);
if (archiveFile != null) {
// Unarchive calls the pre/postInstallHook methods.
if (unarchive(archiveInfo, osSdkRoot, archiveFile, sdkManager, monitor)) {
@@ -143,6 +144,7 @@ public class ArchiveInstaller { @VisibleForTesting(visibility=Visibility.PRIVATE)
protected File downloadFile(Archive archive,
String osSdkRoot,
+ DownloadCache cache,
ITaskMonitor monitor,
boolean forceHttp) {
@@ -219,7 +221,7 @@ public class ArchiveInstaller { mFileOp.deleteFileOrFolder(tmpFile);
}
- if (fetchUrl(archive, tmpFile, link, pkgName, monitor)) {
+ if (fetchUrl(archive, tmpFile, link, pkgName, cache, monitor)) {
// Fetching was successful, let's use this file.
return tmpFile;
} else {
@@ -303,12 +305,13 @@ public class ArchiveInstaller { File tmpFile,
String urlString,
String pkgName,
+ DownloadCache cache,
ITaskMonitor monitor) {
FileOutputStream os = null;
InputStream is = null;
try {
- is = UrlOpener.openUrl(urlString, monitor);
+ is = cache.openDirectUrl(urlString, monitor);
os = new FileOutputStream(tmpFile);
MessageDigest digester = archive.getChecksumType().getMessageDigest();
diff --git a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/DownloadCache.java b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/DownloadCache.java new file mode 100755 index 0000000..039e165 --- /dev/null +++ b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/DownloadCache.java @@ -0,0 +1,667 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.sdklib.internal.repository; + +import com.android.annotations.Nullable; +import com.android.annotations.VisibleForTesting; +import com.android.annotations.VisibleForTesting.Visibility; +import com.android.prefs.AndroidLocation; +import com.android.prefs.AndroidLocation.AndroidLocationException; +import com.android.sdklib.SdkConstants; +import com.android.sdklib.internal.repository.UrlOpener.CanceledByUserException; +import com.android.util.Pair; + +import org.apache.http.Header; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.message.BasicHeader; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + + +/** + * A simple cache for the XML resources handled by the SDK Manager. + * <p/> + * Callers should use {@link #openDirectUrl(String, ITaskMonitor)} to download "large files" + * that should not be cached (like actual installation packages which are several MBs big) + * and call {@link #openCachedUrl(String, ITaskMonitor)} to download small XML files. + * <p/> + * The cache can work in 3 different strategies (direct is a pass-through, fresh-cache is the + * default and tries to update resources if they are older than 10 minutes by respecting + * either ETag or Last-Modified, and finally server-cache is a strategy to always serve + * cached entries if present.) + */ +public class DownloadCache { + + /* + * HTTP/1.1 references: + * - Possible headers: + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html + * - Rules about conditional requests: + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.3.4 + * - Error codes: + * http://www.w3.org/Protocols/rfc2616/rfc2616-sec6.html#sec6.1.1 + */ + + private static final boolean DEBUG = System.getenv("SDKMAN_DEBUG_CACHE") != null; + + /** Key for the Status-Code in the info properties. */ + private static final String KEY_STATUS_CODE = "Status-Code"; //$NON-NLS-1$ + /** Key for the URL in the info properties. */ + private static final String KEY_URL = "URL"; //$NON-NLS-1$ + + /** Prefix of binary files stored in the {@link SdkConstants#FD_CACHE} directory. */ + private final static String BIN_FILE_PREFIX = "sdkbin-"; //$NON-NLS-1$ + /** Prefix of meta info files stored in the {@link SdkConstants#FD_CACHE} directory. */ + private final static String INFO_FILE_PREFIX = "sdkinf-"; //$NON-NLS-1$ + + /** + * Minimum time before we consider a cached entry is potentially stale. + * Expressed in milliseconds. + * <p/> + * When using the {@link Strategy#FRESH_CACHE}, the cache will not try to refresh + * a cached file if it's has been saved more recently than this time. + * When using the direct mode or the serve mode, the cache either doesn't serve + * cached files or always serves caches files so this expiration delay is not used. + * <p/> + * Default is 10 minutes. + * <p/> + * TODO: change for a dynamic preference later. + */ + private final static long MIN_TIME_EXPIRED_MS = 10*60*1000; + /** + * Maximum time before we consider a cache entry to be stale. + * Expressed in milliseconds. + * <p/> + * When using the {@link Strategy#FRESH_CACHE}, entries that have no ETag + * or Last-Modified will be refreshed if their file timestamp is older than + * this value. + * <p/> + * Default is 4 hours. + * <p/> + * TODO: change for a dynamic preference later. + */ + private final static long MAX_TIME_EXPIRED_MS = 4*60*60*1000; + + /** + * The maximum file size we'll cache for "small" files. + * 640KB is more than enough and is already a stretch since these are read in memory. + * (The actual typical size of the files handled here is in the 4-64KB range.) + */ + private final static int MAX_SMALL_FILE_SIZE = 640 * 1024; + + /** + * HTTP Headers that are saved in an info file. + * For HTTP/1.1 header names, see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html + */ + private final static String[] INFO_HTTP_HEADERS = { + HttpHeaders.LAST_MODIFIED, + HttpHeaders.ETAG, + HttpHeaders.CONTENT_LENGTH, + HttpHeaders.DATE + }; + + private final Strategy mStrategy; + private final File mCacheRoot; + + public enum Strategy { + /** + * If the files are available in the cache, serve them as-is, otherwise + * download them and return the cached version. No expiration or refresh + * is attempted if a file is in the cache. + */ + SERVE_CACHE, + /** + * If the files are available in the cache, check if there's an update + * (either using an e-tag check or comparing to the default time expiration). + * If files have expired or are not in the cache then download them and return + * the cached version. + */ + FRESH_CACHE, + /** + * Disables caching. URLs are always downloaded and returned directly. + * Downloaded streams aren't cached locally. + */ + DIRECT + } + + /** Creates a default instance of the URL cache */ + public DownloadCache(Strategy strategy) { + mCacheRoot = initCacheRoot(); + mStrategy = mCacheRoot == null ? Strategy.DIRECT : strategy; + } + + public Strategy getStrategy() { + return mStrategy; + } + + /** + * Returns the directory to be used as a cache. + * Creates it if necessary. + * Makes it possible to disable or override the cache location in unit tests. + * + * @return An existing directory to use as a cache root dir, + * or null in case of error in which case the cache will be disabled. + */ + @VisibleForTesting(visibility=Visibility.PRIVATE) + protected File initCacheRoot() { + try { + File root = new File(AndroidLocation.getFolder()); + root = new File(root, SdkConstants.FD_CACHE); + if (!root.exists()) { + root.mkdirs(); + } + return root; + } catch (AndroidLocationException e) { + // No root? Disable the cache. + return null; + } + } + + /** + * Does a direct download of the given URL using {@link UrlOpener}. + * This does not check the download cache and does not attempt to cache the file. + * Instead the HttpClient library returns a progressive download stream. + * <p/> + * For details on realm authentication and user/password handling, + * check the underlying {@link UrlOpener#openUrl(String, ITaskMonitor, Header[])} + * documentation. + * + * @param urlString the URL string to be opened. + * @param monitor {@link ITaskMonitor} which is related to this URL + * fetching. + * @return Returns an {@link InputStream} holding the URL content. + * @throws IOException Exception thrown when there are problems retrieving + * the URL or its content. + * @throws CanceledByUserException Exception thrown if the user cancels the + * authentication dialog. + */ + public InputStream openDirectUrl(String urlString, ITaskMonitor monitor) + throws IOException, CanceledByUserException { + if (DEBUG) { + System.out.println(String.format("%s : Direct download", urlString)); //$NON-NLS-1$ + } + Pair<InputStream, HttpResponse> result = + UrlOpener.openUrl(urlString, monitor, null /*headers*/); + return result.getFirst(); + } + + /** + * Downloads a small file, typically XML manifests. + * The current {@link Strategy} governs whether the file is served as-is + * from the cache, potentially updated first or directly downloaded. + * <p/> + * For large downloads (e.g. installable archives) please do not invoke the + * cache and instead use the {@link #openDirectUrl(String, ITaskMonitor)} + * method. + * <p/> + * For details on realm authentication and user/password handling, + * check the underlying {@link UrlOpener#openUrl(String, ITaskMonitor, Header[])} + * documentation. + * + * @param urlString the URL string to be opened. + * @param monitor {@link ITaskMonitor} which is related to this URL + * fetching. + * @return Returns an {@link InputStream} holding the URL content. + * @throws IOException Exception thrown when there are problems retrieving + * the URL or its content. + * @throws CanceledByUserException Exception thrown if the user cancels the + * authentication dialog. + */ + public InputStream openCachedUrl(String urlString, ITaskMonitor monitor) + throws IOException, CanceledByUserException { + // Don't cache in direct mode. Don't try to cache non-http URLs. + if (mStrategy == Strategy.DIRECT || !urlString.startsWith("http")) { //$NON-NLS-1$ + return openDirectUrl(urlString, monitor); + } + + File cached = new File(mCacheRoot, getCacheFilename(urlString)); + File info = new File(mCacheRoot, getInfoFilename(cached.getName())); + + boolean useCached = cached.exists(); + + if (useCached && mStrategy == Strategy.FRESH_CACHE) { + // Check whether the file should be served from the cache or + // refreshed first. + + long cacheModifiedMs = cached.lastModified(); /* last mod time in epoch/millis */ + boolean checkCache = true; + + Properties props = readInfo(info); + if (props == null) { + // No properties, no chocolate for you. + useCached = false; + } else { + long minExpiration = System.currentTimeMillis() - MIN_TIME_EXPIRED_MS; + checkCache = cacheModifiedMs < minExpiration; + + if (!checkCache && DEBUG) { + System.out.println(String.format( + "%s : Too fresh [%,d ms], not checking yet.", //$NON-NLS-1$ + urlString, cacheModifiedMs - minExpiration)); + } + } + + if (useCached && checkCache) { + assert props != null; + + // Right now we only support 200 codes and will requery all 404s. + String code = props.getProperty(KEY_STATUS_CODE, ""); //$NON-NLS-1$ + useCached = Integer.toString(HttpStatus.SC_OK).equals(code); + + if (!useCached && DEBUG) { + System.out.println(String.format( + "%s : cache disabled by code %s", //$NON-NLS-1$ + urlString, code)); + } + + if (useCached) { + // Do we have a valid Content-Length? If so, it should match the file size. + try { + long length = Long.parseLong(props.getProperty(HttpHeaders.CONTENT_LENGTH, + "-1")); //$NON-NLS-1$ + if (length >= 0) { + useCached = length == cached.length(); + + if (!useCached && DEBUG) { + System.out.println(String.format( + "%s : cache disabled by length mismatch %d, expected %d", //$NON-NLS-1$ + urlString, length, cached.length())); + } + } + } catch (NumberFormatException ignore) {} + } + + if (useCached) { + // Do we have an ETag and/or a Last-Modified? + String etag = props.getProperty(HttpHeaders.ETAG); + String lastMod = props.getProperty(HttpHeaders.LAST_MODIFIED); + + if (etag != null || lastMod != null) { + // Details on how to use them is defined at + // http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.3.4 + // Bottom line: + // - if there's an ETag, it should be used first with an + // If-None-Match header. That's a strong comparison for HTTP/1.1 servers. + // - otherwise use a Last-Modified if an If-Modified-Since header exists. + // In this case, we place both and the rules indicates a spec-abiding + // server should strongly match ETag and weakly the Modified-Since. + + // TODO there are some servers out there which report ETag/Last-Mod + // yet don't honor them when presented with a precondition. In this + // case we should identify it in the reply and invalidate ETag support + // for these servers and instead fallback on the pure-timeout case below. + + AtomicInteger statusCode = new AtomicInteger(0); + InputStream is = null; + List<Header> headers = new ArrayList<Header>(2); + + if (etag != null) { + headers.add(new BasicHeader(HttpHeaders.IF_NONE_MATCH, etag)); + } + + if (lastMod != null) { + headers.add(new BasicHeader(HttpHeaders.IF_MODIFIED_SINCE, lastMod)); + } + + if (!headers.isEmpty()) { + is = downloadAndCache(urlString, monitor, cached, info, + headers.toArray(new Header[headers.size()]), + statusCode); + } + + if (is != null && statusCode.get() == HttpStatus.SC_OK) { + // The resource was modified, the server said there was something + // new, which has been cached. We can return that to the caller. + return is; + } + + // If we get here, we should have is == null and code + // could be: + // - 304 for not-modified -- same resource, still available, in + // which case we'll use the cached one. + // - 404 -- resource doesn't exist anymore in which case there's + // no point in retrying. + // - For any other code, just retry a download. + + if (is != null) { + try { + is.close(); + } catch (Exception ignore) {} + is = null; + } + + if (statusCode.get() == HttpStatus.SC_NOT_MODIFIED) { + // Cached file was not modified. + // Change its timestamp for the next MIN_TIME_EXPIRED_MS check. + cached.setLastModified(System.currentTimeMillis()); + + // At this point useCached==true so we'll return + // the cached file below. + } else { + // URL fetch returned something other than 200 or 304. + // For 404, we're done, no need to check the server again. + // For all other codes, we'll retry a download below. + useCached = false; + if (statusCode.get() == HttpStatus.SC_NOT_FOUND) { + return null; + } + } + } else { + // If we don't have an Etag nor Last-Modified, let's use a + // basic file timestamp and compare to a 1 hour threshold. + + long maxExpiration = System.currentTimeMillis() - MAX_TIME_EXPIRED_MS; + useCached = cacheModifiedMs >= maxExpiration; + + if (!useCached && DEBUG) { + System.out.println(String.format( + "[%1$s] cache disabled by timestamp %2$tD %2$tT < %3$tD %3$tT", //$NON-NLS-1$ + urlString, cacheModifiedMs, maxExpiration)); + } + } + } + } + } + + if (useCached) { + // The caller needs an InputStream that supports the reset() operation. + // The default FileInputStream does not, so load the file into a byte + // array and return that. + try { + InputStream is = readCachedFile(cached); + if (is != null) { + if (DEBUG) { + System.out.println(String.format("%s : Use cached file", urlString)); //$NON-NLS-1$ + } + + return is; + } + } catch (IOException ignore) {} + } + + // If we're not using the cache, try to remove the cache and download again. + try { + cached.delete(); + info.delete(); + } catch (SecurityException ignore) {} + + return downloadAndCache(urlString, monitor, cached, info, + null /*headers*/, null /*statusCode*/); + } + + // -------------- + + private InputStream readCachedFile(File cached) throws IOException { + InputStream is = null; + + int inc = 65536; + int curr = 0; + long len = cached.length(); + assert len < Integer.MAX_VALUE; + if (len >= MAX_SMALL_FILE_SIZE) { + // This is supposed to cache small files, not 2+ GB files. + return null; + } + byte[] result = new byte[(int) (len > 0 ? len : inc)]; + + try { + is = new FileInputStream(cached); + + int n; + while ((n = is.read(result, curr, result.length - curr)) != -1) { + curr += n; + if (curr == result.length) { + byte[] temp = new byte[curr + inc]; + System.arraycopy(result, 0, temp, 0, curr); + result = temp; + } + } + + return new ByteArrayInputStream(result, 0, curr); + + } finally { + if (is != null) { + try { + is.close(); + } catch (IOException ignore) {} + } + } + } + + /** + * Download, cache and return as an in-memory byte stream. + * The download is only done if the server returns 200/OK. + * On success, store an info file next to the download with + * a few headers. + * <p/> + * This method deletes the cached file and the info file ONLY if it + * attempted a download and it failed to complete. It doesn't erase + * anything if there's no download because the server returned a 404 + * or 304 or similar. + * + * @return An in-memory byte buffer input stream for the downloaded + * and locally cached file, or null if nothing was downloaded + * (including if it was a 304 Not-Modified status code.) + */ + private InputStream downloadAndCache( + String urlString, + ITaskMonitor monitor, + File cached, + File info, + @Nullable Header[] headers, + @Nullable AtomicInteger outStatusCode) + throws FileNotFoundException, IOException, CanceledByUserException { + InputStream is = null; + OutputStream os = null; + + int inc = 65536; + int curr = 0; + byte[] result = new byte[inc]; + + try { + Pair<InputStream, HttpResponse> r = UrlOpener.openUrl(urlString, monitor, headers); + + is = r.getFirst(); + HttpResponse response = r.getSecond(); + + if (DEBUG) { + System.out.println(String.format("%s : fetch: %s => %s", //$NON-NLS-1$ + urlString, + headers == null ? "" : Arrays.toString(headers), //$NON-NLS-1$ + response.getStatusLine())); + } + + int code = response.getStatusLine().getStatusCode(); + + if (outStatusCode != null) { + outStatusCode.set(code); + } + + if (code != HttpStatus.SC_OK) { + // Only a 200 response code makes sense here. + // Even the other 20x codes should not apply, e.g. no content or partial + // content are not statuses we want to handle and should never happen. + // (see http://www.w3.org/Protocols/rfc2616/rfc2616-sec6.html#sec6.1.1 for list) + return null; + } + + os = new FileOutputStream(cached); + + int n; + while ((n = is.read(result, curr, result.length - curr)) != -1) { + if (os != null && n > 0) { + os.write(result, curr, n); + } + + curr += n; + + if (os != null && curr > MAX_SMALL_FILE_SIZE) { + // If the file size exceeds our "small file size" threshold, + // stop caching. We don't want to fill the disk. + try { + os.close(); + } catch (IOException ignore) {} + try { + cached.delete(); + info.delete(); + } catch (SecurityException ignore) {} + os = null; + } + if (curr == result.length) { + byte[] temp = new byte[curr + inc]; + System.arraycopy(result, 0, temp, 0, curr); + result = temp; + } + } + + // Close the output stream, signaling it was stored properly. + if (os != null) { + try { + os.close(); + os = null; + + saveInfo(urlString, response, info); + } catch (IOException ignore) {} + } + + return new ByteArrayInputStream(result, 0, curr); + + } finally { + if (is != null) { + try { + is.close(); + } catch (IOException ignore) {} + } + if (os != null) { + try { + os.close(); + } catch (IOException ignore) {} + // If we get here with the output stream not null, it means there + // was an issue and we don't want to keep that file. We'll try to + // delete it. + try { + cached.delete(); + info.delete(); + } catch (SecurityException ignore) {} + } + } + } + + /** + * Saves part of the HTTP Response to the info file. + */ + private void saveInfo(String urlString, HttpResponse response, File info) throws IOException { + Properties props = new Properties(); + + // we don't need the status code & URL right now. + // Save it in case we want to have it later (e.g. to differentiate 200 and 404.) + props.setProperty(KEY_URL, urlString); + props.setProperty(KEY_STATUS_CODE, + Integer.toString(response.getStatusLine().getStatusCode())); + + for (String name : INFO_HTTP_HEADERS) { + Header h = response.getFirstHeader(name); + if (h != null) { + props.setProperty(name, h.getValue()); + } + } + + FileOutputStream os = null; + try { + os = new FileOutputStream(info); + props.store(os, "## Meta data for SDK Manager cache. Do not modify."); //$NON-NLS-1$ + } finally { + if (os != null) { + os.close(); + } + } + } + + /** + * Reads the info properties file. + * @return The properties found or null if there's no file or it can't be read. + */ + private Properties readInfo(File info) { + if (info.exists()) { + Properties props = new Properties(); + + InputStream is = null; + try { + is = new FileInputStream(info); + props.load(is); + return props; + } catch (IOException ignore) { + } finally { + if (is != null) { + try { + is.close(); + } catch (IOException ignore) {} + } + } + } + return null; + } + + /** + * Computes the cache filename for the given URL. + * The filename uses the {@link #BIN_FILE_PREFIX}, the full URL string's hashcode and + * a sanitized portion of the URL filename. The returned filename is never + * more than 64 characters to ensure maximum file system compatibility. + * + * @param urlString The download URL. + * @return A leaf filename for the cached download file. + */ + private String getCacheFilename(String urlString) { + String hash = String.format("%08x", urlString.hashCode()); + + String leaf = urlString.toLowerCase(Locale.US); + if (leaf.length() >= 2) { + int index = urlString.lastIndexOf('/', leaf.length() - 2); + leaf = urlString.substring(index + 1); + } + + leaf = leaf.replaceAll("[^a-z0-9_-]+", "_"); + leaf = leaf.replaceAll("__+", "_"); + + leaf = hash + '-' + leaf; + int n = 64 - BIN_FILE_PREFIX.length(); + if (leaf.length() > n) { + leaf = leaf.substring(0, n); + } + + return BIN_FILE_PREFIX + leaf; + } + + private String getInfoFilename(String cacheFilename) { + return cacheFilename.replaceFirst(BIN_FILE_PREFIX, INFO_FILE_PREFIX); + } +} diff --git a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/SdkSource.java b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/SdkSource.java index 38417f7..749bc01 100755 --- a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/SdkSource.java +++ b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/SdkSource.java @@ -32,7 +32,6 @@ import org.xml.sax.InputSource; import org.xml.sax.SAXException;
import org.xml.sax.SAXParseException;
-import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
@@ -192,7 +191,7 @@ public abstract class SdkSource implements IDescription, Comparable<SdkSource> { /**
* Returns the list of known packages found by the last call to load().
* This is null when the source hasn't been loaded yet -- caller should
- * then call {@link #load(ITaskMonitor, boolean)} to load the packages.
+ * then call {@link #load} to load the packages.
*/
public Package[] getPackages() {
return mPackages;
@@ -306,7 +305,7 @@ public abstract class SdkSource implements IDescription, Comparable<SdkSource> { * null in case of error, in which case {@link #getFetchError()} can be used to an
* error message.
*/
- public void load(ITaskMonitor monitor, boolean forceHttp) {
+ public void load(DownloadCache cache, ITaskMonitor monitor, boolean forceHttp) {
setDefaultDescription();
monitor.setProgressMax(7);
@@ -338,7 +337,7 @@ public abstract class SdkSource implements IDescription, Comparable<SdkSource> { String[] defaultNames = getDefaultXmlFileUrls();
String firstDefaultName = defaultNames.length > 0 ? defaultNames[0] : "";
- InputStream xml = fetchUrl(url, monitor.createSubMonitor(1), exception);
+ InputStream xml = fetchUrl(url, cache, monitor.createSubMonitor(1), exception);
if (xml != null) {
int version = getXmlSchemaVersion(xml);
if (version == 0) {
@@ -365,7 +364,7 @@ public abstract class SdkSource implements IDescription, Comparable<SdkSource> { if (newUrl.equals(url)) {
continue;
}
- xml = fetchUrl(newUrl, subMonitor.createSubMonitor(1), exception);
+ xml = fetchUrl(newUrl, cache, subMonitor.createSubMonitor(1), exception);
if (xml != null) {
int version = getXmlSchemaVersion(xml);
if (version == 0) {
@@ -394,7 +393,7 @@ public abstract class SdkSource implements IDescription, Comparable<SdkSource> { }
url += firstDefaultName;
- xml = fetchUrl(url, monitor.createSubMonitor(1), exception);
+ xml = fetchUrl(url, cache, monitor.createSubMonitor(1), exception);
usingAlternateUrl = true;
} else {
monitor.incProgress(1);
@@ -467,7 +466,8 @@ public abstract class SdkSource implements IDescription, Comparable<SdkSource> { }
url += firstDefaultName;
- xml = fetchUrl(url, subMonitor.createSubMonitor(1), null /* outException */);
+ xml = fetchUrl(url, cache, subMonitor.createSubMonitor(1),
+ null /* outException */);
subMonitor.incProgress(1);
// Loop to try the alternative document
if (xml != null) {
@@ -596,40 +596,12 @@ public abstract class SdkSource implements IDescription, Comparable<SdkSource> { * happens during the fetch.
* @see UrlOpener UrlOpener, which handles all URL logic.
*/
- private InputStream fetchUrl(String urlString, ITaskMonitor monitor, Exception[] outException) {
+ private InputStream fetchUrl(String urlString,
+ DownloadCache cache,
+ ITaskMonitor monitor,
+ Exception[] outException) {
try {
-
- InputStream is = null;
-
- int inc = 65536;
- int curr = 0;
- byte[] result = new byte[inc];
-
- try {
- is = UrlOpener.openUrl(urlString, monitor);
-
- int n;
- while ((n = is.read(result, curr, result.length - curr)) != -1) {
- curr += n;
- if (curr == result.length) {
- byte[] temp = new byte[curr + inc];
- System.arraycopy(result, 0, temp, 0, curr);
- result = temp;
- }
- }
-
- return new ByteArrayInputStream(result, 0, curr);
-
- } finally {
- if (is != null) {
- try {
- is.close();
- } catch (IOException e) {
- // pass
- }
- }
- }
-
+ return cache.openCachedUrl(urlString, monitor);
} catch (Exception e) {
if (outException != null) {
outException[0] = e;
diff --git a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/UrlOpener.java b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/UrlOpener.java index 3114fc0..69d232a 100644 --- a/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/UrlOpener.java +++ b/sdkmanager/libs/sdklib/src/com/android/sdklib/internal/repository/UrlOpener.java @@ -16,9 +16,15 @@ package com.android.sdklib.internal.repository; +import com.android.annotations.NonNull; +import com.android.annotations.Nullable; +import com.android.util.Pair; + +import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; +import org.apache.http.ProtocolVersion; import org.apache.http.auth.AuthScope; import org.apache.http.auth.AuthState; import org.apache.http.auth.Credentials; @@ -30,6 +36,7 @@ import org.apache.http.client.params.AuthPolicy; import org.apache.http.client.protocol.ClientContext; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.conn.ProxySelectorRoutePlanner; +import org.apache.http.message.BasicHttpResponse; import org.apache.http.protocol.BasicHttpContext; import org.apache.http.protocol.HttpContext; @@ -46,9 +53,9 @@ import java.util.Map; /** * This class holds methods for adding URLs management. - * @see #openUrl(String, ITaskMonitor) + * @see #openUrl(String, ITaskMonitor, Header[]) */ -public class UrlOpener { +class UrlOpener { public static class CanceledByUserException extends Exception { private static final long serialVersionUID = -7669346110926032403L; @@ -79,35 +86,50 @@ public class UrlOpener { * - {@code http://hc.apache.org/httpcomponents-client-ga/} <br/> * - {@code http://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/ProxySelectorRoutePlanner.html} * <p/> - * There's a very simple <b>cache</b> implementation. + * There's a very simple realm cache implementation. * Login/Password for each realm are stored in a static {@link Map}. - * Before asking the user the method verifies if the information is already available in cache. + * Before asking the user the method verifies if the information is already + * available in the memory cache. * * @param url the URL string to be opened. * @param monitor {@link ITaskMonitor} which is related to this URL * fetching. - * @return Returns an {@link InputStream} holding the URL content. + * @param headers An optional array of HTTP headers to use in the GET request. + * @return Returns an {@link InputStream} holding the URL content and + * the HttpResponse (locale, headers and an status line). + * This never returns null; an exception is thrown instead in case of + * error or if the user canceled an authentication dialog. * @throws IOException Exception thrown when there are problems retrieving * the URL or its content. * @throws CanceledByUserException Exception thrown if the user cancels the * authentication dialog. */ - static InputStream openUrl(String url, ITaskMonitor monitor) + static @NonNull Pair<InputStream, HttpResponse> openUrl( + @NonNull String url, + @NonNull ITaskMonitor monitor, + @Nullable Header[] headers) throws IOException, CanceledByUserException { try { - return openWithHttpClient(url, monitor); + return openWithHttpClient(url, monitor, headers); } catch (ClientProtocolException e) { // If the protocol is not supported by HttpClient (e.g. file:///), // revert to the standard java.net.Url.open URL u = new URL(url); - return u.openStream(); + InputStream is = u.openStream(); + HttpResponse response = new BasicHttpResponse( + new ProtocolVersion(u.getProtocol(), 1, 0), + 200, ""); + return Pair.of(is, response); } } - private static InputStream openWithHttpClient(String url, ITaskMonitor monitor) + private static @NonNull Pair<InputStream, HttpResponse> openWithHttpClient( + @NonNull String url, + @NonNull ITaskMonitor monitor, + Header[] headers) throws IOException, ClientProtocolException, CanceledByUserException { UserCredentials result = null; String realm = null; @@ -117,7 +139,12 @@ public class UrlOpener { // create local execution context HttpContext localContext = new BasicHttpContext(); - HttpGet httpget = new HttpGet(url); + final HttpGet httpGet = new HttpGet(url); + if (headers != null) { + for (Header header : headers) { + httpGet.addHeader(header); + } + } // retrieve local java configured network in case there is the need to // authenticate a proxy @@ -143,7 +170,7 @@ public class UrlOpener { // loop while the response is being fetched while (trying) { // connect and get status code - HttpResponse response = httpClient.execute(httpget, localContext); + HttpResponse response = httpClient.execute(httpGet, localContext); int statusCode = response.getStatusLine().getStatusCode(); // check whether any authentication is required @@ -158,7 +185,7 @@ public class UrlOpener { authenticationState = (AuthState) localContext .getAttribute(ClientContext.PROXY_AUTH_STATE); } - if (statusCode == HttpStatus.SC_OK) { + if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_NOT_MODIFIED) { // in case the status is OK and there is a realm and result, // cache it if (realm != null && result != null) { @@ -226,8 +253,7 @@ public class UrlOpener { // Note: don't use something like a BufferedHttpEntity since it would consume // all content and store it in memory, resulting in an OutOfMemory exception // on a large download. - - return new FilterInputStream(entity.getContent()) { + InputStream is = new FilterInputStream(entity.getContent()) { @Override public void close() throws IOException { // Since Http Client is no longer needed, close it. @@ -240,7 +266,20 @@ public class UrlOpener { super.close(); } }; + + HttpResponse outResponse = new BasicHttpResponse(response.getStatusLine()); + outResponse.setHeaders(response.getAllHeaders()); + outResponse.setLocale(response.getLocale()); + + return Pair.of(is, outResponse); } + } else if (statusCode == HttpStatus.SC_NOT_MODIFIED) { + // It's ok to not have an entity (e.g. nothing to download) for a 304 + HttpResponse outResponse = new BasicHttpResponse(response.getStatusLine()); + outResponse.setHeaders(response.getAllHeaders()); + outResponse.setLocale(response.getLocale()); + + return Pair.of(null, outResponse); } } diff --git a/sdkmanager/libs/sdklib/tests/src/com/android/sdklib/internal/repository/ArchiveInstallerTest.java b/sdkmanager/libs/sdklib/tests/src/com/android/sdklib/internal/repository/ArchiveInstallerTest.java index 46390b5..8f42f8a 100755 --- a/sdkmanager/libs/sdklib/tests/src/com/android/sdklib/internal/repository/ArchiveInstallerTest.java +++ b/sdkmanager/libs/sdklib/tests/src/com/android/sdklib/internal/repository/ArchiveInstallerTest.java @@ -58,6 +58,7 @@ public class ArchiveInstallerTest extends TestCase { protected File downloadFile( Archive archive, String osSdkRoot, + DownloadCache cache, ITaskMonitor monitor, boolean forceHttp) { File file = mDownloadMap.get(archive); @@ -102,7 +103,8 @@ public class ArchiveInstallerTest extends TestCase { MockEmptyPackage p = new MockEmptyPackage("testPkg"); ArchiveReplacement ar = new ArchiveReplacement(p.getArchives()[0], null /*replaced*/); - assertFalse(mArchInst.install(ar, mSdkRoot, false /*forceHttp*/, mSdkMan, mMon)); + assertFalse(mArchInst.install(ar, mSdkRoot, false /*forceHttp*/, mSdkMan, + null /*UrlCache*/, mMon)); assertTrue(mMon.getCapturedLog().indexOf("Skipping already installed archive") != -1); } @@ -116,7 +118,8 @@ public class ArchiveInstallerTest extends TestCase { mArchInst.setDownloadResponse( p.getArchives()[0], createFile("/sdk", "tmp", "download1.zip")); - assertTrue(mArchInst.install(ar, mSdkRoot, false /*forceHttp*/, mSdkMan, mMon)); + assertTrue(mArchInst.install(ar, mSdkRoot, false /*forceHttp*/, mSdkMan, + null /*UrlCache*/, mMon)); // check what was created assertEquals( @@ -162,7 +165,8 @@ public class ArchiveInstallerTest extends TestCase { mArchInst.setDownloadResponse( newPkg.getArchives()[0], createFile("/sdk", "tmp", "download1.zip")); - assertTrue(mArchInst.install(ar, mSdkRoot, false /*forceHttp*/, mSdkMan, mMon)); + assertTrue(mArchInst.install(ar, mSdkRoot, false /*forceHttp*/, mSdkMan, + null /*UrlCache*/, mMon)); // check what was created assertEquals( @@ -237,7 +241,8 @@ public class ArchiveInstallerTest extends TestCase { mArchInst.setDownloadResponse( newPkg.getArchives()[0], createFile("/sdk", "tmp", "download1.zip")); - assertTrue(mArchInst.install(ar, mSdkRoot, false /*forceHttp*/, mSdkMan, mMon)); + assertTrue(mArchInst.install(ar, mSdkRoot, false /*forceHttp*/, mSdkMan, + null /*UrlCache*/, mMon)); // check what was created assertEquals( |