From af19cec19a8c77a913f4462bd54fa97eadef8c7a Mon Sep 17 00:00:00 2001 From: Jeffrey McAteer Date: Mon, 13 Jan 2020 19:30:03 -0500 Subject: [PATCH] Added a global caching capability to WMS layers. Caching only occurs when the system property "worldwind.ImageRetriever.decodeUrlCacheDir" has been set. Both the layer meta-data as well as individual tiles are cached within the directory referenced by System.getProperty("worldwind.ImageRetriever.decodeUrlCacheDir"). --- .../nasa/worldwind/layer/LayerFactory.java | 95 +++++++++++++- .../nasa/worldwind/render/ImageRetriever.java | 121 +++++++++++++++++- 2 files changed, 211 insertions(+), 5 deletions(-) diff --git a/worldwind/src/main/java/gov/nasa/worldwind/layer/LayerFactory.java b/worldwind/src/main/java/gov/nasa/worldwind/layer/LayerFactory.java index 32e8efd2b..ded235eab 100644 --- a/worldwind/src/main/java/gov/nasa/worldwind/layer/LayerFactory.java +++ b/worldwind/src/main/java/gov/nasa/worldwind/layer/LayerFactory.java @@ -11,6 +11,9 @@ import java.io.BufferedInputStream; import java.io.InputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileInputStream; import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; @@ -457,15 +460,101 @@ protected WmsCapabilities retrieveWmsCapabilities(String serviceAddress) throws // Parse and read the input stream wmsCapabilities = WmsCapabilities.getCapabilities(inputStream); } catch (Exception e) { - throw new RuntimeException( - Logger.makeMessage("LayerFactory", "retrieveWmsCapabilities", "Unable to open connection and read from service address")); + // Attempt to read capabilities from cache file + wmsCapabilities = attemptGlobalCacheWmsCapabilitiesResolution(serviceAddress); + if (wmsCapabilities == null) { + throw new RuntimeException( + Logger.makeMessage("LayerFactory", "retrieveWmsCapabilities", "Unable to open connection and read from service address")); + } } finally { - WWUtil.closeSilently(inputStream); + if (inputStream != null) { + WWUtil.closeSilently(inputStream); + } + } + + if (wmsCapabilities != null) { + addToGlobalCache(serviceAddress, wmsCapabilities); } return wmsCapabilities; } + private WmsCapabilities attemptGlobalCacheWmsCapabilitiesResolution(String serviceAddress) { + File cacheFile = getGlobalCacheUrlFile(serviceAddress); + if (!cacheFile.exists()) { + return null; + } + FileInputStream inputStream = null; + WmsCapabilities wmsCapabilities = null; + try { + inputStream = new FileInputStream(cacheFile); + // Parse and read the input stream + wmsCapabilities = WmsCapabilities.getCapabilities(inputStream); + } catch (Exception e) { + throw new RuntimeException( + Logger.makeMessage("LayerFactory", "retrieveWmsCapabilities", "Unable to read WmsCapabilities from global cache")); + } finally { + if (inputStream != null) { + WWUtil.closeSilently(inputStream); + } + } + return wmsCapabilities; + } + + private void addToGlobalCache(String serviceAddress, WmsCapabilities wmsCapabilities) { + // We do not actually use wmsCapabilities except to exit on empty data; + // instead we download the XML data from a URL constructed the same way as retrieveWmsCapabilities() + if (wmsCapabilities == null) { + return; + } + File cacheFile = getGlobalCacheUrlFile(serviceAddress); + InputStream inputStream = null; + FileOutputStream fos = null; + try { + // Build the appropriate request Uri given the provided service address + Uri serviceUri = Uri.parse(serviceAddress).buildUpon() + .appendQueryParameter("VERSION", "1.3.0") + .appendQueryParameter("SERVICE", "WMS") + .appendQueryParameter("REQUEST", "GetCapabilities") + .build(); + + // Open the connection as an input stream + URLConnection conn = new URL(serviceUri.toString()).openConnection(); + conn.setConnectTimeout(3000); + conn.setReadTimeout(30000); + inputStream = new BufferedInputStream(conn.getInputStream()); + + // Open the cache file for writing + fos = new FileOutputStream(cacheFile); + + byte[] buffer = new byte[8 * 1024]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + fos.write(buffer, 0, bytesRead); + } + + } catch (Exception e) { + // Throwing this error does not make sense; we may be offline and doing nothing is appropriate. + + } finally { + if (inputStream != null) { + WWUtil.closeSilently(inputStream); + } + if (fos != null) { + WWUtil.closeSilently(fos); + } + } + } + + /** + * This uses the method gov.nasa.worldwind.render.ImageRetriever.getGlobalCacheUrlFile + * to resolve a url-specific cache file that ends in ".xml" for use caching WMS capabilities + * for offline use. + */ + protected static File getGlobalCacheUrlFile(String serviceAddress) { + return gov.nasa.worldwind.render.ImageRetriever.getGlobalCacheUrlFile(serviceAddress, "%s.xml"); + } + protected WmtsCapabilities retrieveWmtsCapabilities(String serviceAddress) throws Exception { InputStream inputStream = null; WmtsCapabilities wmtsCapabilities = null; diff --git a/worldwind/src/main/java/gov/nasa/worldwind/render/ImageRetriever.java b/worldwind/src/main/java/gov/nasa/worldwind/render/ImageRetriever.java index 7f47590d3..c7be6bce7 100644 --- a/worldwind/src/main/java/gov/nasa/worldwind/render/ImageRetriever.java +++ b/worldwind/src/main/java/gov/nasa/worldwind/render/ImageRetriever.java @@ -11,9 +11,15 @@ import java.io.BufferedInputStream; import java.io.IOException; +import java.io.FileNotFoundException; import java.io.InputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileInputStream; import java.net.URL; import java.net.URLConnection; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import gov.nasa.worldwind.WorldWind; import gov.nasa.worldwind.util.Logger; @@ -95,10 +101,14 @@ protected Bitmap decodeFilePath(String pathName, ImageOptions imageOptions) { } protected Bitmap decodeUrl(String urlString, ImageOptions imageOptions) throws IOException { - // TODO establish a file caching service for remote resources // TODO retry absent resources, they are currently handled but suppressed entirely after the first failure // TODO configurable connect and read timeouts + Bitmap cached = attemptGlobalCacheSourceResolution(urlString); + if (cached != null) { + return cached; + } + InputStream stream = null; try { URLConnection conn = new URL(urlString).openConnection(); @@ -108,12 +118,119 @@ protected Bitmap decodeUrl(String urlString, ImageOptions imageOptions) throws I stream = new BufferedInputStream(conn.getInputStream()); BitmapFactory.Options factoryOptions = this.bitmapFactoryOptions(imageOptions); - return BitmapFactory.decodeStream(stream, null, factoryOptions); + Bitmap fetchedBmp = BitmapFactory.decodeStream(stream, null, factoryOptions); + if (fetchedBmp != null) { + addToGlobalCache(urlString, fetchedBmp); + } + return fetchedBmp; } finally { WWUtil.closeSilently(stream); } } + protected Bitmap attemptGlobalCacheSourceResolution(String urlString) { + // At the moment the cache directory is global for all ImageRetrievers in a given JVM. + // Minor // TODO may be to add a configuration object to this class for setting per-retriever cache directories. + String cacheDir = System.getProperty("worldwind.ImageRetriever.decodeUrlCacheDir"); + if (cacheDir == null) { + return null; // Calling app did not set a cache dir + } + File cacheDirFD = new File(cacheDir); + if (!cacheDirFD.exists()) { + try { + cacheDirFD.mkdirs(); + } + catch (SecurityException sx) { + Logger.logMessage(Logger.ERROR, "ImageRetriever", "attemptGlobalCacheSourceResolution", "Cannot create cache directory"); + return null; // Coult not create cache dir + } + } + + File cacheFileFD = getGlobalCacheUrlFile(urlString); + if (!cacheFileFD.exists()) { + return null; + } + long fileSizeInBytes = cacheFileFD.length(); + if (fileSizeInBytes < 1) { + return null; // File exists, but is empty. + } + Bitmap bmp = null; + try { + bmp = BitmapFactory.decodeStream(new FileInputStream(cacheFileFD)); + } + catch (FileNotFoundException fnfex) { + Logger.logMessage(Logger.ERROR, "ImageRetriever", "attemptGlobalCacheSourceResolution", "Cannot decode cache file into Bitmap"); + } + return bmp; + } + + protected void addToGlobalCache(String urlString, Bitmap bitmap) { + File cacheFileFD = getGlobalCacheUrlFile(urlString); + try (FileOutputStream out = new FileOutputStream(cacheFileFD)) { + bitmap.compress(Bitmap.CompressFormat.PNG, 100, out); + } + catch (IOException e) { + Logger.logMessage(Logger.ERROR, "ImageRetriever", "addToGlobalCache", "Cannot write bitmap to cache file"); + } + } + + /** + * This performs concatination of {@code System.getProperty("worldwind.ImageRetriever.decodeUrlCacheDir")} + * with a SHA-256 hex string of {@code urlString}, returning {@code null} if + * anything does not exist ({@code System.getProperty("worldwind.ImageRetriever.decodeUrlCacheDir") == null} etc) + * + * This will NOT create a cache directory, and instead returns null when the directory path stored in + * {@code System.getProperty("worldwind.ImageRetriever.decodeUrlCacheDir")} does not exist. + * + * This has been made public and static to as to be used elsewhere; this likely deserves a class + * of it's own as it performs a caching operation which is not unique to ImageRetriever. + * + * @param urlString The url which produces the cached file's name + * @param fileNameFormatStr A format string which takes a single %s argument used to create the cache file name. + * @return a file object pointing to the cached file (file may not exist) + */ + public static File getGlobalCacheUrlFile(String urlString, String fileNameFormatStr) { + String cacheDir = System.getProperty("worldwind.ImageRetriever.decodeUrlCacheDir"); + if (cacheDir == null) { + return null; + } + File cacheDirFD = new File(cacheDir); + if (!cacheDirFD.exists()) { + return null; + } + + MessageDigest md; + try { + md = MessageDigest.getInstance("SHA-256"); + } + catch (NoSuchAlgorithmException nsax) { + Logger.logMessage(Logger.ERROR, "ImageRetriever", "getGlobalCacheUrlFile", "No MessageDigest instance available for \"SHA-256\""); + return null; // No sha-256 hash implementation for us to use + } + + md.update(urlString.getBytes()); + + byte[] digest = md.digest(); + String digestStr = decodeTohexStr(digest).toUpperCase(); + + // At the moment all cached bitmaps are stored as .png files + String cacheFilename = String.format(fileNameFormatStr, digestStr); + File cacheFileFD = new File(cacheDirFD, cacheFilename); + + return cacheFileFD; + } + public static File getGlobalCacheUrlFile(String urlString) { + return getGlobalCacheUrlFile(urlString, "%s.png"); + } + + private static String decodeTohexStr(byte[] data) { + StringBuilder sb = new StringBuilder(); + for (byte b : data) { + sb.append(String.format("%02X ", b)); + } + return sb.toString(); + } + protected Bitmap decodeUnrecognized(ImageSource imageSource) { Logger.log(Logger.WARN, "Unrecognized image source \'" + imageSource + "\'"); return null;