diff --git a/.travis.yml b/.travis.yml index a3bf927d..3373b4ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,12 @@ cache: jdk: - oraclejdk11 - oraclejdk8 - - openjdk7 + +before_script: + - cd location-server && ./start-location.sh && cd .. + +script: + - mvn test -B -Dtestng.runHttpRemoteTests=1 -Dtestng.runS3RemoteTests=1 matrix: fast_finish: true diff --git a/location-server/.gitignore b/location-server/.gitignore new file mode 100644 index 00000000..476165b5 --- /dev/null +++ b/location-server/.gitignore @@ -0,0 +1,5 @@ +bioformats.test.private +bioformats.test.public +mc +minio +.minio.sys diff --git a/location-server/start-location.sh b/location-server/start-location.sh new file mode 100755 index 00000000..cb9f9343 --- /dev/null +++ b/location-server/start-location.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -eux + +PORT=31836 +PLATFORM=`uname | tr '[:upper:]' '[:lower:]'` + +[ -f minio ] || \ + curl -sfSo minio "https://dl.minio.io/server/minio/release/$PLATFORM-amd64/minio" +[ -f mc ] || \ + curl -sfSo mc "https://dl.minio.io/client/mc/release/$PLATFORM-amd64/mc" +chmod +x minio mc +./minio version +./mc version + +export MINIO_ACCESS_KEY=accesskey MINIO_SECRET_KEY=secretkey +./minio server --address localhost:$PORT . & +sleep 2; + +./mc config host add ome-common-java-minio-test http://localhost:$PORT ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY} + +set +x + +for SUFFIX in public private; do + ./mc ls ome-common-java-minio-test/bioformats.test.$SUFFIX || \ + ./mc mb ome-common-java-minio-test/bioformats.test.$SUFFIX + ./mc ls ome-common-java-minio-test/bioformats.test.$SUFFIX/single-channel.ome.tiff || \ + curl -sfS https://downloads.openmicroscopy.org/images/OME-TIFF/2016-06/bioformats-artificial/single-channel.ome.tiff | \ + ./mc pipe ome-common-java-minio-test/bioformats.test.$SUFFIX/single-channel.ome.tiff + # 2MB file for testing seek + for n in `seq 65536`; do printf '.% 30d\n' $n; done | \ + ./mc pipe ome-common-java-minio-test/bioformats.test.$SUFFIX/2MBfile.txt +done + +./mc policy public ome-common-java-minio-test/bioformats.test.public diff --git a/pom.xml b/pom.xml index 58d6478a..d29419a3 100644 --- a/pom.xml +++ b/pom.xml @@ -101,6 +101,11 @@ + + io.minio + minio + 5.0.2 + com.esotericsoftware.kryo kryo @@ -233,6 +238,10 @@ src/test/java/loci/common/utests/testng-template.xml + + 0 + 0 + diff --git a/src/main/java/loci/common/AbstractNIOHandle.java b/src/main/java/loci/common/AbstractNIOHandle.java index f614d25e..db9e13f3 100644 --- a/src/main/java/loci/common/AbstractNIOHandle.java +++ b/src/main/java/loci/common/AbstractNIOHandle.java @@ -56,6 +56,12 @@ public abstract class AbstractNIOHandle implements IRandomAccess { // -- AbstractNIOHandle methods -- + /* @see IRandomAccess#exists() */ + @Override + public boolean exists() throws IOException { + return length() >= 0; + } + /** * Ensures that the file mode is either "r" or "rw". * @param mode Mode to validate. diff --git a/src/main/java/loci/common/FileHandle.java b/src/main/java/loci/common/FileHandle.java index c8bddd79..2203c4e9 100644 --- a/src/main/java/loci/common/FileHandle.java +++ b/src/main/java/loci/common/FileHandle.java @@ -101,6 +101,12 @@ public long getFilePointer() throws IOException { return raf.getFilePointer(); } + /* @see IRandomAccess#exists() */ + @Override + public boolean exists() throws IOException { + return length() >= 0; + } + /* @see IRandomAccess.length() */ @Override public long length() throws IOException { diff --git a/src/main/java/loci/common/IRandomAccess.java b/src/main/java/loci/common/IRandomAccess.java index f4acc887..bc4b8446 100644 --- a/src/main/java/loci/common/IRandomAccess.java +++ b/src/main/java/loci/common/IRandomAccess.java @@ -61,6 +61,14 @@ public interface IRandomAccess extends DataInput, DataOutput { */ long getFilePointer() throws IOException; + /** + * Returns whether this refers to a valid object + * + * @return true if this refers to a valid object + * @throws IOException if unable to determine whether the object is valid + */ + boolean exists() throws IOException; + /** * Returns the length of this stream. * diff --git a/src/main/java/loci/common/Location.java b/src/main/java/loci/common/Location.java index 585eb853..a5fc14e2 100644 --- a/src/main/java/loci/common/Location.java +++ b/src/main/java/loci/common/Location.java @@ -9,13 +9,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -37,7 +37,10 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.UncheckedIOException; import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; @@ -45,6 +48,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,6 +68,12 @@ public class Location { private static final boolean IS_WINDOWS = System.getProperty("os.name").startsWith("Windows"); + // -- Enumerations -- + protected enum UrlType { + GENERIC, + S3 + }; + // -- Static fields -- /** Map from given filenames to actual filenames. */ @@ -91,12 +101,61 @@ protected class ListingsResult { private static final Map fileListings = new MapMaker().makeMap(); // like Java's ConcurrentHashMap + /** Pattern to match child URLs */ + private static final Pattern URL_MATCHER = Pattern.compile( + "\\p{Alnum}+(\\+\\p{Alnum}+)?://.*"); + + /** Pattern to detect when getParent has gone past the parent of a URL */ + private static final Pattern URL_ABOVE_PARENT = Pattern.compile( + "\\p{Alnum}+(\\+\\p{Alnum}+)?:/$"); + + // -- Fields -- - private boolean isURL = true; + private boolean isURL = false; + private UrlType urlType; private URL url; + private URI uri; private File file; + class URLLocationProperties { + public final long length; + public final boolean exists; + + public URLLocationProperties(Location loc) { + LOGGER.trace("Getting LocationProperties"); + boolean bexists = false; + long llength = 0; + if (!loc.isURL) { + throw new IllegalArgumentException("Location must be a URL"); + } + try { + IRandomAccess handle = Location.getHandle(uri.toString()); + try { + bexists = handle.exists(); + } + catch (IOException e) { + LOGGER.trace("Failed to retrieve content from URL", e); + } + if (bexists) { + try { + llength = handle.length(); + } catch (IOException e) { + LOGGER.trace("Could not determine URL's content length", e); + } + } + handle.close(); + } + catch (IOException e) { + LOGGER.trace("Failed to retrieve content from URL", e); + } + this.exists = bexists; + this.length = llength; + LOGGER.trace("exists:{} length:{}", bexists, llength); + } + } + private URLLocationProperties cachedProperties; + // -- Constructors -- /** @@ -107,22 +166,7 @@ protected class ListingsResult { * @see #getMappedFile(String) */ public Location(String pathname) { - LOGGER.trace("Location({})", pathname); - if (pathname.contains("://")) { - // Avoid expensive exception handling in case when path is - // obviously not an URL - try { - url = new URL(getMappedId(pathname)); - } - catch (MalformedURLException e) { - LOGGER.trace("Location is not a URL", e); - isURL = false; - } - } else { - LOGGER.trace("Location is not a URL"); - isURL = false; - } - if (!isURL) file = new File(getMappedId(pathname)); + this((String) null, pathname); } /** @@ -145,7 +189,53 @@ public Location(File file) { * @param child the relative path name */ public Location(String parent, String child) { - this(parent + File.separator + child); + LOGGER.trace("Location({}, {})", parent, child); + + String mapped = null; + String pathname = null; + + // First handle possible URIs + if (child != null && URL_MATCHER.matcher(child).matches()) { + // Avoid expensive exception handling in case when path is + // obviously not an URL + try { + mapped = getMappedId(child); + pathname = child; + uri = new URI(mapped); + isURL = true; + if (S3Handle.canHandleScheme(uri.toString())) { + urlType = UrlType.S3; + url = null; + } + else { + urlType = UrlType.GENERIC; + url = uri.toURL(); + } + } + catch (URISyntaxException | MalformedURLException e) { + // Readers such as FilePatternReader may pass invalid URI paths + // containing <> so don't throw, instead treat as a non-URL + LOGGER.debug("Invalid URL: {} {}", child, e); + isURL = false; + urlType = null; + url = null; + uri = null; + } + } + + // If not a URI, then deal with relative vs. absolute paths + if (pathname == null) { + if (parent != null) { + // TODO: in some cases child here may be null + pathname = parent + File.separator + child; + } else { + pathname = child; + } + mapped = getMappedId(pathname); + } + + if (!isURL) file = new File(mapped); + } /** @@ -395,7 +485,23 @@ public static IRandomAccess getHandle(String id, boolean writable, LOGGER.trace("no handle was mapped for this ID"); String mapId = getMappedId(id); - if (id.startsWith("http://") || id.startsWith("https://")) { + if (S3Handle.canHandleScheme(id)) { + StreamHandle.Settings ss = new StreamHandle.Settings(); + if (ss.getRemoteCacheRootDir() != null) { + String cachedFile = S3Handle.cacheObject(mapId, ss); + if (bufferSize > 0) { + handle = new NIOFileHandle( + new File(cachedFile), "r", bufferSize); + } + else { + handle = new NIOFileHandle(cachedFile, "r"); + } + } + else { + handle = new S3Handle(mapId); + } + } + else if (id.startsWith("http://") || id.startsWith("https://")) { handle = new URLHandle(mapId); } else if (allowArchiveHandles && ZipHandle.isZipFile(mapId)) { @@ -416,6 +522,10 @@ else if (allowArchiveHandles && BZip2Handle.isBZip2File(mapId)) { handle = new NIOFileHandle(mapId, writable ? "rw" : "r"); } } + LOGGER.trace("Created new handle {} -> {}", id, handle); + // TODO: We should cache the handle, but we can't prevent callers from closing it which + // would make the cached handle useless to future fetches + //mapFile(id, handle); } LOGGER.trace("Location.getHandle: {} -> {}", id, handle); return handle; @@ -464,6 +574,18 @@ public String[] list(boolean noHiddenFiles) { final List files = new ArrayList(); if (isURL) { try { + if (urlType == UrlType.S3) { + if (isDirectory()) { + // TODO: This is complicated, not sure what to do here + // See comment in isDirectory() + LOGGER.trace("list s3 {}: Returning []", uri); + return new String[0]; + } + else { + LOGGER.trace("list s3 {}: Returning null", uri); + return null; + } + } URLConnection c = url.openConnection(); InputStream is = c.getInputStream(); boolean foundEnd = false; @@ -539,7 +661,8 @@ public String[] list(boolean noHiddenFiles) { */ public boolean canRead() { LOGGER.trace("canRead()"); - return isURL ? (isDirectory() || isFile() || exists()) : file.canRead(); + // Note: isFile calls exist + return isURL ? (isDirectory() || isFile()) : file.canRead(); } /** @@ -639,17 +762,14 @@ public int hashCode() { * @see java.io.File#exists() */ public boolean exists() { - LOGGER.trace("exists()"); if (isURL) { - try { - url.getContent(); - return true; - } - catch (IOException e) { - LOGGER.trace("Failed to retrieve content from URL", e); - return false; + LOGGER.trace("exists(url)"); + if (cachedProperties == null) { + cachedProperties = new URLLocationProperties(this); } + return cachedProperties.exists; } + LOGGER.trace("exists(file)"); if (file.exists()) return true; if (getMappedFile(file.getPath()) != null) return true; @@ -675,7 +795,7 @@ public Location getAbsoluteFile() { */ public String getAbsolutePath() { LOGGER.trace("getAbsolutePath()"); - return isURL ? url.toExternalForm() : file.getAbsolutePath(); + return isURL ? uri.normalize().toString() : file.getAbsolutePath(); } /** @@ -715,9 +835,8 @@ public String getCanonicalPath() throws IOException { public String getName() { LOGGER.trace("getName()"); if (isURL) { - String name = url.getFile(); - name = name.substring(name.lastIndexOf("/") + 1); - return name; + // TODO: we should just store new File(uri) in file + return new File(uri.getPath()).getName(); } return file.getName(); } @@ -733,8 +852,12 @@ public String getName() { public String getParent() { LOGGER.trace("getParent()"); if (isURL) { + // TODO For S3 we should take account of directories not really existing String absPath = getAbsolutePath(); absPath = absPath.substring(0, absPath.lastIndexOf("/")); + if (URL_ABOVE_PARENT.matcher(absPath).matches()) { + return null; + } return absPath; } return file.getParent(); @@ -757,7 +880,7 @@ public Location getParentFile() { * @see java.io.File#getPath() */ public String getPath() { - return isURL ? url.getHost() + url.getPath() : file.getPath(); + return isURL ? uri.getHost() + uri.getPath() : file.getPath(); } /** @@ -769,7 +892,7 @@ public String getPath() { */ public boolean isAbsolute() { LOGGER.trace("isAbsolute()"); - return isURL ? true : file.isAbsolute(); + return isURL ? uri.isAbsolute() : file.isAbsolute(); } /** @@ -781,8 +904,32 @@ public boolean isAbsolute() { public boolean isDirectory() { LOGGER.trace("isDirectory()"); if (isURL) { - String[] list = list(); - return list != null; + if (urlType == UrlType.S3) { + // TODO: This is complicated + // + // S3 doesn't have directories, but keys can contain / which we + // can pretend is a file path. However this "directory" doesn't + // actually exist, only the "contents" of the directory exist. + // + // Minio.listObjects() lists all objects in a bucket that + // match an optional prefix so this could be an option for checking + // whether to trest this as a directory. + // + // S3 buckets are the closest thing to a proper directory + // so for now + try { + S3Handle h = new S3Handle(uri.toString()); + boolean isBucket = h.isBucket(); + h.close(); + return isBucket; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + // TODO: this should be removed as well. + String[] list = list(); + return list != null; + } } return file.isDirectory(); } @@ -829,7 +976,11 @@ public long lastModified() { LOGGER.trace("lastModified()"); if (isURL) { try { - return url.openConnection().getLastModified(); + if (url != null) { + return url.openConnection().getLastModified(); + } else { + return 0; + } } catch (IOException e) { LOGGER.trace("Could not determine URL's last modification time", e); @@ -847,16 +998,13 @@ public long lastModified() { * @see java.net.URLConnection#getContentLength() */ public long length() { - LOGGER.trace("length()"); if (isURL) { - try { - return url.openConnection().getContentLength(); - } - catch (IOException e) { - LOGGER.trace("Could not determine URL's content length", e); - return 0; - } + LOGGER.trace("length(url)"); + // Ensure cachedProperties is populated + exists(); + return cachedProperties.length; } + LOGGER.trace("length(file)"); return file.length(); } @@ -908,7 +1056,7 @@ public URL toURL() throws MalformedURLException { */ @Override public String toString() { - return isURL ? url.toString() : file.toString(); + return isURL ? uri.toString() : file.toString(); } } diff --git a/src/main/java/loci/common/S3Handle.java b/src/main/java/loci/common/S3Handle.java new file mode 100644 index 00000000..9862007e --- /dev/null +++ b/src/main/java/loci/common/S3Handle.java @@ -0,0 +1,455 @@ +/* + * #%L + * Common package for I/O and related utilities + * %% + * Copyright (C) 2018 Open Microscopy Environment: + * - Board of Regents of the University of Wisconsin-Madison + * - Glencoe Software, Inc. + * - University of Dundee + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package loci.common; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.regex.Pattern; + +import io.minio.MinioClient; +import io.minio.errors.MinioException; +import io.minio.ObjectStat; +import org.xmlpull.v1.XmlPullParserException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Provides random access to S3 buckets using the IRandomAccess interface. + * Instances of S3Handle are read-only. + * + * @see IRandomAccess + * @see StreamHandle + * @see java.net.URLConnection + * + */ +public class S3Handle extends StreamHandle { + + /** + * An S3 IOException that was not thrown immediately + */ + class DelayedObjectNotFound extends IOException { + DelayedObjectNotFound(S3Handle s3) { + super(String.format("Object not found: [%s] %s", s3, s3.objectNotFound), s3.objectNotFound); + } + } + + /** Default protocol for fetching s3:// */ + public final static String DEFAULT_S3_PROTOCOL = "https"; + + private static final Logger LOGGER = LoggerFactory.getLogger(S3Handle.class); + + protected final static Pattern SCHEME_PARSER = Pattern.compile("s3(\\+\\p{Alnum}+)?(://.*)?"); + + /** S3 configuration */ + private final Settings settings; + + /** Parsed URI used to configure this handle */ + private final URI uri; + + /** access key, if provided */ + private final String accessKey; + + /** secret key, if provided */ + private final String secretKey; + + /** name of the bucket */ + private final String bucket; + + /** endpoint to which requests will be sent */ + private final String server; + + /** port at the given server */ + private final int port; + + /** remaining path, or key, for this accessed resource */ + private final String path; + + /** Minio client */ + private MinioClient s3Client; + + /** Remote file stat */ + private ObjectStat stat; + + /** Is this a directory (currently only buckets are considered directories */ + private boolean isBucket; + + /** + * Exception if thrown during construction + */ + private Throwable objectNotFound; + + /** If seeking more than this distance reset and reopen at offset */ + protected static final int S3_MAX_FORWARD_SEEK = 1048576; + + /** + * Return true if this is a URL with an s3 scheme + * @param url URL + * @return true if this class can handle url + */ + public static boolean canHandleScheme(String url) { + return SCHEME_PARSER.matcher(url).matches(); + } + + /** + * Open an S3 file + * + * @param url the full URL to the S3 resource + * @throws IOException if there is an error during opening + */ + public S3Handle(String url) throws IOException { + this(url, true, null); + } + + /** + * Open an S3 file + * + * @param uristr the full URL to the S3 resource + * @param initialize If true open the stream, otherwise just parse connection + * string + * @param s custom settings object + * @throws IOException if there is an error during opening + */ + public S3Handle(String uristr, boolean initialize, Settings s) throws + IOException { + if (s == null) { + this.settings = new StreamHandle.Settings(); + } + else { + this.settings = s; + } + + try { + this.uri = new URI(uristr); + } catch (URISyntaxException e) { + throw new RuntimeException("Invalid URI " + uristr, e); + } + + // access[:secret] + String auth = this.uri.getUserInfo(); + String accessKey = null; + String secretKey = null; + if (auth != null) { + String[] authparts = auth.split(":", 2); + accessKey = authparts[0]; + if (authparts.length > 1) { + secretKey = authparts[1]; + } + } + this.accessKey = accessKey; + this.secretKey = secretKey; + + String protocol; + String scheme = this.uri.getScheme(); + if (scheme.equals("s3")) { + protocol = DEFAULT_S3_PROTOCOL; + } + else if (scheme.startsWith("s3+")) { + protocol = scheme.substring(3); + } + else { + protocol = scheme; + } + this.server = protocol + "://" + this.uri.getHost(); + + if (this.uri.getPort() == -1) { + this.port = 0; + } + else { + this.port = this.uri.getPort(); + } + + // First path component is the bucket + // TODO: Parsing this seems way more complicated than it should be + String fullpath = this.uri.getPath(); + if (fullpath == null || fullpath.length() == 0) { + fullpath = "/"; + } + // Leading / means first element is always "" + String[] pathparts = fullpath.split("/", 3); + if (pathparts[1].length() > 0) { + this.bucket = pathparts[1]; + } + else { + this.bucket = null; + } + if (pathparts.length > 2 && pathparts[2].length() > 0) { + this.path = pathparts[2]; + } + else { + this.path = null; + } + + this.isBucket = false; + this.stat = null; + + if (initialize) { + // Throw if there is an IOException, otherwise save the exception and only throw if a method + // that requires a valid object is called + this.connect(); + try { + this.initialize(); + } + catch ( + MinioException | + InvalidKeyException | + NoSuchAlgorithmException | + XmlPullParserException e) { + this.objectNotFound = e; + LOGGER.debug("Object not found: [{}] {}", this, e); + } + LOGGER.trace("isBucket:{} stat:{}", isBucket, stat); + } + } + + /** + * Connect to the server + * @throws IOException if there was an error connecting to the server + */ + protected void connect() throws IOException { + try { + s3Client = new MinioClient(server, port, accessKey, secretKey); + // TODO: Replace "dev" with a version + s3Client.setAppInfo("Bio-Formats", "dev"); + } + catch (MinioException e) { + throw new IOException(String.format( + "Failed to connect: %s", this), e); + } + LOGGER.trace("connected: server:{} port:{}", server, port); + } + + /** + * Check bucket or object exists + * @throws IOException if unable to get the object + * @throws MinioException if unable to get the object + * @throws InvalidKeyException if unable to get the object + * @throws NoSuchAlgorithmException if unable to get the object + * @throws XmlPullParserException if unable to get the object + */ + protected void initialize() throws + IOException, + MinioException, + InvalidKeyException, + NoSuchAlgorithmException, + XmlPullParserException { + if (path == null) { + isBucket = s3Client.bucketExists(bucket); + } + else { + isBucket = false; + stat = s3Client.statObject(bucket, path); + resetStream(); + } + } + + public String getServer() { + return server; + } + + public int getPort() { + return port; + } + + public String getBucket() { + return bucket; + } + + public String getPath() { + return path; + } + + /** + * Download an S3 object to a file system cache if it doesn't already exist + * + * @param url the full URL to the S3 resource + * @param s custom settings object + * @return File path to the cached object + * @throws IOException if there is an error during reading or writing + * @throws HandleException if no destination for the cache is provided + */ + public static String cacheObject(String url, Settings s) throws + IOException, + HandleException { + String cacheroot = s.getRemoteCacheRootDir(); + if (cacheroot == null) { + throw new HandleException("Remote cache root dir is not set"); + } + S3Handle s3 = new S3Handle(url, true, s); + // TODO: Need to ensure this path is safe. Is there a Java method to check? + String cacheobj = s3.getCacheKey(); + // Hopefully creates a cross-platform path + Path cachepath = Paths.get(cacheroot, cacheobj); + + if (Files.exists(cachepath)) { + LOGGER.debug("Found existing cache for {} at {}", s3, cachepath); + } + else { + LOGGER.debug("Caching {} to {}", s3, cachepath); + s3.downloadObject(cachepath); + LOGGER.debug("Downloaded {}", cachepath); + } + return cachepath.toString(); + } + + public String getCacheKey(){ + String cachekey = + getServer().replace("://", "/") + "/" + + getPort() + "/" + + getBucket() + "/" + + getPath(); + return cachekey; + } + + protected void downloadObject(Path destination) throws HandleException, IOException { + LOGGER.trace("destination:{}", destination); + if (this.stat == null || this.objectNotFound != null) { + throw new IOException("Object not found " + this, this.objectNotFound); + } + if (path == null) { + throw new HandleException("Download path=null not allowed"); + } + try { + Files.createDirectories(destination.getParent()); + s3Client.getObject(bucket, path, destination.toString()); + } + catch ( + IOException | + InvalidKeyException | + MinioException | + NoSuchAlgorithmException | + XmlPullParserException e) { + throw new HandleException("Download failed " + toString(), e); + } + } + + /** + * Is this an accessible bucket? + * TODO: If this bucket doesn't exist do we return false or thrown an exception? + * + * @return True if a bucket + */ + public boolean isBucket() { + //if (this.objectNotFound != null) { + // throw new DelayedObjectNotFound(this); + //} + return isBucket; + } + + /* @see IRandomAccess#length() */ + @Override + public long length() throws IOException { + if (this.stat == null || this.objectNotFound != null) { + throw new DelayedObjectNotFound(this); + } + return length; + } + + /** + * @see StreamHandle#seek(long) + */ + @Override + public void seek(long pos) throws IOException { + LOGGER.trace("{}", pos); + if (this.stat == null || this.objectNotFound != null) { + throw new DelayedObjectNotFound(this); + } + long diff = pos - fp; + + if (diff < 0 || diff > S3_MAX_FORWARD_SEEK) { + resetStream(pos); + } + else { + super.seek(pos); + } + } + + /** + * @see StreamHandle#resetStream() + */ + @Override + protected void resetStream() throws IOException { + resetStream(0); + } + + /** + * Does this represent an accessible location? + * @return true if this location is accessible + * @throws IOException if unable to determine whether this location is accessible + */ + @Override + public boolean exists() throws IOException { + return (objectNotFound == null) && (isBucket || stat != null); + } + + /** + * Reset the stream to an offset position + * @param offset Offset into object + * @throws IOException if there is an error during reading or writing + */ + protected void resetStream(long offset) throws IOException { + LOGGER.trace("Resetting {}", offset); + if (this.stat == null || this.objectNotFound != null) { + throw new DelayedObjectNotFound(this); + } + try { + length = stat.length(); + stream = new DataInputStream(new BufferedInputStream( + s3Client.getObject(bucket, path, offset))); + fp = offset; + mark = offset; + } + catch ( + InvalidKeyException | + MinioException | + NoSuchAlgorithmException | + XmlPullParserException e) { + throw new IOException(String.format( + "failed to load s3: %s\n\t%s", uri, this), e); + } + } + + public String toString() { + boolean found = (objectNotFound == null) && (isBucket || stat != null); + return String.format("server:%s port:%d bucket:%s path:%s found:%s", + server, port, bucket, path, found); + } +} diff --git a/src/main/java/loci/common/StreamHandle.java b/src/main/java/loci/common/StreamHandle.java index 8a088c85..dc045c94 100644 --- a/src/main/java/loci/common/StreamHandle.java +++ b/src/main/java/loci/common/StreamHandle.java @@ -39,6 +39,9 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Abstract IRandomAccess implementation for reading from InputStreams and * writing to OutputStreams. @@ -49,6 +52,19 @@ */ public abstract class StreamHandle implements IRandomAccess { + // TODO: Decide how to handle S3Handle and other reader settings + public static class Settings { + public String get(String key) { + return System.getenv(key); + } + + public String getRemoteCacheRootDir() { + return get("BF_REMOTE_CACHE_ROOTDIR"); + } + } + + private static final Logger LOGGER = LoggerFactory.getLogger(StreamHandle.class); + // -- Fields -- /** Name of the open stream. */ @@ -89,6 +105,7 @@ public StreamHandle() { /* @see IRandomAccess#close() */ @Override public void close() throws IOException { + LOGGER.trace("closing"); length = fp = mark = 0; if (stream != null) stream.close(); if (outStream != null) outStream.close(); @@ -103,21 +120,31 @@ public long getFilePointer() throws IOException { return fp; } + /* @see IRandomAccess#exists() */ + @Override + public boolean exists() throws IOException { + return length >= 0; + } + /* @see IRandomAccess#length() */ @Override public long length() throws IOException { + // Too verbose + // LOGGER.trace("{}", length); return length; } /* @see IRandomAccess#read(byte[]) */ @Override public int read(byte[] b) throws IOException { + LOGGER.trace("0 {}", b.length); return read(b, 0, b.length); } /* @see IRandomAccess#read(byte[], int, int) */ @Override public int read(byte[] b, int off, int len) throws IOException { + LOGGER.trace("{} {}", off, len); int n = stream.read(b, off, len); if (n >= 0) fp += n; else n = 0; @@ -139,6 +166,7 @@ public int read(ByteBuffer buffer) throws IOException { /* @see IRandomAccess#read(ByteBuffer, int, int) */ @Override public int read(ByteBuffer buffer, int off, int len) throws IOException { + LOGGER.trace("{} {}", off, len); if (buffer.hasArray()) { return read(buffer.array(), off, len); } @@ -152,6 +180,7 @@ public int read(ByteBuffer buffer, int off, int len) throws IOException { /* @see IRandomAccess#seek(long) */ @Override public void seek(long pos) throws IOException { + LOGGER.trace("{}", pos); long diff = pos - fp; fp = pos; @@ -170,12 +199,14 @@ public void seek(long pos) throws IOException { /* @see IRandomAccess.write(ByteBuffer) */ @Override public void write(ByteBuffer buf) throws IOException { + LOGGER.trace("0 {}", buf.capacity()); write(buf, 0, buf.capacity()); } /* @see IRandomAccess.write(ByteBuffer, int, int) */ @Override public void write(ByteBuffer buf, int off, int len) throws IOException { + LOGGER.trace("{} {}", off, len); buf.position(off); if (buf.hasArray()) { write(buf.array(), off, len); diff --git a/src/test/java/loci/common/utests/LocationTest.java b/src/test/java/loci/common/utests/LocationTest.java index ddd47e78..dc6c5371 100644 --- a/src/test/java/loci/common/utests/LocationTest.java +++ b/src/test/java/loci/common/utests/LocationTest.java @@ -9,13 +9,13 @@ * %% * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * + * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE @@ -36,7 +36,7 @@ import java.io.File; import java.io.IOException; -import java.net.Socket; +import java.net.MalformedURLException; import java.net.URL; import java.util.Arrays; import java.util.List; @@ -58,13 +58,21 @@ public class LocationTest { // -- Fields -- + private enum LocalRemoteType { + LOCAL, + HTTP, + S3, + }; + private Location[] files; + private Location[] rootFiles; private boolean[] exists; private boolean[] isDirectory; private boolean[] isHidden; private String[] mode; - private boolean[] isRemote; - private boolean isOnline; + private LocalRemoteType[] isRemote; + private static boolean runHttpRemoteTests; + private static boolean runS3RemoteTests; // -- Setup methods -- @@ -92,146 +100,237 @@ public void setup() throws IOException { new Location("http://www.openmicroscopy.org/"), new Location("https://www.openmicroscopy.org/"), new Location("https://www.openmicroscopy.org/nonexisting"), - new Location(hiddenFile) + new Location("https://www.openmicroscopy.org/nonexisting/:/+/symbols"), + new Location(hiddenFile), + new Location("s3+http://localhost:31836/bucket-dne"), + new Location("s3+http://localhost:31836/bioformats.test.public"), + new Location("s3+http://localhost:31836/bioformats.test.public/single-channel.ome.tiff"), + new Location("s3+http://localhost:31836/bioformats.test.private/single-channel.ome.tiff"), + new Location("s3+http://accesskey:secretkey@localhost:31836/bioformats.test.private/single-channel.ome.tiff") + }; + + rootFiles = new Location[] { + new Location("/"), + new Location("https://www.openmicroscopy.org"), + new Location("s3://s3.example.org"), }; exists = new boolean[] { - true, false, true, true, true, false, true + true, + false, + true, + true, + true, + false, + false, + true, + false, + true, + true, + false, + true, }; isDirectory = new boolean[] { - false, false, true, false, false, false, false + false, + false, + true, + false, + false, + false, + false, + false, + false, + true, + false, + false, + false, }; isHidden = new boolean[] { - false, false, false, false, false, false, true + false, + false, + false, + false, + false, + false, + false, + true, + false, + false, + false, + false, + false, }; mode = new String[] { - "rw", "", "rw", "r", "r", "","rw" + "rw", + "", + "rw", + "r", + "r", + "", + "", + "rw", + "", + "r", + "r", + "", + "r", }; - isRemote = new boolean[] { - false, false, false, true, true, true, false + isRemote = new LocalRemoteType[] { + LocalRemoteType.LOCAL, + LocalRemoteType.LOCAL, + LocalRemoteType.LOCAL, + LocalRemoteType.HTTP, + LocalRemoteType.HTTP, + LocalRemoteType.HTTP, + LocalRemoteType.HTTP, + LocalRemoteType.LOCAL, + LocalRemoteType.S3, + LocalRemoteType.S3, + LocalRemoteType.S3, + LocalRemoteType.S3, + LocalRemoteType.S3, }; } @BeforeClass - public void checkIfOnline() throws IOException { - try { - new Socket("www.openmicroscopy.org", 80).close(); - isOnline = true; - } catch (IOException e) { - isOnline = false; + public void checkProperties() throws IOException { + runHttpRemoteTests = TestUtilities.getPropValueInt("testng.runHttpRemoteTests") > 0; + runS3RemoteTests = TestUtilities.getPropValueInt("testng.runS3RemoteTests") > 0; + + if (!runHttpRemoteTests) { + System.err.println("WARNING: HTTP tests are disabled!"); + } + if (!runS3RemoteTests) { + System.err.println("WARNING: S3 tests are disabled!"); } } - private void skipIfOffline(int i) throws SkipException { - if (isRemote[i] && !isOnline) { - throw new SkipException("must be online to test " + files[i].getName()); + private void skipIfHttpDisabled(int i) throws SkipException { + if (isRemote[i] == LocalRemoteType.HTTP && !runHttpRemoteTests) { + throw new SkipException("HTTP tests are disabled " + files[i].getName()); + } + } + + private void skipIfS3Disabled(int i) throws SkipException { + if (isRemote[i] == LocalRemoteType.S3 && !runS3RemoteTests) { + throw new SkipException("S3 tests are disabled " + files[i].getName()); } } // -- Tests -- + // Order of assertEquals parameters is assertEquals(message, expected, actual) @Test public void testReadWriteMode() { for (int i=0; i complete = Arrays.asList(completeList); for (String child : unhiddenList) { - assertEquals(files[i].getName(), complete.contains(child), true); - assertEquals(files[i].getName(), - new Location(files[i], child).isHidden(), false); + assertEquals(files[i].getName(), true, complete.contains(child)); + assertEquals(files[i].getName(), false, new Location(files[i], child).isHidden()); } for (int f=0; f 0; + + if (!runS3RemoteTests) { + System.err.println("WARNING: S3 tests are disabled!"); + } + } + + private void skipIfS3Disabled() throws SkipException { + if (!runS3RemoteTests) { + throw new SkipException("S3 tests are disabled"); + } + } + + // -- Test methods -- + + @Test + public void testCanHandleScheme() { + assertTrue(S3Handle.canHandleScheme("s3://")); + assertTrue(S3Handle.canHandleScheme("s3+transport://abc")); + assertTrue(S3Handle.canHandleScheme("s3+transport")); + assertFalse(S3Handle.canHandleScheme("s345://")); + assertFalse(S3Handle.canHandleScheme("http+s3://")); + assertFalse(S3Handle.canHandleScheme("https")); + } + + @Test + public void testParseLocalhost() throws IOException { + S3Handle s3 = new S3Handle("s3://localhost:9000/bucket/key/file.tif", false, null); + assertEquals("https://localhost", s3.getServer()); + assertEquals(9000, s3.getPort()); + assertEquals("bucket", s3.getBucket()); + assertEquals("key/file.tif", s3.getPath()); + } + + @Test + public void testParseAuth() throws IOException { + S3Handle s3 = new S3Handle( + "s3://access:secret@s3.example.org/bucket/key/file.tif", false, null); + assertEquals("https://s3.example.org", s3.getServer()); + assertEquals(0, s3.getPort()); + assertEquals("bucket", s3.getBucket()); + assertEquals("key/file.tif", s3.getPath()); + } + + @Test + public void testParseAuthLocalhost() throws IOException { + S3Handle s3 = new S3Handle("s3://access:secret@localhost:9000/bucket/key/file.tif", false, null); + assertEquals("https://localhost", s3.getServer()); + assertEquals(9000, s3.getPort()); + assertEquals("bucket", s3.getBucket()); + assertEquals("key/file.tif", s3.getPath()); + } + + @Test + public void testParseProtocol() throws IOException { + S3Handle s3 = new S3Handle( + "example://localhost/bucket/key/file.tif", false, null); + assertEquals("example://localhost", s3.getServer()); + assertEquals(0, s3.getPort()); + assertEquals("bucket", s3.getBucket()); + assertEquals("key/file.tif", s3.getPath()); + } + + @Test + public void testDefaultProtocol() throws IOException { + S3Handle s3 = new S3Handle("s3+custom://localhost/bucket/key/file.tif", false, null); + assertEquals("custom://localhost", s3.getServer()); + assertEquals(0, s3.getPort()); + assertEquals("bucket", s3.getBucket()); + assertEquals("key/file.tif", s3.getPath()); + } + + @Test + public void testParseNoSlash() throws IOException { + S3Handle s3 = new S3Handle("s3://localhost", false, null); + assertEquals("https://localhost", s3.getServer()); + assertEquals(0, s3.getPort()); + assertEquals(null, s3.getBucket()); + assertEquals(null, s3.getPath()); + } + + @Test + public void testParseSlashNoBucket() throws IOException { + S3Handle s3 = new S3Handle("s3://localhost/", false, null); + assertEquals("https://localhost", s3.getServer()); + assertEquals(0, s3.getPort()); + assertEquals(null, s3.getBucket()); + assertEquals(null, s3.getPath()); + } + + @Test + public void testParseBucketNoSlash() throws IOException { + S3Handle s3 = new S3Handle("s3://localhost/bucket", false, null); + assertEquals("https://localhost", s3.getServer()); + assertEquals(0, s3.getPort()); + assertEquals("bucket", s3.getBucket()); + assertEquals(null, s3.getPath()); + } + + @Test + public void testParseBucketSlash() throws IOException { + S3Handle s3 = new S3Handle("s3://localhost/bucket/", false, null); + assertEquals("https://localhost", s3.getServer()); + assertEquals(0, s3.getPort()); + assertEquals("bucket", s3.getBucket()); + assertEquals(null, s3.getPath()); + } + + @Test + public void testIsBucket() throws IOException { + skipIfS3Disabled(); + S3Handle s3 = new S3Handle(s3public + "/bioformats.test.public"); + assertTrue(s3.isBucket()); + } + + @Test + public void testReadPublic() throws IOException { + skipIfS3Disabled(); + S3Handle s3 = new S3Handle(s3public + "/bioformats.test.public/single-channel.ome.tiff"); + assertFalse(s3.isBucket()); + assertTrue(s3.exists()); + assertEquals(76097, s3.length()); + } + + @Test + public void testReadPrivate() throws IOException { + skipIfS3Disabled(); + S3Handle s3 = new S3Handle(s3private + "/bioformats.test.private/single-channel.ome.tiff"); + assertFalse(s3.isBucket()); + assertTrue(s3.exists()); + assertEquals(76097, s3.length()); + } + + @Test + public void testReadAndSeek() throws IOException { + skipIfS3Disabled(); + S3Handle s3 = new S3Handle(s3public + "/bioformats.test.public/2MBfile.txt"); + assertFalse(s3.isBucket()); + assertTrue(s3.exists()); + assertEquals(2097152, s3.length()); + + byte[] buffer = new byte[32]; + int r; + + r = s3.read(buffer, 0, 32); + assertEquals(32, r); + assertEquals(". 1\n", new String(buffer)); + + r = s3.read(buffer, 0, 32); + assertEquals(32, r); + assertEquals(". 2\n", new String(buffer)); + + s3.seek(80); + r = s3.read(buffer, 0, 32); + assertEquals(32, r); + assertEquals(" 3\n. ", new String(buffer)); + + // Large seek (S3Handle.S3_MAX_FORWARD_SEEK) + s3.seek(2097056); + r = s3.read(buffer, 0, 32); + assertEquals(32, r); + assertEquals(". 65534\n", new String(buffer)); + + // Reverse seek + s3.seek(144); + r = s3.read(buffer, 0, 32); + assertEquals(32, r); + assertEquals(" 5\n. ", new String(buffer)); + } + + @Test + public void testResetStream() throws IOException { + class S3HandleWrapper extends S3Handle { + public S3HandleWrapper(String url) throws IOException { + super(url); + } + + @Override + public void resetStream() throws IOException { + super.resetStream(); + } + + @Override + public void resetStream(long offset) throws IOException { + super.resetStream(offset); + } + } + + skipIfS3Disabled(); + S3HandleWrapper s3 = new S3HandleWrapper(s3private + "/bioformats.test.private/2MBfile.txt"); + assertFalse(s3.isBucket()); + assertTrue(s3.exists()); + assertEquals(2097152, s3.length()); + + byte[] buffer = new byte[32]; + int r; + s3.resetStream(750144); + r = s3.read(buffer, 0, 32); + assertEquals(32, r); + assertEquals(". 23443\n", new String(buffer)); + + s3.resetStream(); + r = s3.read(buffer, 0, 32); + assertEquals(32, r); + assertEquals(". 1\n", new String(buffer)); + } + + @Test + public void testCache() throws IOException { + class MockSettings extends StreamHandle.Settings { + @Override + public String getRemoteCacheRootDir() { + return TEMPDIR.toString(); + } + } + + skipIfS3Disabled(); + final String expectedPath = TEMPDIR + "/http/localhost/31836/bioformats.test.public/2MBfile.txt"; + + String downloaded = S3Handle.cacheObject( + s3public + "/bioformats.test.public/2MBfile.txt", new MockSettings()); + assertEquals(expectedPath, downloaded); + assertEquals(2097152, Files.size(Paths.get(downloaded))); + } +} \ No newline at end of file diff --git a/src/test/java/loci/common/utests/TestUtilities.java b/src/test/java/loci/common/utests/TestUtilities.java new file mode 100644 index 00000000..7d37ddc4 --- /dev/null +++ b/src/test/java/loci/common/utests/TestUtilities.java @@ -0,0 +1,51 @@ +/* + * #%L + * BSD implementations of Bio-Formats readers and writers + * %% + * Copyright (C) 2005 - 2018 Open Microscopy Environment: + * - Board of Regents of the University of Wisconsin-Madison + * - Glencoe Software, Inc. + * - University of Dundee + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package loci.common.utests; + +import loci.common.DataTools; + +public final class TestUtilities { + + /** + * Lookup an integer property + * @param propertyName Lookup this property + * @return an integer, 0 if not found + */ + public static int getPropValueInt(String propertyName) { + String prop = System.getProperty(propertyName); + if (prop == null || + prop.equals("${"+ propertyName + "}")) return 0; + if (DataTools.parseInteger(prop) == null) return 0; + int propertyValue = DataTools.parseInteger(prop); + return propertyValue; + } +}