diff --git a/.travis.yml b/.travis.yml index 9d277ce4..a2d293d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,6 @@ cache: jdk: - oraclejdk10 - oraclejdk8 - - openjdk7 matrix: fast_finish: true diff --git a/pom.xml b/pom.xml index b17b8182..5e1fdad5 100644 --- a/pom.xml +++ b/pom.xml @@ -101,6 +101,11 @@ + + io.minio + minio + 5.0.0 + com.esotericsoftware.kryo kryo diff --git a/src/main/java/loci/common/Location.java b/src/main/java/loci/common/Location.java index 585eb853..30eb22c0 100644 --- a/src/main/java/loci/common/Location.java +++ b/src/main/java/loci/common/Location.java @@ -37,9 +37,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLConnection; +import java.net.*; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -93,8 +91,9 @@ protected class ListingsResult { // -- Fields -- - private boolean isURL = true; + private boolean isURL = false; private URL url; + private URI uri; private File file; // -- Constructors -- @@ -107,22 +106,7 @@ protected class ListingsResult { * @see #getMappedFile(String) */ public Location(String pathname) { - LOGGER.trace("Location({})", pathname); - if (pathname.contains("://")) { - // Avoid expensive exception handling in case when path is - // obviously not an URL - try { - url = new URL(getMappedId(pathname)); - } - catch (MalformedURLException e) { - LOGGER.trace("Location is not a URL", e); - isURL = false; - } - } else { - LOGGER.trace("Location is not a URL"); - isURL = false; - } - if (!isURL) file = new File(getMappedId(pathname)); + this((String) null, pathname); } /** @@ -145,7 +129,41 @@ public Location(File file) { * @param child the relative path name */ public Location(String parent, String child) { - this(parent + File.separator + child); + LOGGER.trace("Location({}, {})", parent, child); + + String mapped = null; + String pathname = null; + + // First handle possible URIs + if (child != null && child.contains("://")) { + // Avoid expensive exception handling in case when path is + // obviously not an URL + try { + mapped = getMappedId(child); + pathname = child; + uri = new URI(mapped); + isURL = true; + url = uri.toURL(); + } + catch (URISyntaxException | MalformedURLException e) { + // TODO: this should possibly throw + // possibly leaves url null + } + } + + // If not a URI, then deal with relative vs. absolute paths + if (pathname == null) { + if (parent != null) { + // TODO: in some cases child here may be null + pathname = parent + File.separator + child; + } else { + pathname = child; + } + mapped = getMappedId(pathname); + } + + if (!isURL) file = new File(mapped); + } /** @@ -395,7 +413,10 @@ public static IRandomAccess getHandle(String id, boolean writable, LOGGER.trace("no handle was mapped for this ID"); String mapId = getMappedId(id); - if (id.startsWith("http://") || id.startsWith("https://")) { + if (id.startsWith("s3://")) { + handle = new S3Handle(mapId); + } + else if (id.startsWith("http://") || id.startsWith("https://")) { handle = new URLHandle(mapId); } else if (allowArchiveHandles && ZipHandle.isZipFile(mapId)) { @@ -464,6 +485,17 @@ public String[] list(boolean noHiddenFiles) { final List files = new ArrayList(); if (isURL) { try { + if (url == null) { + // Likely s3 + if (!exists()) { + return null; + } + if (uri.toString().endsWith("/")) { + return new String[0]; + } else { + return null; + } + } URLConnection c = url.openConnection(); InputStream is = c.getInputStream(); boolean foundEnd = false; @@ -539,7 +571,8 @@ public String[] list(boolean noHiddenFiles) { */ public boolean canRead() { LOGGER.trace("canRead()"); - return isURL ? (isDirectory() || isFile() || exists()) : file.canRead(); + // Note: isFile calls exist + return isURL ? (isDirectory() || isFile()) : file.canRead(); } /** @@ -642,7 +675,9 @@ public boolean exists() { LOGGER.trace("exists()"); if (isURL) { try { - url.getContent(); + // TODO: existence should almost certainly be cached. + IRandomAccess handle = getHandle(uri.toString()); + handle.length(); return true; } catch (IOException e) { @@ -675,7 +710,7 @@ public Location getAbsoluteFile() { */ public String getAbsolutePath() { LOGGER.trace("getAbsolutePath()"); - return isURL ? url.toExternalForm() : file.getAbsolutePath(); + return isURL ? uri.normalize().toString() : file.getAbsolutePath(); } /** @@ -715,9 +750,8 @@ public String getCanonicalPath() throws IOException { public String getName() { LOGGER.trace("getName()"); if (isURL) { - String name = url.getFile(); - name = name.substring(name.lastIndexOf("/") + 1); - return name; + // TODO: we should just store new File(uri) in file + return new File(uri.getPath()).getName(); } return file.getName(); } @@ -757,7 +791,7 @@ public Location getParentFile() { * @see java.io.File#getPath() */ public String getPath() { - return isURL ? url.getHost() + url.getPath() : file.getPath(); + return isURL ? uri.getHost() + uri.getPath() : file.getPath(); } /** @@ -769,7 +803,7 @@ public String getPath() { */ public boolean isAbsolute() { LOGGER.trace("isAbsolute()"); - return isURL ? true : file.isAbsolute(); + return isURL ? uri.isAbsolute() : file.isAbsolute(); } /** @@ -781,8 +815,13 @@ public boolean isAbsolute() { public boolean isDirectory() { LOGGER.trace("isDirectory()"); if (isURL) { - String[] list = list(); - return list != null; + if ("s3".equals(uri.getScheme())) { + return uri.toString().endsWith("/") && exists(); + } else { + // TODO: this should be removed as well. + String[] list = list(); + return list != null; + } } return file.isDirectory(); } @@ -829,7 +868,11 @@ public long lastModified() { LOGGER.trace("lastModified()"); if (isURL) { try { - return url.openConnection().getLastModified(); + if (url != null) { + return url.openConnection().getLastModified(); + } else { + return 0; + } } catch (IOException e) { LOGGER.trace("Could not determine URL's last modification time", e); @@ -908,7 +951,7 @@ public URL toURL() throws MalformedURLException { */ @Override public String toString() { - return isURL ? url.toString() : file.toString(); + return isURL ? uri.toString() : file.toString(); } } diff --git a/src/main/java/loci/common/S3Handle.java b/src/main/java/loci/common/S3Handle.java new file mode 100644 index 00000000..16f2f339 --- /dev/null +++ b/src/main/java/loci/common/S3Handle.java @@ -0,0 +1,195 @@ +/* + * #%L + * Common package for I/O and related utilities + * %% + * Copyright (C) 2018 Open Microscopy Environment: + * - Board of Regents of the University of Wisconsin-Madison + * - Glencoe Software, Inc. + * - University of Dundee + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package loci.common; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.net.ConnectException; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import io.minio.MinioClient; +import io.minio.ObjectStat; +import io.minio.errors.*; +import org.xmlpull.v1.XmlPullParserException; + +/** + * Provides random access to S3 buckets using the IRandomAccess interface. + * Instances of S3Handle are read-only. + * + * TODO: How does one handle buckets with periods + * + * @see IRandomAccess + * @see StreamHandle + * @see java.net.URLConnection + * + */ +public class S3Handle extends StreamHandle { + + public final static String DEFAULT_SERVER = "https://s3.amazonaws.com"; + + /** Format: "s3://accessKey:secrectKey@bucket.endpoint/key" */ + public final static String URI_PATTERN = "(s3://)?" + + "((?.*):(?.*)@)?" + + "(?.*?)"+ + "([.](?.*?)((:)(?\\d+))?)?"+ + "/(?.*)"; + + public final static Pattern URI_PARSER = Pattern.compile(URI_PATTERN); + + /** full string used to configure this handle */ + private final String uri; + + /** access key, if provided */ + private final String accessKey; + + /** secret key, if provided */ + private final String secretKey; + + /** name of the bucket */ + private final String bucket; + + /** endpoint to which requests will be sent */ + private final String server; + + /** port at the given server */ + private final int port; + + /** remaining path, or key, for this accessed resource */ + private final String path; + + private MinioClient s3Client; + + public S3Handle(String url) throws IOException { + this(null, url); + } + + public S3Handle(String server, String uri) throws IOException { + this(server, uri, true); + } + + public S3Handle(String uri, boolean initialize) throws IOException { + this(null, uri, initialize); + } + + public S3Handle(String server, String uri, boolean initialize) throws IOException { + this.uri = uri; + Matcher m = URI_PARSER.matcher(uri); + if (!m.matches()) { + throw new RuntimeException(String.format( + "%s does not match pattern %s", uri, URI_PATTERN)); + } + this.accessKey = m.group("access"); + this.secretKey = m.group("secret"); + this.bucket = m.group("bucket"); + this.server = server(m, server); + this.path = m.group("path"); + this.port = port(m); + if (initialize) { + resetStream(); + } + } + + private int port(Matcher m) { + String p = m.group("port"); + if (p == null) { + return 0; + } else { + return Integer.valueOf(p); + } + } + + private String server(Matcher m, String server) { + String s = m.group("server"); + if (s == null) { + s = server != null ? server : DEFAULT_SERVER; + } + if (!s.contains("://")) { + s = "http://" + s; + } + return s; + } + + public String getServer() { + return server; + } + + public int getPort() { + return port; + } + + public String getBucket() { + return bucket; + } + + public String getPath() { + return path; + } + + @Override + protected void resetStream() throws IOException { + try { + s3Client = new MinioClient(server, port, accessKey, secretKey); + ObjectStat stat = s3Client.statObject(bucket, path); + length = stat.length(); + stream = new DataInputStream(new BufferedInputStream( + s3Client.getObject(bucket, path))); + stream.skip(-1L); + fp = 0; + mark = 0; + } catch (ConnectException | + InvalidEndpointException | + InvalidPortException | + InvalidBucketNameException | + NoSuchAlgorithmException | + InsufficientDataException | + InvalidKeyException | + NoResponseException | + XmlPullParserException | + ErrorResponseException | + InternalException | + InvalidArgumentException e) { + throw new IOException(String.format( + "failed to load s3: %s\n" + + "\tserver:%s\n"+ + "\tport:%d\n"+ + "\tbucket:%s\n"+ + "\tpath:%s", uri, server, port, bucket, path), e); + } + } +} diff --git a/src/test/java/loci/common/utests/LocationTest.java b/src/test/java/loci/common/utests/LocationTest.java index ddd47e78..f4a69e81 100644 --- a/src/test/java/loci/common/utests/LocationTest.java +++ b/src/test/java/loci/common/utests/LocationTest.java @@ -36,6 +36,7 @@ import java.io.File; import java.io.IOException; +import java.net.MalformedURLException; import java.net.Socket; import java.net.URL; import java.util.Arrays; @@ -92,27 +93,29 @@ public void setup() throws IOException { new Location("http://www.openmicroscopy.org/"), new Location("https://www.openmicroscopy.org/"), new Location("https://www.openmicroscopy.org/nonexisting"), - new Location(hiddenFile) + new Location(hiddenFile), + new Location("s3://server/bucket-dne/key/"), + new Location("s3://server/bucket-dne/key/file.tif"), }; exists = new boolean[] { - true, false, true, true, true, false, true + true, false, true, true, true, false, true, false, false }; isDirectory = new boolean[] { - false, false, true, false, false, false, false + false, false, true, false, false, false, false, false, false }; isHidden = new boolean[] { - false, false, false, false, false, false, true + false, false, false, false, false, false, true, false, false }; mode = new String[] { - "rw", "", "rw", "r", "r", "","rw" + "rw", "", "rw", "r", "r", "","rw", "", "" // S3 isn't readable }; isRemote = new boolean[] { - false, false, false, true, true, true, false + false, false, false, true, true, true, false, true, true }; } @@ -244,7 +247,11 @@ public void testToURL() throws IOException { if (file.isDirectory() && !path.endsWith(File.separator)) { path += File.separator; } - assertEquals(file.getName(), file.toURL(), new URL(path)); + try { + assertEquals(file.getName(), file.toURL(), new URL(path)); + } catch (MalformedURLException e) { + assertEquals(path, true, path.contains("s3://")); + } } } diff --git a/src/test/java/loci/common/utests/S3HandleTest.java b/src/test/java/loci/common/utests/S3HandleTest.java new file mode 100644 index 00000000..b0c6acdf --- /dev/null +++ b/src/test/java/loci/common/utests/S3HandleTest.java @@ -0,0 +1,106 @@ +/* + * #%L + * Common package for I/O and related utilities + * %% + * Copyright (C) 2005 - 2016 Open Microscopy Environment: + * - Board of Regents of the University of Wisconsin-Madison + * - Glencoe Software, Inc. + * - University of Dundee + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package loci.common.utests; + +import loci.common.S3Handle; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.io.IOException; + +import static org.testng.AssertJUnit.assertEquals; + +/** + * Unit tests for the loci.common.S3Handle class. + * + * @see loci.common.URLHandle + */ +public class S3HandleTest { + + // -- Fields -- + + // -- Setup methods -- + + @BeforeMethod + public void setup() { + // no-op + } + + // -- Test methods -- + + @Test + public void testParseDefault() throws IOException { + S3Handle s3 = new S3Handle("s3://bucket/key/file.tif", false); + assertEquals(S3Handle.DEFAULT_SERVER, s3.getServer()); + assertEquals(0, s3.getPort()); + assertEquals("bucket", s3.getBucket()); + assertEquals("key/file.tif", s3.getPath()); + } + + @Test + public void testParseLocalhost() throws IOException { + S3Handle s3 = new S3Handle("s3://bucket.localhost:9000/key/file.tif", false); + assertEquals("http://localhost", s3.getServer()); + assertEquals(9000, s3.getPort()); + assertEquals("bucket", s3.getBucket()); + assertEquals("key/file.tif", s3.getPath()); + } + + @Test + public void testSetLocalhost() throws IOException { + S3Handle s3 = new S3Handle("localhost", "s3://bucket/key/file.tif", false); + assertEquals("http://localhost", s3.getServer()); + assertEquals(0, s3.getPort()); + assertEquals("bucket", s3.getBucket()); + assertEquals("key/file.tif", s3.getPath()); + } + + @Test + public void testParseAuth() throws IOException { + S3Handle s3 = new S3Handle("s3://access:secret@bucket/key/file.tif", false); + assertEquals(S3Handle.DEFAULT_SERVER, s3.getServer()); + assertEquals(0, s3.getPort()); + assertEquals("bucket", s3.getBucket()); + assertEquals("key/file.tif", s3.getPath()); + } + + @Test + public void testParseAuthLocalhost() throws IOException { + S3Handle s3 = new S3Handle("s3://access:secret@bucket.localhost:9000/key/file.tif", false); + assertEquals("http://localhost", s3.getServer()); + assertEquals(9000, s3.getPort()); + assertEquals("bucket", s3.getBucket()); + assertEquals("key/file.tif", s3.getPath()); + } + +}