diff --git a/pom.xml b/pom.xml index b6dd4529..decc5481 100644 --- a/pom.xml +++ b/pom.xml @@ -109,7 +109,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma commons-lang3 3.1 - + + + com.amazonaws + aws-java-sdk + 1.3.22 + diff --git a/src/main/java/org/chombo/util/Utility.java b/src/main/java/org/chombo/util/Utility.java index 6cc95b60..781b694b 100644 --- a/src/main/java/org/chombo/util/Utility.java +++ b/src/main/java/org/chombo/util/Utility.java @@ -29,6 +29,8 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; @@ -40,6 +42,12 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import com.amazonaws.auth.PropertiesCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.S3Object; + /** * Generic Utility * @author pranab @@ -50,9 +58,20 @@ public class Utility { private static final String FS_DEF_CONFIG_DIR = "/var/mawazo/"; private static final String HDFS_DEF_CONFIG_DIR = "/var/mawazo/"; private static final String HDFS_PREFIX = "hdfs:"; + private static final String S3_PREFIX = "s3n:"; private static final int HDFS_PREFIX_LEN = 5; private static final String PROP_FILE_EXT = ".properties"; + private static Pattern s3pattern = Pattern.compile("s3n:/+([^/]+)/+(.*)"); + static AmazonS3 s3 = null; + static { + try { + s3 = new AmazonS3Client(new PropertiesCredentials(Utility.class.getResourceAsStream("AwsCredentials.properties"))); + } + catch (Exception e) { + e.printStackTrace(); + } + } /** * @param conf * @throws Exception @@ -60,9 +79,20 @@ public class Utility { public static void setConfiguration(Configuration conf) throws Exception{ String confFilePath = conf.get("conf.path"); if (null != confFilePath){ - FileInputStream fis = new FileInputStream(confFilePath); + InputStream is = null; + if (confFilePath.startsWith(S3_PREFIX)) { + Matcher matcher = s3pattern.matcher(confFilePath); + matcher.matches(); + String bucket = matcher.group(1); + String key = matcher.group(2); + S3Object object = s3.getObject(new GetObjectRequest(bucket, key)); + is = object.getObjectContent(); + } + else { + is = new FileInputStream(confFilePath); + } Properties configProps = new Properties(); - configProps.load(fis); + configProps.load(is); for (Object key : configProps.keySet()){ String keySt = key.toString(); @@ -329,5 +359,4 @@ public static String[] filterOutFields(String[] items , int[] filteredFields) { } return extractedFields; } - }