diff --git a/pom.xml b/pom.xml
index b6dd4529..decc5481 100644
--- a/pom.xml
+++ b/pom.xml
@@ -109,7 +109,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
commons-lang3
3.1
-
+
+
+ com.amazonaws
+ aws-java-sdk
+ 1.3.22
+
diff --git a/src/main/java/org/chombo/util/Utility.java b/src/main/java/org/chombo/util/Utility.java
index 6cc95b60..781b694b 100644
--- a/src/main/java/org/chombo/util/Utility.java
+++ b/src/main/java/org/chombo/util/Utility.java
@@ -29,6 +29,8 @@
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
@@ -40,6 +42,12 @@
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import com.amazonaws.auth.PropertiesCredentials;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.S3Object;
+
/**
* Generic Utility
* @author pranab
@@ -50,9 +58,20 @@ public class Utility {
private static final String FS_DEF_CONFIG_DIR = "/var/mawazo/";
private static final String HDFS_DEF_CONFIG_DIR = "/var/mawazo/";
private static final String HDFS_PREFIX = "hdfs:";
+ private static final String S3_PREFIX = "s3n:";
private static final int HDFS_PREFIX_LEN = 5;
private static final String PROP_FILE_EXT = ".properties";
+ private static Pattern s3pattern = Pattern.compile("s3n:/+([^/]+)/+(.*)");
+ static AmazonS3 s3 = null;
+ static {
+ try {
+ s3 = new AmazonS3Client(new PropertiesCredentials(Utility.class.getResourceAsStream("AwsCredentials.properties")));
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
/**
* @param conf
* @throws Exception
@@ -60,9 +79,20 @@ public class Utility {
public static void setConfiguration(Configuration conf) throws Exception{
String confFilePath = conf.get("conf.path");
if (null != confFilePath){
- FileInputStream fis = new FileInputStream(confFilePath);
+ InputStream is = null;
+ if (confFilePath.startsWith(S3_PREFIX)) {
+ Matcher matcher = s3pattern.matcher(confFilePath);
+ matcher.matches();
+ String bucket = matcher.group(1);
+ String key = matcher.group(2);
+ S3Object object = s3.getObject(new GetObjectRequest(bucket, key));
+ is = object.getObjectContent();
+ }
+ else {
+ is = new FileInputStream(confFilePath);
+ }
Properties configProps = new Properties();
- configProps.load(fis);
+ configProps.load(is);
for (Object key : configProps.keySet()){
String keySt = key.toString();
@@ -329,5 +359,4 @@ public static String[] filterOutFields(String[] items , int[] filteredFields) {
}
return extractedFields;
}
-
}