AllAlertAdapterTest instance.
+ * @param name
+ */
+ public AllAlertAdapterTest(String name) {
+ super(name);
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ protected static void setUpBeforeClass() throws Exception {
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ protected static void tearDownAfterClass() throws Exception {
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see junit.framework.TestCase#setUp()
+ */
+
+ @SuppressWarnings("unchecked")
+ protected void setUp() throws Exception {
+ super.setUp("com.opensoc.alerts.adapters.AllAlertAdapter");
+ Properties prop = super.getTestProperties();
+ assertNotNull(prop);
+ // this.setMode("global");
+ if(skipTests(this.getMode())){
+ System.out.println(getClass().getName()+" Skipping Tests !!Local Mode");
+ return;//skip tests
+ }else{
+ Map
@@ -76,12 +71,9 @@ public void execute(Tuple input) {
try {
this.connector.getTable().put(conf.getPutFromTuple(input));
} catch (IOException ex) {
-
- String error_as_string = org.apache.commons.lang.exception.ExceptionUtils
- .getStackTrace(ex);
JSONObject error = ErrorGenerator.generateErrorMessage(
- "Alerts problem: " + input.getBinary(0), error_as_string);
+ "Alerts problem: " + input.getBinary(0), ex);
collector.emit("error", new Values(error));
throw new RuntimeException(ex);
diff --git a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/helpers/services/PcapServiceCli.java b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/helpers/services/PcapServiceCli.java
new file mode 100644
index 0000000000..70f8683613
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/helpers/services/PcapServiceCli.java
@@ -0,0 +1,110 @@
+package com.opensoc.helpers.services;
+
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+public class PcapServiceCli {
+
+ private String[] args = null;
+ private Options options = new Options();
+
+ int port = 8081;
+ String uri = "/pcapGetter";
+
+ public int getPort() {
+ return port;
+ }
+
+ public void setPort(int port) {
+ this.port = port;
+ }
+
+ public String getUri() {
+ return uri;
+ }
+
+ public void setUri(String uri) {
+ this.uri = uri;
+ }
+
+ public PcapServiceCli(String[] args) {
+
+ this.args = args;
+
+ Option help = new Option("h", "Display help menue");
+ options.addOption(help);
+ options.addOption(
+ "port",
+ true,
+ "OPTIONAL ARGUMENT [portnumber] If this argument sets the port for starting the service. If this argument is not set the port will start on defaut port 8081");
+ options.addOption(
+ "endpoint_uri",
+ true,
+ "OPTIONAL ARGUMENT [/uri/to/service] This sets the URI for the service to be hosted. The default URI is /pcapGetter");
+ }
+
+ public void parse() {
+ CommandLineParser parser = new BasicParser();
+
+ CommandLine cmd = null;
+
+ try {
+ cmd = parser.parse(options, args);
+ } catch (ParseException e1) {
+
+ e1.printStackTrace();
+ }
+
+ if (cmd.hasOption("h"))
+ help();
+
+ if (cmd.hasOption("port")) {
+
+ try {
+ port = Integer.parseInt(cmd.getOptionValue("port").trim());
+ } catch (Exception e) {
+
+ System.out.println("[OpenSOC] Invalid value for port entered");
+ help();
+ }
+ }
+ if (cmd.hasOption("endpoint_uri")) {
+
+ try {
+
+ if (uri == null || uri.equals(""))
+ throw new Exception("invalid uri");
+
+ uri = cmd.getOptionValue("uri").trim();
+
+ if (uri.charAt(0) != '/')
+ uri = "/" + uri;
+
+ if (uri.charAt(uri.length()) == '/')
+ uri = uri.substring(0, uri.length() - 1);
+
+ } catch (Exception e) {
+ System.out.println("[OpenSOC] Invalid URI entered");
+ help();
+ }
+ }
+
+ }
+
+ private void help() {
+ // This prints out some help
+ HelpFormatter formater = new HelpFormatter();
+
+ formater.printHelp("Topology Options:", options);
+
+ // System.out
+ // .println("[OpenSOC] Example usage: \n storm jar OpenSOC-Topologies-0.3BETA-SNAPSHOT.jar com.opensoc.topology.Bro -local_mode true -config_path OpenSOC_Configs/ -generator_spout true");
+
+ System.exit(0);
+ }
+}
diff --git a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/Cli.java b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/helpers/topology/Cli.java
similarity index 99%
rename from opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/Cli.java
rename to opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/helpers/topology/Cli.java
index 9f8bae3fd2..0d9486e3be 100644
--- a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/Cli.java
+++ b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/helpers/topology/Cli.java
@@ -1,4 +1,4 @@
-package com.opensoc.topologyhelpers;
+package com.opensoc.helpers.topology;
import java.io.File;
diff --git a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/ErrorGenerator.java b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/helpers/topology/ErrorGenerator.java
similarity index 52%
rename from opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/ErrorGenerator.java
rename to opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/helpers/topology/ErrorGenerator.java
index c21205e894..97f0ba7aeb 100644
--- a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/ErrorGenerator.java
+++ b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/helpers/topology/ErrorGenerator.java
@@ -1,26 +1,36 @@
-package com.opensoc.topologyhelpers;
+package com.opensoc.helpers.topology;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import org.apache.commons.lang.exception.ExceptionUtils;
import org.json.simple.JSONObject;
public class ErrorGenerator {
- public static JSONObject generateErrorMessage(String message, String exception)
+ @SuppressWarnings("unchecked")
+ public static JSONObject generateErrorMessage(String message, Exception e)
{
JSONObject error_message = new JSONObject();
+ /*
+ * Save full stack trace in object.
+ */
+ String stackTrace = ExceptionUtils.getStackTrace(e);
+
+ String exception = e.toString();
+
error_message.put("time", System.currentTimeMillis());
try {
error_message.put("hostname", InetAddress.getLocalHost().getHostName());
- } catch (UnknownHostException e) {
+ } catch (UnknownHostException ex) {
// TODO Auto-generated catch block
- e.printStackTrace();
+ ex.printStackTrace();
}
error_message.put("message", message);
error_message.put("exception", exception);
+ error_message.put("stack", stackTrace);
return error_message;
}
diff --git a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/SettingsLoader.java b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/helpers/topology/SettingsLoader.java
similarity index 80%
rename from opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/SettingsLoader.java
rename to opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/helpers/topology/SettingsLoader.java
index bb2a460694..261d4816bd 100644
--- a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/topologyhelpers/SettingsLoader.java
+++ b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/helpers/topology/SettingsLoader.java
@@ -1,4 +1,4 @@
-package com.opensoc.topologyhelpers;
+package com.opensoc.helpers.topology;
import java.util.HashMap;
import java.util.Iterator;
@@ -15,6 +15,7 @@
public class SettingsLoader {
+ @SuppressWarnings("unchecked")
public static JSONObject loadEnvironmentIdnetifier(String config_path)
throws ConfigurationException {
Configuration config = new PropertiesConfiguration(config_path);
@@ -31,6 +32,7 @@ public static JSONObject loadEnvironmentIdnetifier(String config_path)
return identifier;
}
+ @SuppressWarnings("unchecked")
public static JSONObject loadTopologyIdnetifier(String config_path)
throws ConfigurationException {
Configuration config = new PropertiesConfiguration(config_path);
@@ -52,6 +54,7 @@ public static String generateTopologyName(JSONObject env, JSONObject topo) {
+ env.get("instance") + "_" + topo.get("topology") + "_" + topo.get("topology_instance"));
}
+ @SuppressWarnings("unchecked")
public static JSONObject generateAlertsIdentifier(JSONObject env, JSONObject topo)
{
JSONObject identifier = new JSONObject();
@@ -67,7 +70,7 @@ public static Map
+ *
+ * @version $Revision: 1.1 $
+ */
+public class AbstractConfigTest extends AbstractTestContext{
+ /**
+ * The configPath.
+ */
+ protected String configPath=null;
+
+ /**
+ * The configName.
+ */
+ protected String configName=null;
+
+ /**
+ * The config.
+ */
+ private Configuration config=null;
+
+ /**
+ * The settings.
+ */
+ MapAbstractConfigTest is
+ * an abstract base class for implementing JUnit tests that need to use
+ * config to connect to ZooKeeper and HBase. The setup method will attempt to
+ * load a properties from a file, located in src/test/resources,
+ * with the same name as the class.AbstractConfigTest instance.
+ * @throws Exception
+ */
+ public AbstractConfigTest() throws Exception {
+ super.setUp();
+ }
+
+ /**
+ * Constructs a new AbstractTestContext instance.
+ * @param name the name of the test case.
+ */
+ public AbstractConfigTest(String name) {
+ super(name);
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see junit.framework.TestCase#setUp()
+ */
+ protected void setUp(String configName) throws Exception {
+ super.setUp();
+ this.setConfigPath("src/test/resources/config/"+getClass().getSimpleName()+".config");
+ try {
+ this.setConfig(new PropertiesConfiguration(this.getConfigPath()));
+
+ Map configOptions= SettingsLoader.getConfigOptions((PropertiesConfiguration)this.config, configName+"=");
+ this.setSettings(SettingsLoader.getConfigOptions((PropertiesConfiguration)this.config, configName + "."));
+ this.getSettings().put(configName, (String) configOptions.get(configName));
+ } catch (ConfigurationException e) {
+ fail("Config not found !!"+e);
+ e.printStackTrace();
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see junit.framework.TestCase#tearDown()
+ */
+ @Override
+ protected void tearDown() throws Exception {
+
+ }
+
+
+ /**
+ * validateJsonData
+ * @param jsonSchema
+ * @param jsonData
+ * @return
+ * @throws Exception
+ */
+
+ protected boolean validateJsonData(final String jsonSchema, final String jsonData)
+ throws Exception {
+
+ final JsonNode d = JsonLoader.fromString(jsonData);
+ final JsonNode s = JsonLoader.fromString(jsonSchema);
+
+ final JsonSchemaFactory factory = JsonSchemaFactory.byDefault();
+ JsonValidator v = factory.getValidator();
+
+ ProcessingReport report = v.validate(s, d);
+ System.out.println(report);
+
+ return report.toString().contains("success");
+ }
+
+ protected String readSchemaFromFile(URL schema_url) throws Exception {
+ BufferedReader br = new BufferedReader(new FileReader(
+ schema_url.getFile()));
+ String line;
+ StringBuilder sb = new StringBuilder();
+ while ((line = br.readLine()) != null) {
+ System.out.println(line);
+ sb.append(line);
+ }
+ br.close();
+
+ String schema_string = sb.toString().replaceAll("\n", "");
+ schema_string = schema_string.replaceAll(" ", "");
+
+ System.out.println("Read in schema: " + schema_string);
+
+ return schema_string;
+ }
+
+ protected String[] readTestDataFromFile(String test_data_url) throws Exception {
+ BufferedReader br = new BufferedReader(new FileReader(
+ new File(test_data_url)));
+ ArrayList
+ *
+ * @version $Revision: 1.1 $
+ */
+public class AbstractSchemaTest extends AbstractConfigTest{
+
+
+ /**
+ * The schemaJsonString.
+ */
+ private String schemaJsonString = null;
+ /**
+ * Any Object for mavenMode
+ * @parameter
+ * expression="${mode}"
+ * default-value="local"
+ */
+ private Object mode="local";
+
+ /**
+ * Constructs a new AbstractSchemaTest is
+ * an abstract base class for implementing JUnit tests that need to load a
+ * Json Schema. The setup method will attempt to
+ * load a properties from a file, located in src/test/resources,
+ * with the same name as the class.AbstractTestContext instance.
+ * @throws Exception
+ */
+ public AbstractSchemaTest() throws Exception {
+ super.setUp();
+ }
+
+ /**
+ * Constructs a new AbstractTestContext instance.
+ * @param name the name of the test case.
+ */
+ public AbstractSchemaTest(String name) {
+ super(name);
+ try{
+ if(System.getProperty("mode")!=null){
+ setMode(System.getProperty("mode") );
+ }else
+ {
+ setMode("local");
+ }
+ }catch(Exception ex){
+ setMode("local");
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see junit.framework.TestCase#setUp()
+ */
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see junit.framework.TestCase#tearDown()
+ */
+ @Override
+ protected void tearDown() throws Exception {
+
+ }
+
+
+ /**
+ * validateJsonData
+ * @param jsonSchema
+ * @param jsonData
+ * @return
+ * @throws Exception
+ */
+
+ protected boolean validateJsonData(final String jsonSchema, final String jsonData)
+ throws Exception {
+
+ final JsonNode d = JsonLoader.fromString(jsonData);
+ final JsonNode s = JsonLoader.fromString(jsonSchema);
+
+ final JsonSchemaFactory factory = JsonSchemaFactory.byDefault();
+ JsonValidator v = factory.getValidator();
+
+ ProcessingReport report = v.validate(s, d);
+ System.out.println(report);
+
+ return report.toString().contains("success");
+ }
+
+ protected String readSchemaFromFile(URL schema_url) throws Exception {
+ BufferedReader br = new BufferedReader(new FileReader(
+ schema_url.getFile()));
+ String line;
+ StringBuilder sb = new StringBuilder();
+ while ((line = br.readLine()) != null) {
+ System.out.println(line);
+ sb.append(line);
+ }
+ br.close();
+
+ String schema_string = sb.toString().replaceAll("\n", "");
+ schema_string = schema_string.replaceAll(" ", "");
+
+ System.out.println("Read in schema: " + schema_string);
+
+ return schema_string;
+
+ }
+
+ /**
+ * Skip Tests
+ */
+ public boolean skipTests(Object mode){
+ if(mode.toString().equals("local")){
+ return true;
+ }else {
+ return false;
+ }
+ }
+
+ /**
+ * Returns the mode.
+ * @return the mode.
+ */
+
+ public Object getMode() {
+ return mode;
+ }
+
+ /**
+ * Sets the mode.
+ * @param mode the mode.
+ */
+
+ public void setMode(Object mode) {
+
+ this.mode = mode;
+ }
+
+
+ /**
+
+ * @param readSchemaFromFile
+ */
+
+ public void setSchemaJsonString(String schemaJsonString) {
+ this.schemaJsonString=schemaJsonString;
+ }
+
+
+ /**
+
+ * @return
+ */
+
+ public String getSchemaJsonString() {
+ return this.schemaJsonString;
+ }
+
+}
+
+
diff --git a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractTestContext.java b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractTestContext.java
index 7f7f34a721..ea5b04fb6d 100644
--- a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractTestContext.java
+++ b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/AbstractTestContext.java
@@ -52,7 +52,7 @@ public class AbstractTestContext extends TestCase{
* Any Object for mavenMode
* @parameter
* expression="${mode}"
- * default-value="local"
+ * default-value="global"
*/
private Object mode="local";
diff --git a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/ISEParserTest.java b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/ISEParserTest.java
deleted file mode 100644
index 47061b6dbd..0000000000
--- a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/test/ISEParserTest.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package com.opensoc.test;
-
-import java.io.BufferedReader;
-import java.io.DataInputStream;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.StringReader;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.json.simple.JSONObject;
-
-import com.opensoc.ise.parser.ISEParser;
-import com.opensoc.ise.parser.ParseException;
-
-public class ISEParserTest {
-
- public static void main(String[] args) throws ParseException, IOException {
- }
-
-}
-
-
diff --git a/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/tldextractor/BasicTldExtractor.java b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/tldextractor/BasicTldExtractor.java
new file mode 100644
index 0000000000..2dbcd955ba
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Common/src/main/java/com/opensoc/tldextractor/BasicTldExtractor.java
@@ -0,0 +1,137 @@
+package com.opensoc.tldextractor;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class BasicTldExtractor implements Serializable {
+ private static final long serialVersionUID = -7440226111118873815L;
+ private StringBuilder sb = new StringBuilder();
+
+ private Pattern pattern;
+
+ /**
+ * The inputFile.
+ */
+ private String inputFile ="effective_tld_names.dat";
+
+ public BasicTldExtractor(String filePath) {
+ this.inputFile=filePath;
+ this.init();
+ }
+
+ public BasicTldExtractor() {
+ this.init();
+ }
+
+ private void init(){
+ try {
+ ArrayList
+ *
+ * @author $Author: $
+ * @version $Revision: 1.1 $
+ */
+public class BasicTldExtractorTest extends AbstractConfigTest {
+ /**
+ * The tldExtractor.
+ */
+
+ private BasicTldExtractor tldExtractor=null;
+
+ /**
+ * Constructs a new BasicTldExtractorTest instance.
+ * @param name
+ */
+
+ public BasicTldExtractorTest(String name) {
+ super(name);
+ }
+
+ /**
+
+ * @throws java.lang.Exception
+ */
+ protected static void setUpBeforeClass() throws Exception {
+ }
+
+ /**
+
+ * @throws java.lang.Exception
+ */
+ protected static void tearDownAfterClass() throws Exception {
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see junit.framework.TestCase#setUp()
+ */
+
+ protected void setUp() throws Exception {
+ super.setUp("com.opensoc.tldextractor.test.BasicTldExtractorTest");
+ this.tldExtractor=new BasicTldExtractor(this.getConfig().getString("logFile"));
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see junit.framework.TestCase#tearDown()
+ */
+
+ protected void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ /**
+ * Test method for {@link com.opensoc.tldextractor.BasicTldExtractor#BasicTldExtractor()}.
+ */
+ public void testBasicTldExtractor() {
+ assertNotNull(this.tldExtractor);
+ }
+
+ /**
+ * Test method for {@link com.opensoc.tldextractor.BasicTldExtractor#extract2LD(java.lang.String)}.
+ */
+ public void testExtract2LD() {
+ //fail("Not yet implemented");
+ }
+
+ /**
+ * Test method for {@link com.opensoc.tldextractor.BasicTldExtractor#extractTLD(java.lang.String)}.
+ */
+ public void testExtractTLD()
+ {
+ String result = this.tldExtractor.extractTLD("cisco.com");
+ System.out.println("result ="+result);
+ }
+ /**
+ * Returns the tldExtractor.
+ * @return the tldExtractor.
+ */
+
+ public BasicTldExtractor getTldExtractor() {
+ return tldExtractor;
+ }
+
+ /**
+ * Sets the tldExtractor.
+ * @param tldExtractor the tldExtractor.
+ */
+
+ public void setTldExtractor(BasicTldExtractor tldExtractor) {
+
+ this.tldExtractor = tldExtractor;
+ }
+
+}
+
diff --git a/opensoc-streaming/OpenSOC-Common/src/test/resources/config/BasicTldExtractorTest.config b/opensoc-streaming/OpenSOC-Common/src/test/resources/config/BasicTldExtractorTest.config
new file mode 100644
index 0000000000..6b3dc05f02
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Common/src/test/resources/config/BasicTldExtractorTest.config
@@ -0,0 +1,2 @@
+#BasicTldExtractorConfig
+logFile=effective_tld_names.dat
diff --git a/opensoc-streaming/OpenSOC-Common/src/test/resources/effective_tld_names.dat b/opensoc-streaming/OpenSOC-Common/src/test/resources/effective_tld_names.dat
new file mode 100644
index 0000000000..36e5d4c1c8
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-Common/src/test/resources/effective_tld_names.dat
@@ -0,0 +1,9719 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// ===BEGIN ICANN DOMAINS===
+
+// ac : http://en.wikipedia.org/wiki/.ac
+ac
+com.ac
+edu.ac
+gov.ac
+net.ac
+mil.ac
+org.ac
+
+// ad : http://en.wikipedia.org/wiki/.ad
+ad
+nom.ad
+
+// ae : http://en.wikipedia.org/wiki/.ae
+// see also: "Domain Name Eligibility Policy" at http://www.aeda.ae/eng/aepolicy.php
+ae
+co.ae
+net.ae
+org.ae
+sch.ae
+ac.ae
+gov.ae
+mil.ae
+
+// aero : see http://www.information.aero/index.php?id=66
+aero
+accident-investigation.aero
+accident-prevention.aero
+aerobatic.aero
+aeroclub.aero
+aerodrome.aero
+agents.aero
+aircraft.aero
+airline.aero
+airport.aero
+air-surveillance.aero
+airtraffic.aero
+air-traffic-control.aero
+ambulance.aero
+amusement.aero
+association.aero
+author.aero
+ballooning.aero
+broker.aero
+caa.aero
+cargo.aero
+catering.aero
+certification.aero
+championship.aero
+charter.aero
+civilaviation.aero
+club.aero
+conference.aero
+consultant.aero
+consulting.aero
+control.aero
+council.aero
+crew.aero
+design.aero
+dgca.aero
+educator.aero
+emergency.aero
+engine.aero
+engineer.aero
+entertainment.aero
+equipment.aero
+exchange.aero
+express.aero
+federation.aero
+flight.aero
+freight.aero
+fuel.aero
+gliding.aero
+government.aero
+groundhandling.aero
+group.aero
+hanggliding.aero
+homebuilt.aero
+insurance.aero
+journal.aero
+journalist.aero
+leasing.aero
+logistics.aero
+magazine.aero
+maintenance.aero
+marketplace.aero
+media.aero
+microlight.aero
+modelling.aero
+navigation.aero
+parachuting.aero
+paragliding.aero
+passenger-association.aero
+pilot.aero
+press.aero
+production.aero
+recreation.aero
+repbody.aero
+res.aero
+research.aero
+rotorcraft.aero
+safety.aero
+scientist.aero
+services.aero
+show.aero
+skydiving.aero
+software.aero
+student.aero
+taxi.aero
+trader.aero
+trading.aero
+trainer.aero
+union.aero
+workinggroup.aero
+works.aero
+
+// af : http://www.nic.af/help.jsp
+af
+gov.af
+com.af
+org.af
+net.af
+edu.af
+
+// ag : http://www.nic.ag/prices.htm
+ag
+com.ag
+org.ag
+net.ag
+co.ag
+nom.ag
+
+// ai : http://nic.com.ai/
+ai
+off.ai
+com.ai
+net.ai
+org.ai
+
+// al : http://www.ert.gov.al/ert_alb/faq_det.html?Id=31
+al
+com.al
+edu.al
+gov.al
+mil.al
+net.al
+org.al
+
+// am : http://en.wikipedia.org/wiki/.am
+am
+
+// an : http://www.una.an/an_domreg/default.asp
+an
+com.an
+net.an
+org.an
+edu.an
+
+// ao : http://en.wikipedia.org/wiki/.ao
+// http://www.dns.ao/REGISTR.DOC
+ao
+ed.ao
+gv.ao
+og.ao
+co.ao
+pb.ao
+it.ao
+
+// aq : http://en.wikipedia.org/wiki/.aq
+aq
+
+// ar : https://nic.ar/normativa-vigente.xhtml
+ar
+com.ar
+edu.ar
+gob.ar
+gov.ar
+int.ar
+mil.ar
+net.ar
+org.ar
+tur.ar
+
+// arpa : http://en.wikipedia.org/wiki/.arpa
+// Confirmed by registry --table Source class to use
+ --table CIFHbaseAdapterTest instance.
* @param name
@@ -70,8 +72,33 @@ protected static void tearDownAfterClass() throws Exception {
protected void setUp() throws Exception {
super.setUp();
+
Properties prop = super.getTestProperties();
assertNotNull(prop);
+
+ if(skipTests(this.getMode())){
+ return;//skip tests
+ }
+
+ String[] zk = prop.get("kafka.zk.list").toString().split(",");
+
+ for(String z : zk)
+ {
+ InetAddress address = InetAddress.getByName(z);
+ boolean reachable = address.isReachable(100);
+
+ if(!reachable)
+ {
+ this.setMode("local");
+ //throw new Exception("Unable to reach zookeeper, skipping CIF adapter test");
+ break;
+ }
+
+ }
+
+ if(skipTests(this.getMode()))
+ return;//skip tests
+
System.out.println("kafka.zk.list ="+(String) prop.get("kafka.zk.list"));
System.out.println("kafka.zk.list ="+(String) prop.get("kafka.zk.port"));
System.out.println("kafka.zk.list ="+(String) prop.get("bolt.enrichment.cif.tablename"));
diff --git a/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/java/com/opensoc/enrichment/adapters/geo/GeoMysqlAdapterTest.java b/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/java/com/opensoc/enrichment/adapters/geo/GeoMysqlAdapterTest.java
index 173819b14a..ca545002fc 100644
--- a/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/java/com/opensoc/enrichment/adapters/geo/GeoMysqlAdapterTest.java
+++ b/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/java/com/opensoc/enrichment/adapters/geo/GeoMysqlAdapterTest.java
@@ -16,21 +16,22 @@
*/
package com.opensoc.enrichment.adapters.geo;
+import java.net.URL;
import java.util.Properties;
import org.json.simple.JSONObject;
-import com.opensoc.test.AbstractTestContext;
+import com.opensoc.test.AbstractSchemaTest;
/**
*
- *
* @version $Revision: 1.1 $
*/
-public class GeoMysqlAdapterTest extends AbstractTestContext {
+public class GeoMysqlAdapterTest extends AbstractSchemaTest {
private static GeoMysqlAdapter geoMySqlAdapter=null;
private static boolean connected=false;
@@ -72,9 +73,12 @@ protected void setUp() throws Exception {
System.out.println(getClass().getName()+" Skipping Tests !!Local Mode");
return;//skip tests
}else{
- geoMySqlAdapter=new GeoMysqlAdapter((String)prop.get("mysql.ip"), (new Integer((String)prop.get("mysql.port"))).intValue(),(String)prop.get("mysql.username"),(String)prop.get("mysql.password"), (String)prop.get("bolt.enrichment.geo.adapter.table"));
- connected =geoMySqlAdapter.initializeAdapter();
- assertTrue(connected);
+ GeoMysqlAdapterTest.setGeoMySqlAdapter(new GeoMysqlAdapter((String)prop.get("mysql.ip"), (new Integer((String)prop.get("mysql.port"))).intValue(),(String)prop.get("mysql.username"),(String)prop.get("mysql.password"), (String)prop.get("bolt.enrichment.geo.adapter.table")));
+ connected =geoMySqlAdapter.initializeAdapter();
+ assertTrue(connected);
+ URL schema_url = getClass().getClassLoader().getResource(
+ "TestSchemas/GeoMySqlSchema.json");
+ super.setSchemaJsonString(super.readSchemaFromFile(schema_url));
}
}
@@ -85,7 +89,7 @@ protected void setUp() throws Exception {
protected void tearDown() throws Exception {
super.tearDown();
- geoMySqlAdapter=null;
+ GeoMysqlAdapterTest.setGeoMySqlAdapter(null);
}
/**
@@ -95,16 +99,24 @@ public void testEnrich() {
if(skipTests(this.getMode())){
return;//skip tests
}else{
- JSONObject json = geoMySqlAdapter.enrich("72.163.4.161");
-
- //assert Geo Response is not null
- assertNotNull(json);
-
- //assert LocId is not null
- assertNotNull(json.get("locID"));
+
+ try {
+ JSONObject json = geoMySqlAdapter.enrich("72.163.4.161");
+
+ //assert Geo Response is not null
+ System.out.println("json ="+json);
+ assertNotNull(json);
- //assert right LocId is being returned
- assertEquals("4522",json.get("locID"));
+ assertEquals(true, super.validateJsonData(super.getSchemaJsonString(), json.toString()));
+ //assert LocId is not null
+ assertNotNull(json.get("locID"));
+
+ //assert right LocId is being returned
+ assertEquals("4522",json.get("locID"));
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail("Json validation Failed");
+ }
}
}
diff --git a/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/java/com/opensoc/enrichment/adapters/whois/WhoisHBaseAdapterTest.java b/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/java/com/opensoc/enrichment/adapters/whois/WhoisHBaseAdapterTest.java
index 3d2e2193e8..3057c13a2c 100644
--- a/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/java/com/opensoc/enrichment/adapters/whois/WhoisHBaseAdapterTest.java
+++ b/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/java/com/opensoc/enrichment/adapters/whois/WhoisHBaseAdapterTest.java
@@ -18,6 +18,7 @@
*/
package com.opensoc.enrichment.adapters.whois;
+import java.net.InetAddress;
import java.util.Properties;
import org.json.simple.JSONObject;
@@ -68,13 +69,39 @@ protected void setUp() throws Exception {
super.setUp();
Properties prop = super.getTestProperties();
assertNotNull(prop);
+
if(skipTests(this.getMode())){
return;//skip tests
- }else{
+ }
+
+ String[] zk = prop.get("kafka.zk.list").toString().split(",");
+
+ for(String z : zk)
+ {
+ InetAddress address = InetAddress.getByName(z);
+ boolean reachable = address.isReachable(100);
+
+ if(!reachable)
+ {
+ this.setMode("local");
+ break;
+ //throw new Exception("Unable to reach zookeeper, skipping WHois adapter test");
+ }
+
+ System.out.println("kafka.zk.list ="+(String) prop.get("kafka.zk.list"));
+ System.out.println("kafka.zk.list ="+(String) prop.get("kafka.zk.port"));
+ System.out.println("kafka.zk.list ="+(String) prop.get("bolt.enrichment.cif.tablename"));
+
+ }
+
+ if(skipTests(this.getMode())){
+ System.out.println("Local Mode Skipping tests !! ");
+ }else{
whoisHbaseAdapter=new WhoisHBaseAdapter((String)prop.get("bolt.enrichment.whois.hbase.table.name"),(String)prop.get("kafka.zk.list"),(String)prop.get("kafka.zk.port"));
connected =whoisHbaseAdapter.initializeAdapter();
assertTrue(connected);
- }
+ }
+
}
/*
diff --git a/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/CIFHbaseAdapterTest.properties b/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/CIFHbaseAdapterTest.properties
index 43ef4f6d25..8217353a69 100644
--- a/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/CIFHbaseAdapterTest.properties
+++ b/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/CIFHbaseAdapterTest.properties
@@ -1,11 +1,11 @@
kafka.zk.port=2181
-kafka.zk.list=zkpr1,zkpr2,zkpr3
-kafka.zk=zkpr1:2181,zkpr2:2181,zkpr3:2181
+kafka.zk.list=zkpr1
+kafka.zk=zkpr1:2181
#CIF Enrichment
bolt.enrichment.cif.tablename=cif_table
bolt.enrichment.cif.host=tld
bolt.enrichment.cif.email=email
-bolt.enrichment.cif.MAX_CACHE_SIZE=10000
-bolt.enrichment.cif.MAX_TIME_RETAIN=10
+bolt.enrichment.cif.MAX_CACHE_SIZE_OBJECTS_NUM=10000
+bolt.enrichment.cif.MAX_TIME_RETAIN_MINUTES=10
bolt.enrichment.cif.enrichment_tag=cif
diff --git a/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/GeoMysqlAdapterTest.properties b/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/GeoMysqlAdapterTest.properties
index fe95233456..3a4e17964a 100644
--- a/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/GeoMysqlAdapterTest.properties
+++ b/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/GeoMysqlAdapterTest.properties
@@ -1,11 +1,11 @@
mysql.ip=172.30.9.120
-mysql.port=0
+mysql.port=3306
mysql.username=test
mysql.password=123123
#GeoEnrichment
-
bolt.enrichment.geo.enrichment_tag=geo
bolt.enrichment.geo.adapter.table=GEO
-bolt.enrichment.geo.MAX_CACHE_SIZE=10000
-bolt.enrichment.geo.MAX_TIME_RETAIN=10
+bolt.enrichment.geo.MAX_CACHE_SIZE_OBJECTS_NUM=10000
+bolt.enrichment.geo.MAX_TIME_RETAIN_MINUTES=10
+bolt.enrichment.geo.source=ip_src_addr,ip_dst_addr
diff --git a/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/TestSchemas/CIFHbaseSchema.json b/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/TestSchemas/CIFHbaseSchema.json
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/TestSchemas/GeoMySqlSchema.json b/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/TestSchemas/GeoMySqlSchema.json
new file mode 100644
index 0000000000..c4f2a82ed2
--- /dev/null
+++ b/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/TestSchemas/GeoMySqlSchema.json
@@ -0,0 +1,42 @@
+{
+"title": "GeoMySql Schema",
+"type": "object",
+"properties": {
+
+ "city" : {
+ "type": "string"
+ },
+ "country" : {
+ "type": "string"
+ },
+ "dmaCode" :
+ {
+ "type": "string"
+ },
+ "geoHash" :
+ {
+ "type": "string"
+ },
+ "latitude" :
+ {
+ "type": "string"
+ },
+ "locID" :
+ {
+ "type": "string"
+ },
+ "location_point" :
+ {
+ "type": "string"
+ },
+ "longitude" :
+ {
+ "type": "string"
+ },
+ "postalCode" :
+ {
+ "type": "string"
+ }
+ },
+ "required": ["city", "country", "dmaCode","latitude","locID","location_point","postalCode"]
+}
diff --git a/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/TestSchemas/WhoisHbaseSchema.json b/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/TestSchemas/WhoisHbaseSchema.json
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/WhoisHbaseAdapterTest.properties b/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/WhoisHbaseAdapterTest.properties
index b80dfcd443..4f264ed3c5 100644
--- a/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/WhoisHbaseAdapterTest.properties
+++ b/opensoc-streaming/OpenSOC-EnrichmentAdapters/src/test/resources/WhoisHbaseAdapterTest.properties
@@ -1,11 +1,11 @@
kafka.zk.port=2181
-kafka.zk.list=zkpr1,zkpr2,zkpr3
-kafka.zk=zkpr1:2181,zkpr2:2181,zkpr3:2181
+kafka.zk.list=zkpr1
+kafka.zk=zkpr1:2181
#WhoisEnrichment
bolt.enrichment.whois.hbase.table.name=whois
bolt.enrichment.whois.enrichment_tag=whois
bolt.enrichment.whois.source=tld
-bolt.enrichment.whois.MAX_CACHE_SIZE=10000
-bolt.enrichment.whois.MAX_TIME_RETAIN=10
+bolt.enrichment.whois.MAX_CACHE_SIZE_OBJECTS_NUM=10000
+bolt.enrichment.whois.MAX_TIME_RETAIN_MINUTES=10
diff --git a/opensoc-streaming/OpenSOC-Indexing/pom.xml b/opensoc-streaming/OpenSOC-Indexing/pom.xml
index 96e2bd0f5c..d55ab7f64e 100644
--- a/opensoc-streaming/OpenSOC-Indexing/pom.xml
+++ b/opensoc-streaming/OpenSOC-Indexing/pom.xml
@@ -15,11 +15,13 @@