diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv index fded05737b..766a376bcc 100644 --- a/dependencies_with_url.csv +++ b/dependencies_with_url.csv @@ -7,6 +7,7 @@ org.ow2.asm:asm:jar:5.0.3:compile,BSD,http://asm.ow2.org/ org.reflections:reflections:jar:0.9.10:compile,BSD,https://github.com/ronmamo/reflections org.javassist:javassist:jar:3.19.0-GA:compile,Apache v2,https://github.com/jboss-javassist/javassist org.javassist:javassist:jar:3.17.1-GA:compile,Apache v2,https://github.com/jboss-javassist/javassist +org.javassist:javassist:jar:3.20.0-GA:compile,Apache v2,https://github.com/jboss-javassist/javassist org.fusesource.jansi:jansi:jar:1.11:compile,Apache v2,https://github.com/fusesource/jansi de.javakaffee:kryo-serializers:jar:0.38:compile,Apache v2,https://github.com/magro/kryo-serializers com.tdunning:t-digest:jar:3.1:compile,Apache v2,https://github.com/tdunning/t-digest @@ -78,6 +79,7 @@ org.slf4j:slf4j-api:jar:1.7.10:compile,MIT,http://www.slf4j.org org.slf4j:slf4j-api:jar:1.7.5:compile,MIT,http://www.slf4j.org org.slf4j:slf4j-api:jar:1.7.6:compile,MIT,http://www.slf4j.org org.slf4j:slf4j-api:jar:1.7.7:compile,MIT,http://www.slf4j.org +org.slf4j:slf4j-api:jar:1.7.21:compile,MIT,http://www.slf4j.org org.slf4j:slf4j-log4j12:jar:1.6.1:compile,MIT,http://www.slf4j.org org.slf4j:slf4j-log4j12:jar:1.7.10:compile,MIT,http://www.slf4j.org org.slf4j:slf4j-log4j12:jar:1.7.10:runtime,MIT,http://www.slf4j.org @@ -85,6 +87,8 @@ org.slf4j:slf4j-log4j12:jar:1.7.21:compile,MIT,http://www.slf4j.org org.slf4j:slf4j-log4j12:jar:1.7.5:compile,MIT,http://www.slf4j.org org.slf4j:slf4j-log4j12:jar:1.7.7:compile,MIT,http://www.slf4j.org org.slf4j:slf4j-simple:jar:1.7.7:compile,MIT,http://www.slf4j.org +org.slf4j:jcl-over-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org +org.slf4j:jul-to-slf4j:jar:1.7.21:compile,MIT,http://www.slf4j.org aopalliance:aopalliance:jar:1.0:compile,Public Domain,http://aopalliance.sourceforge.net com.101tec:zkclient:jar:0.8:compile,The Apache Software License, Version 2.0,https://github.com/sgroschupf/zkclient com.github.stephenc.findbugs:findbugs-annotations:jar:1.3.9-1:compile,Apache License, Version 2.0,http://stephenc.github.com/findbugs-annotations @@ -98,15 +102,21 @@ com.codahale.metrics:metrics-graphite:jar:3.0.2:compile,MIT,https://github.com/c com.esotericsoftware.reflectasm:reflectasm:jar:shaded:1.07:compile,BSD,https://github.com/EsotericSoftware/reflectasm com.fasterxml.jackson.core:jackson-annotations:jar:2.2.3:compile,ASLv2,http://wiki.fasterxml.com/JacksonHome com.fasterxml.jackson.core:jackson-annotations:jar:2.7.4:compile,ASLv2,http://github.com/FasterXML/jackson +com.fasterxml.jackson.core:jackson-annotations:jar:2.8.3:compile,ASLv2,http://github.com/FasterXML/jackson com.fasterxml.jackson.core:jackson-core:jar:2.2.3:compile,ASLv2,http://wiki.fasterxml.com/JacksonHome com.fasterxml.jackson.core:jackson-core:jar:2.6.6:compile,ASLv2,https://github.com/FasterXML/jackson-core com.fasterxml.jackson.core:jackson-core:jar:2.7.4:compile,ASLv2,https://github.com/FasterXML/jackson-core +com.fasterxml.jackson.core:jackson-core:jar:2.8.3:compile,ASLv2,https://github.com/FasterXML/jackson-core com.fasterxml.jackson.core:jackson-databind:jar:2.2.3:compile,ASLv2,http://wiki.fasterxml.com/JacksonHome com.fasterxml.jackson.core:jackson-databind:jar:2.7.4:compile,ASLv2,http://github.com/FasterXML/jackson +com.fasterxml.jackson.core:jackson-databind:jar:2.8.3:compile,ASLv2,http://github.com/FasterXML/jackson com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:jar:2.6.6:compile,ASLv2,http://wiki.fasterxml.com/JacksonForCbor com.fasterxml.jackson.dataformat:jackson-dataformat-smile:jar:2.6.6:compile,ASLv2,http://wiki.fasterxml.com/JacksonForSmile com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:jar:2.6.6:compile,ASLv2,https://github.com/FasterXML/jackson +com.fasterxml.jackson.datatype:jackson-datatype-joda:jar:2.8.1:compile,ASLv2,https://github.com/FasterXML/jackson-datatype-joda +com.fasterxml:classmate:jar:1.3.1:compile,ASLv2,http://github.com/cowtowncoder/java-classmate com.google.code.gson:gson:jar:2.2.4:compile,The Apache Software License, Version 2.0,http://code.google.com/p/google-gson/ +com.google.code.gson:gson:jar:2.7:compile,The Apache Software License, Version 2.0,http://code.google.com/p/google-gson/ com.google.guava:guava:jar:11.0.2:compile,ASLv2, com.google.guava:guava:jar:12.0.1:compile,ASLv2, com.google.guava:guava:jar:12.0:compile,ASLv2, @@ -129,6 +139,7 @@ commons-beanutils:commons-beanutils-core:jar:1.8.0:compile,ASLv2,http://commons. commons-beanutils:commons-beanutils-core:jar:1.8.0:provided,ASLv2,http://commons.apache.org/beanutils/ commons-beanutils:commons-beanutils:jar:1.7.0:compile,ASLv2, commons-beanutils:commons-beanutils:jar:1.8.3:compile,ASLv2,http://commons.apache.org/beanutils/ +commons-beanutils:commons-beanutils:jar:1.9.2:compile,ASLv2,http://commons.apache.org/beanutils/ commons-cli:commons-cli:jar:1.2:compile,ASLv2,http://commons.apache.org/cli/ commons-cli:commons-cli:jar:1.3.1:compile,ASLv2,http://commons.apache.org/proper/commons-cli/ commons-codec:commons-codec:jar:1.10:compile,ASLv2,http://commons.apache.org/proper/commons-codec/ @@ -142,12 +153,14 @@ commons-configuration:commons-configuration:jar:1.6:compile,The Apache Software commons-daemon:commons-daemon:jar:1.0.13:compile,ASLv2,http://commons.apache.org/daemon/ commons-digester:commons-digester:jar:1.8.1:compile,ASLv2,http://commons.apache.org/digester/ commons-digester:commons-digester:jar:1.8:compile,The Apache Software License, Version 2.0,http://jakarta.apache.org/commons/digester/ +commons-digester:commons-digester:jar:2.1:compile,ASLv2,http://commons.apache.org/digester/ commons-el:commons-el:jar:1.0:provided,The Apache Software License, Version 2.0,http://jakarta.apache.org/commons/el/ commons-el:commons-el:jar:1.0:runtime,The Apache Software License, Version 2.0,http://jakarta.apache.org/commons/el/ commons-httpclient:commons-httpclient:jar:3.1:compile,Apache License,http://jakarta.apache.org/httpcomponents/httpclient-3.x/ commons-io:commons-io:jar:2.4:compile,ASLv2,http://commons.apache.org/io/ commons-io:commons-io:jar:2.5:compile,ASLv2,http://commons.apache.org/io/ commons-io:commons-io:jar:2.4:provided,ASLv2,http://commons.apache.org/io/ +commons-lang:commons-lang:jar:2.4:compile,ASLv2,http://commons.apache.org/lang/ commons-lang:commons-lang:jar:2.5:compile,ASLv2,http://commons.apache.org/lang/ commons-lang:commons-lang:jar:2.6:compile,ASLv2,http://commons.apache.org/lang/ commons-lang:commons-lang:jar:2.6:provided,ASLv2,http://commons.apache.org/lang/ @@ -204,14 +217,25 @@ org.springframework.integration:spring-integration-core:jar:3.0.0.RELEASE:compil org.springframework.integration:spring-integration-http:jar:3.0.0.RELEASE:compile,The Apache Software License, Version 2.0,http://www.springintegration.org/ org.springframework.retry:spring-retry:jar:1.0.3.RELEASE:compile,Apache 2.0,http://www.springsource.org org.springframework:spring-aop:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework +org.springframework:spring-aop:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework +org.springframework:spring-aspects:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework org.springframework:spring-beans:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework +org.springframework:spring-beans:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework org.springframework:spring-context:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework +org.springframework:spring-context:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework org.springframework:spring-core:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework org.springframework:spring-core:jar:4.1.4.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework +org.springframework:spring-core:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework org.springframework:spring-expression:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework +org.springframework:spring-expression:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework +org.springframework:spring-jdbc:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework +org.springframework:spring-orm:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework org.springframework:spring-tx:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework +org.springframework:spring-tx:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework org.springframework:spring-web:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework +org.springframework:spring-web:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework org.springframework:spring-webmvc:jar:3.2.6.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/SpringSource/spring-framework +org.springframework:spring-webmvc:jar:4.3.3.RELEASE:compile,The Apache Software License, Version 2.0,https://github.com/spring-projects/spring-framework org.tukaani:xz:jar:1.0:compile,Public Domain,http://tukaani.org/xz/java.html org.xerial.snappy:snappy-java:jar:1.0.4.1:compile,The Apache Software License, Version 2.0,http://code.google.com/p/snappy-java/ org.xerial.snappy:snappy-java:jar:1.1.1.7:compile,The Apache Software License, Version 2.0,https://github.com/xerial/snappy-java @@ -221,5 +245,42 @@ org.yaml:snakeyaml:jar:1.15:compile,Apache License Version 2.0,http://www.snakey ring-cors:ring-cors:jar:0.1.5:compile,Eclipse Public License 1.0,https://github.com/r0man/ring-cors xerces:xercesImpl:jar:2.9.1:compile,ASLv2,http://xerces.apache.org/xerces2-j xml-apis:xml-apis:jar:1.3.04:compile,ASLv2,http://xml.apache.org/commons/components/external/ +xml-apis:xml-apis:jar:1.4.01:compile,ASLv2,http://xml.apache.org/commons/components/external/ +dom4j:dom4j:jar:1.6.1:compile,dom4j,https://github.com/dom4j/dom4j +io.springfox:springfox-core:jar:2.5.0:compile,ASLv2,https://github.com/springfox/springfox +io.springfox:springfox-schema:jar:2.5.0:compile,ASLv2,https://github.com/springfox/springfox +io.springfox:springfox-spi:jar:2.5.0:compile,ASLv2,https://github.com/springfox/springfox +io.springfox:springfox-spring-web:jar:2.5.0:compile,ASLv2,https://github.com/springfox/springfox +io.springfox:springfox-swagger-common:jar:2.5.0:compile,ASLv2,https://github.com/springfox/springfox +io.springfox:springfox-swagger-ui:jar:2.5.0:compile,ASLv2,https://github.com/springfox/springfox +io.springfox:springfox-swagger2:jar:2.5.0:compile,ASLv2,https://github.com/springfox/springfox +io.swagger:swagger-annotations:jar:1.5.9:compile,ASLv2,https://github.com/swagger-api/swagger-core +io.swagger:swagger-models:jar:1.5.9:compile,ASLv2,https://github.com/swagger-api/swagger-core +javax.transaction:javax.transaction-api:jar:1.2:compile,CDDL-1.0,https://java.net/projects/jta-spec/ +javax.validation:validation-api:jar:1.1.0.Final:compile,ASLv2,http://beanvalidation.org +joda-time:joda-time:jar:2.9.4:compile,ASLv2,https://github.com/JodaOrg/joda-time +org.aspectj:aspectjweaver:jar:1.8.9:compile,EPL 1.0,https://eclipse.org/aspectj +org.jboss.logging:jboss-logging:jar:3.3.0.Final:compile,ASLv2,https://github.com/jboss-logging +org.jboss:jandex:jar:2.0.0.Final:compile,ASLv2,https://github.com/wildfly/jandex +org.mapstruct:mapstruct:jar:1.0.0.Final:compile,ASLv2,https://github.com/mapstruct/mapstruct +org.springframework.boot:spring-boot-autoconfigure:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot +org.springframework.boot:spring-boot-starter-aop:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot +org.springframework.boot:spring-boot-starter-data-jpa:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot +org.springframework.boot:spring-boot-starter-jdbc:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot +org.springframework.boot:spring-boot-starter-logging:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot +org.springframework.boot:spring-boot-starter-security:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot +org.springframework.boot:spring-boot-starter-tomcat:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot +org.springframework.boot:spring-boot-starter-web:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot +org.springframework.boot:spring-boot-starter:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot +org.springframework.boot:spring-boot:jar:1.4.1.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-boot +org.springframework.data:spring-data-commons:jar:1.12.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-data-commons +org.springframework.data:spring-data-jpa:jar:1.10.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-data-jpa +org.springframework.plugin:spring-plugin-core:jar:1.2.0.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-plugin +org.springframework.plugin:spring-plugin-metadata:jar:1.2.0.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-plugin +org.springframework.security:spring-security-config:jar:4.1.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security +org.springframework.security:spring-security-core:jar:4.1.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security +org.springframework.security:spring-security-web:jar:4.1.3.RELEASE:compile,ASLv2,https://github.com/spring-projects/spring-security +antlr:antlr:jar:2.7.7:compile,BSD 3-Clause License,http://www.antlr2.org +com.h2database:h2:jar:1.4.192:compile,EPL 1.0,http://www.h2database.com/html/license.html de.jollyday:jollyday:jar:0.5.2:compile,ASLv2,http://jollyday.sourceforge.net/license.html org.threeten:threeten-extra:jar:1.0:compile,BSD,http://www.threeten.org/threeten-extra/license.html diff --git a/metron-interface/metron-rest-client/pom.xml b/metron-interface/metron-rest-client/pom.xml new file mode 100644 index 0000000000..0a49986784 --- /dev/null +++ b/metron-interface/metron-rest-client/pom.xml @@ -0,0 +1,40 @@ + + + + + 4.0.0 + + org.apache.metron + metron-interface + 0.3.1 + + metron-rest-client + + + + + org.apache.metron + metron-common + ${project.parent.version} + + + com.fasterxml.jackson.core + jackson-databind + + + + + + diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java new file mode 100644 index 0000000000..279532066a --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/GrokValidation.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model; + +import java.util.HashMap; +import java.util.Map; + +public class GrokValidation { + + private String patternLabel; + private String statement; + private String sampleData; + private Map results; + + public String getPatternLabel() { + return patternLabel; + } + + public void setPatternLabel(String patternLabel) { + this.patternLabel = patternLabel; + } + + public String getStatement() { + return statement; + } + + public void setStatement(String statement) { + this.statement = statement; + } + + public String getSampleData() { + return sampleData; + } + + public void setSampleData(String sampleData) { + this.sampleData = sampleData; + } + + public Map getResults() { + if (results == null) { + return new HashMap<>(); + } + return results; + } + + public void setResults(Map results) { + this.results = results; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + GrokValidation that = (GrokValidation) o; + + if (patternLabel != null ? !patternLabel.equals(that.patternLabel) : that.patternLabel != null) return false; + if (statement != null ? !statement.equals(that.statement) : that.statement != null) return false; + if (sampleData != null ? !sampleData.equals(that.sampleData) : that.sampleData != null) return false; + return results != null ? results.equals(that.results) : that.results == null; + } + + @Override + public int hashCode() { + int result = patternLabel != null ? patternLabel.hashCode() : 0; + result = 31 * result + (statement != null ? statement.hashCode() : 0); + result = 31 * result + (sampleData != null ? sampleData.hashCode() : 0); + result = 31 * result + (results != null ? results.hashCode() : 0); + return result; + } +} diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/KafkaTopic.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/KafkaTopic.java new file mode 100644 index 0000000000..55dd2b2d42 --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/KafkaTopic.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model; + +import java.util.Properties; + +public class KafkaTopic { + + private String name; + private int numPartitions; + private int replicationFactor; + private Properties properties = new Properties(); + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public int getNumPartitions() { + return numPartitions; + } + + public void setNumPartitions(int numPartitions) { + this.numPartitions = numPartitions; + } + + public int getReplicationFactor() { + return replicationFactor; + } + + public void setReplicationFactor(int replicationFactor) { + this.replicationFactor = replicationFactor; + } + + public Properties getProperties() { + return properties; + } + + public void setProperties(Properties properties) { + this.properties = properties; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + KafkaTopic that = (KafkaTopic) o; + + if (numPartitions != that.numPartitions) return false; + if (replicationFactor != that.replicationFactor) return false; + if (name != null ? !name.equals(that.name) : that.name != null) return false; + return properties != null ? properties.equals(that.properties) : that.properties == null; + } + + @Override + public int hashCode() { + int result = name != null ? name.hashCode() : 0; + result = 31 * result + numPartitions; + result = 31 * result + replicationFactor; + result = 31 * result + (properties != null ? properties.hashCode() : 0); + return result; + } +} diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java new file mode 100644 index 0000000000..50eb88de99 --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/ParseMessageRequest.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model; + +import org.apache.metron.common.configuration.SensorParserConfig; + +public class ParseMessageRequest { + + private SensorParserConfig sensorParserConfig; + private String grokStatement; + private String sampleData; + + public SensorParserConfig getSensorParserConfig() { + return sensorParserConfig; + } + + public void setSensorParserConfig(SensorParserConfig sensorParserConfig) { + this.sensorParserConfig = sensorParserConfig; + } + + public String getGrokStatement() { + return grokStatement; + } + + public void setGrokStatement(String grokStatement) { + this.grokStatement = grokStatement; + } + + public String getSampleData() { + return sampleData; + } + + public void setSampleData(String sampleData) { + this.sampleData = sampleData; + } +} diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/RestError.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/RestError.java new file mode 100644 index 0000000000..5f9de622cc --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/RestError.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model; + +public class RestError { + + private int responseCode; + private String message; + private String fullMessage; + + public RestError(int responseCode, String message, String fullMessage) { + this.responseCode = responseCode; + this.message = message; + this.fullMessage = fullMessage; + } + + public int getResponseCode() { + return responseCode; + } + + public String getMessage() { + return message; + } + + public String getFullMessage() { + return fullMessage; + } +} diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java new file mode 100644 index 0000000000..cb10cfe3b8 --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/SensorParserContext.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model; + +import org.apache.metron.common.configuration.SensorParserConfig; + +import java.util.HashMap; +import java.util.Map; + +public class SensorParserContext { + + private Map sampleData; + private SensorParserConfig sensorParserConfig; + + public Map getSampleData() { + if (sampleData == null) { + return new HashMap<>(); + } + return sampleData; + } + + public void setSampleData(Map sampleData) { + this.sampleData = sampleData; + } + + public SensorParserConfig getSensorParserConfig() { + if (sensorParserConfig == null) { + return new SensorParserConfig(); + } + return sensorParserConfig; + } + + public void setSensorParserConfig(SensorParserConfig sensorParserConfig) { + this.sensorParserConfig = sensorParserConfig; + } +} diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/StellarFunctionDescription.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/StellarFunctionDescription.java new file mode 100644 index 0000000000..a1b9f03a21 --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/StellarFunctionDescription.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model; + +public class StellarFunctionDescription { + + private String name; + private String description; + private String[] params; + private String returns; + + public StellarFunctionDescription(String name, String description, String[] params, String returns) { + this.name = name; + this.description = description; + this.params = params; + this.returns = returns; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String[] getParams() { + if (params == null) { + return new String[0]; + } + return params; + } + + public String getReturns() { + return returns; + } +} diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java new file mode 100644 index 0000000000..0ad2e9f487 --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponse.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model; + +public class TopologyResponse { + + private TopologyResponseCode status; + private String message; + + public TopologyResponseCode getStatus() { + return status; + } + + public String getMessage() { + return message; + } + + public void setSuccessMessage(String message) { + this.status = TopologyResponseCode.SUCCESS; + this.message = message; + } + + public void setErrorMessage(String message) { + this.status = TopologyResponseCode.ERROR; + this.message = message; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + TopologyResponse that = (TopologyResponse) o; + + if (status != null ? !status.equals(that.status) : that.status != null) return false; + return message != null ? message.equals(that.message) : that.message == null; + } + + @Override + public int hashCode() { + int result = status != null ? status.hashCode() : 0; + result = 31 * result + (message != null ? message.hashCode() : 0); + return result; + } +} diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponseCode.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponseCode.java new file mode 100644 index 0000000000..a85bd89ccb --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyResponseCode.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model; + +public enum TopologyResponseCode { + + SUCCESS,ERROR +} diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java new file mode 100644 index 0000000000..6fc1c019fe --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatus.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model; + +import java.util.List; +import java.util.Map; + +public class TopologyStatus { + + private String id; + private String name; + private TopologyStatusCode status; + private Map[] topologyStats; + private Double latency = 0.0; + private Double throughput = 0.0; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public TopologyStatusCode getStatus() { + return status; + } + + public void setStatus(TopologyStatusCode status) { + this.status = status; + } + + public double getLatency() { + return latency; + } + + public double getThroughput() { + return throughput; + } + + public void setTopologyStats(List> topologyStats) { + for(Map topologyStatsItem: topologyStats) { + if ("600".equals(topologyStatsItem.get("window"))) { + latency = Double.parseDouble((String) topologyStatsItem.get("completeLatency")); + int acked = 0; + if (topologyStatsItem.get("acked") != null) { + acked = (int) topologyStatsItem.get("acked"); + } + throughput = acked / 600.00; + } + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + TopologyStatus that = (TopologyStatus) o; + + if (id != null ? !id.equals(that.id) : that.id != null) return false; + if (name != null ? !name.equals(that.name) : that.name != null) return false; + if (status != null ? !status.equals(that.status) : that.status != null) return false; + if (!latency.equals(that.latency)) return false; + return throughput.equals(that.throughput); + } + + @Override + public int hashCode() { + int result = id != null ? id.hashCode() : 0; + result = 31 * result + (name != null ? name.hashCode() : 0); + result = 31 * result + (status != null ? status.hashCode() : 0); + result = 31 * result + (latency != null ? latency.hashCode() : 0); + result = 31 * result + (throughput != null ? throughput.hashCode() : 0); + return result; + } +} diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatusCode.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatusCode.java new file mode 100644 index 0000000000..461a4ab448 --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologyStatusCode.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model; + +public enum TopologyStatusCode { + + STARTED, + STOPPED, + ACTIVE, + INACTIVE, + KILLED, + GLOBAL_CONFIG_MISSING, + SENSOR_PARSER_CONFIG_MISSING, + START_ERROR, + STOP_ERROR, + TOPOLOGY_NOT_FOUND; +} diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java new file mode 100644 index 0000000000..8621daf19c --- /dev/null +++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/TopologySummary.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.model; + +import java.util.Arrays; + +public class TopologySummary { + + private TopologyStatus[] topologies; + + public TopologyStatus[] getTopologies() { + if (topologies == null) { + return new TopologyStatus[0]; + } + return topologies; + } + + public void setTopologies(TopologyStatus[] topologies) { + this.topologies = topologies; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + TopologySummary that = (TopologySummary) o; + + return topologies != null ? Arrays.equals(topologies, that.topologies) : that.topologies != null; + } + + @Override + public int hashCode() { + return topologies != null ? Arrays.hashCode(topologies) : 0; + } +} diff --git a/metron-interface/metron-rest/.gitignore b/metron-interface/metron-rest/.gitignore new file mode 100644 index 0000000000..26b3af1e1a --- /dev/null +++ b/metron-interface/metron-rest/.gitignore @@ -0,0 +1 @@ +metrondb.*.db diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md new file mode 100644 index 0000000000..cf41bf33e4 --- /dev/null +++ b/metron-interface/metron-rest/README.md @@ -0,0 +1,523 @@ +# Metron REST + +This module provides a RESTful API for interacting with Metron. + +## Prerequisites + +* A running Metron cluster +* Java 8 installed +* Storm CLI and Metron topology scripts (start_parser_topology.sh, start_enrichment_topology.sh, start_elasticsearch_topology.sh) installed + +## Installation +1. Package the application with Maven: +``` +mvn clean package +``` + +1. Untar the archive in the target directory. The directory structure will look like: +``` +bin + start_metron_rest.sh +lib + metron-rest-$METRON_VERSION.jar +``` + +1. Create an `application.yml` file with the contents of [application-docker.yml](src/main/resources/application-docker.yml). Substitute the appropriate Metron service urls (Kafka, Zookeeper, Storm, etc) in properties containing `${docker.host.address}` and update the `spring.datasource.*` properties as needed (see the [Security](#security) section for more details). + +1. Start the application with this command: +``` +./bin/start_metron_rest.sh /path/to/application.yml +``` + +## Usage + +The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port. + +## Security + +The metron-rest module uses [Spring Security](http://projects.spring.io/spring-security/) for authentication and stores user credentials in a relational database. The H2 database is configured by default and is intended only for development purposes. The "dev" profile can be used to automatically load test users: +``` +./bin/start_metron_rest.sh /path/to/application.yml --spring.profiles.active=dev +``` + +For [production use](http://docs.spring.io/spring-boot/docs/1.4.1.RELEASE/reference/htmlsingle/#boot-features-connect-to-production-database), a relational database should be configured. For example, configuring MySQL would be done as follows: + +1. Create a MySQL user for the Metron REST application (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html). + +1. Connect to MySQL and create a Metron REST database: +``` +CREATE DATABASE IF NOT EXISTS metronrest +``` + +1. Add users: +``` +use metronrest; +insert into users (username, password, enabled) values ('your_username','your_password',1); +insert into authorities (username, authority) values ('your_username', 'ROLE_USER'); +``` + +1. Replace the H2 connection information in the application.yml file with MySQL connection information: +``` +spring: + datasource: + driverClassName: com.mysql.jdbc.Driver + url: jdbc:mysql://mysql_host:3306/metronrest + username: metron_rest_user + password: metron_rest_password + platform: mysql +``` + +1. Add a dependency for the MySQL JDBC connector in the metron-rest pom.xml: +``` + + mysql + mysql-connector-java + ${mysql.client.version} + +``` + +1. Follow the steps in the [Installation](#installation) section + +## API + +Request and Response objects are JSON formatted. The JSON schemas are available in the Swagger UI. + +| | +| ---------- | +| [ `GET /api/v1/global/config`](#get-apiv1globalconfig)| +| [ `DELETE /api/v1/global/config`](#delete-apiv1globalconfig)| +| [ `POST /api/v1/global/config`](#post-apiv1globalconfig)| +| [ `GET /api/v1/grok/list`](#get-apiv1groklist)| +| [ `POST /api/v1/grok/validate`](#post-apiv1grokvalidate)| +| [ `POST /api/v1/hdfs`](#post-apiv1hdfs)| +| [ `GET /api/v1/hdfs`](#get-apiv1hdfs)| +| [ `DELETE /api/v1/hdfs`](#delete-apiv1hdfs)| +| [ `GET /api/v1/hdfs/list`](#get-apiv1hdfslist)| +| [ `GET /api/v1/kafka/topic`](#get-apiv1kafkatopic)| +| [ `POST /api/v1/kafka/topic`](#post-apiv1kafkatopic)| +| [ `GET /api/v1/kafka/topic/{name}`](#get-apiv1kafkatopic{name})| +| [ `DELETE /api/v1/kafka/topic/{name}`](#delete-apiv1kafkatopic{name})| +| [ `GET /api/v1/kafka/topic/{name}/sample`](#get-apiv1kafkatopic{name}sample)| +| [ `GET /api/v1/sensor/enrichment/config`](#get-apiv1sensorenrichmentconfig)| +| [ `GET /api/v1/sensor/enrichment/config/list/available`](#get-apiv1sensorenrichmentconfiglistavailable)| +| [ `DELETE /api/v1/sensor/enrichment/config/{name}`](#delete-apiv1sensorenrichmentconfig{name})| +| [ `POST /api/v1/sensor/enrichment/config/{name}`](#post-apiv1sensorenrichmentconfig{name})| +| [ `GET /api/v1/sensor/enrichment/config/{name}`](#get-apiv1sensorenrichmentconfig{name})| +| [ `GET /api/v1/sensor/indexing/config`](#get-apiv1sensorindexingconfig)| +| [ `DELETE /api/v1/sensor/indexing/config/{name}`](#delete-apiv1sensorindexingconfig{name})| +| [ `POST /api/v1/sensor/indexing/config/{name}`](#post-apiv1sensorindexingconfig{name})| +| [ `GET /api/v1/sensor/indexing/config/{name}`](#get-apiv1sensorindexingconfig{name})| +| [ `POST /api/v1/sensor/parser/config`](#post-apiv1sensorparserconfig)| +| [ `GET /api/v1/sensor/parser/config`](#get-apiv1sensorparserconfig)| +| [ `GET /api/v1/sensor/parser/config/list/available`](#get-apiv1sensorparserconfiglistavailable)| +| [ `POST /api/v1/sensor/parser/config/parseMessage`](#post-apiv1sensorparserconfigparsemessage)| +| [ `GET /api/v1/sensor/parser/config/reload/available`](#get-apiv1sensorparserconfigreloadavailable)| +| [ `DELETE /api/v1/sensor/parser/config/{name}`](#delete-apiv1sensorparserconfig{name})| +| [ `GET /api/v1/sensor/parser/config/{name}`](#get-apiv1sensorparserconfig{name})| +| [ `POST /api/v1/stellar/apply/transformations`](#post-apiv1stellarapplytransformations)| +| [ `GET /api/v1/stellar/list`](#get-apiv1stellarlist)| +| [ `GET /api/v1/stellar/list/functions`](#get-apiv1stellarlistfunctions)| +| [ `GET /api/v1/stellar/list/simple/functions`](#get-apiv1stellarlistsimplefunctions)| +| [ `POST /api/v1/stellar/validate/rules`](#post-apiv1stellarvalidaterules)| +| [ `GET /api/v1/storm`](#get-apiv1storm)| +| [ `GET /api/v1/storm/client/status`](#get-apiv1stormclientstatus)| +| [ `GET /api/v1/storm/enrichment`](#get-apiv1stormenrichment)| +| [ `GET /api/v1/storm/enrichment/activate`](#get-apiv1stormenrichmentactivate)| +| [ `GET /api/v1/storm/enrichment/deactivate`](#get-apiv1stormenrichmentdeactivate)| +| [ `GET /api/v1/storm/enrichment/start`](#get-apiv1stormenrichmentstart)| +| [ `GET /api/v1/storm/enrichment/stop`](#get-apiv1stormenrichmentstop)| +| [ `GET /api/v1/storm/indexing`](#get-apiv1stormindexing)| +| [ `GET /api/v1/storm/indexing/activate`](#get-apiv1stormindexingactivate)| +| [ `GET /api/v1/storm/indexing/deactivate`](#get-apiv1stormindexingdeactivate)| +| [ `GET /api/v1/storm/indexing/start`](#get-apiv1stormindexingstart)| +| [ `GET /api/v1/storm/indexing/stop`](#get-apiv1stormindexingstop)| +| [ `GET /api/v1/storm/parser/activate/{name}`](#get-apiv1stormparseractivate{name})| +| [ `GET /api/v1/storm/parser/deactivate/{name}`](#get-apiv1stormparserdeactivate{name})| +| [ `GET /api/v1/storm/parser/start/{name}`](#get-apiv1stormparserstart{name})| +| [ `GET /api/v1/storm/parser/stop/{name}`](#get-apiv1stormparserstop{name})| +| [ `GET /api/v1/storm/{name}`](#get-apiv1storm{name})| +| [ `GET /api/v1/user`](#get-apiv1user)| + +### `GET /api/v1/global/config` + * Description: Retrieves the current Global Config from Zookeeper + * Returns: + * 200 - Returns current Global Config JSON in Zookeeper + * 404 - Global Config JSON was not found in Zookeeper + +### `DELETE /api/v1/global/config` + * Description: Deletes the current Global Config from Zookeeper + * Returns: + * 200 - Global Config JSON was deleted + * 404 - Global Config JSON was not found in Zookeeper + +### `POST /api/v1/global/config` + * Description: Creates or updates the Global Config in Zookeeper + * Input: + * globalConfig - The Global Config JSON to be saved + * Returns: + * 200 - Global Config updated. Returns saved Global Config JSON + * 201 - Global Config created. Returns saved Global Config JSON + +### `GET /api/v1/grok/list` + * Description: Lists the common Grok statements available in Metron + * Returns: + * 200 - JSON object containing pattern label/Grok statements key value pairs + +### `POST /api/v1/grok/validate` + * Description: Applies a Grok statement to a sample message + * Input: + * grokValidation - Object containing Grok statement and sample message + * Returns: + * 200 - JSON results + +### `POST /api/v1/hdfs` + * Description: Writes contents to an HDFS file. Warning: this will overwite the contents of a file if it already exists. + * Input: + * path - Path to HDFS file + * contents - File contents + * Returns: + * 200 - Contents were written + +### `GET /api/v1/hdfs` + * Description: Reads a file from HDFS and returns the contents + * Input: + * path - Path to HDFS file + * Returns: + * 200 - Returns file contents + +### `DELETE /api/v1/hdfs` + * Description: Deletes a file from HDFS + * Input: + * path - Path to HDFS file + * recursive - Delete files recursively + * Returns: + * 200 - File was deleted + * 404 - File was not found in HDFS + +### `GET /api/v1/hdfs/list` + * Description: Reads a file from HDFS and returns the contents + * Input: + * path - Path to HDFS directory + * Returns: + * 200 - Returns file contents + +### `GET /api/v1/kafka/topic` + * Description: Retrieves all Kafka topics + * Returns: + * 200 - Returns a list of all Kafka topics + +### `POST /api/v1/kafka/topic` + * Description: Creates a new Kafka topic + * Input: + * topic - Kafka topic + * Returns: + * 200 - Returns saved Kafka topic + +### `GET /api/v1/kafka/topic/{name}` + * Description: Retrieves a Kafka topic + * Input: + * name - Kafka topic name + * Returns: + * 200 - Returns Kafka topic + * 404 - Kafka topic is missing + +### `DELETE /api/v1/kafka/topic/{name}` + * Description: Delets a Kafka topic + * Input: + * name - Kafka topic name + * Returns: + * 200 - Kafka topic was deleted + * 404 - Kafka topic is missing + +### `GET /api/v1/kafka/topic/{name}/sample` + * Description: Retrieves a sample message from a Kafka topic using the most recent offset + * Input: + * name - Kafka topic name + * Returns: + * 200 - Returns sample message + * 404 - Either Kafka topic is missing or contains no messages + +### `GET /api/v1/sensor/enrichment/config` + * Description: Retrieves all SensorEnrichmentConfigs from Zookeeper + * Returns: + * 200 - Returns all SensorEnrichmentConfigs + +### `GET /api/v1/sensor/enrichment/config/list/available` + * Description: Lists the available enrichments + * Returns: + * 200 - Returns a list of available enrichments + +### `DELETE /api/v1/sensor/enrichment/config/{name}` + * Description: Deletes a SensorEnrichmentConfig from Zookeeper + * Input: + * name - SensorEnrichmentConfig name + * Returns: + * 200 - SensorEnrichmentConfig was deleted + * 404 - SensorEnrichmentConfig is missing + +### `POST /api/v1/sensor/enrichment/config/{name}` + * Description: Updates or creates a SensorEnrichmentConfig in Zookeeper + * Input: + * sensorEnrichmentConfig - SensorEnrichmentConfig + * name - SensorEnrichmentConfig name + * Returns: + * 200 - SensorEnrichmentConfig updated. Returns saved SensorEnrichmentConfig + * 201 - SensorEnrichmentConfig created. Returns saved SensorEnrichmentConfig + +### `GET /api/v1/sensor/enrichment/config/{name}` + * Description: Retrieves a SensorEnrichmentConfig from Zookeeper + * Input: + * name - SensorEnrichmentConfig name + * Returns: + * 200 - Returns SensorEnrichmentConfig + * 404 - SensorEnrichmentConfig is missing + +### `GET /api/v1/sensor/indexing/config` + * Description: Retrieves all SensorIndexingConfigs from Zookeeper + * Returns: + * 200 - Returns all SensorIndexingConfigs + +### `DELETE /api/v1/sensor/indexing/config/{name}` + * Description: Deletes a SensorIndexingConfig from Zookeeper + * Input: + * name - SensorIndexingConfig name + * Returns: + * 200 - SensorIndexingConfig was deleted + * 404 - SensorIndexingConfig is missing + +### `POST /api/v1/sensor/indexing/config/{name}` + * Description: Updates or creates a SensorIndexingConfig in Zookeeper + * Input: + * sensorIndexingConfig - SensorIndexingConfig + * name - SensorIndexingConfig name + * Returns: + * 200 - SensorIndexingConfig updated. Returns saved SensorIndexingConfig + * 201 - SensorIndexingConfig created. Returns saved SensorIndexingConfig + +### `GET /api/v1/sensor/indexing/config/{name}` + * Description: Retrieves a SensorIndexingConfig from Zookeeper + * Input: + * name - SensorIndexingConfig name + * Returns: + * 200 - Returns SensorIndexingConfig + * 404 - SensorIndexingConfig is missing + +### `POST /api/v1/sensor/parser/config` + * Description: Updates or creates a SensorParserConfig in Zookeeper + * Input: + * sensorParserConfig - SensorParserConfig + * Returns: + * 200 - SensorParserConfig updated. Returns saved SensorParserConfig + * 201 - SensorParserConfig created. Returns saved SensorParserConfig + +### `GET /api/v1/sensor/parser/config` + * Description: Retrieves all SensorParserConfigs from Zookeeper + * Returns: + * 200 - Returns all SensorParserConfigs + +### `GET /api/v1/sensor/parser/config/list/available` + * Description: Lists the available parser classes that can be found on the classpath + * Returns: + * 200 - Returns a list of available parser classes + +### `POST /api/v1/sensor/parser/config/parseMessage` + * Description: Parses a sample message given a SensorParserConfig + * Input: + * parseMessageRequest - Object containing a sample message and SensorParserConfig + * Returns: + * 200 - Returns parsed message + +### `GET /api/v1/sensor/parser/config/reload/available` + * Description: Scans the classpath for available parser classes and reloads the cached parser class list + * Returns: + * 200 - Returns a list of available parser classes + +### `DELETE /api/v1/sensor/parser/config/{name}` + * Description: Deletes a SensorParserConfig from Zookeeper + * Input: + * name - SensorParserConfig name + * Returns: + * 200 - SensorParserConfig was deleted + * 404 - SensorParserConfig is missing + +### `GET /api/v1/sensor/parser/config/{name}` + * Description: Retrieves a SensorParserConfig from Zookeeper + * Input: + * name - SensorParserConfig name + * Returns: + * 200 - Returns SensorParserConfig + * 404 - SensorParserConfig is missing + +### `POST /api/v1/stellar/apply/transformations` + * Description: Executes transformations against a sample message + * Input: + * transformationValidation - Object containing SensorParserConfig and sample message + * Returns: + * 200 - Returns transformation results + +### `GET /api/v1/stellar/list` + * Description: Retrieves field transformations + * Returns: + * 200 - Returns a list field transformations + +### `GET /api/v1/stellar/list/functions` + * Description: Lists the Stellar functions that can be found on the classpath + * Returns: + * 200 - Returns a list of Stellar functions + +### `GET /api/v1/stellar/list/simple/functions` + * Description: Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath + * Returns: + * 200 - Returns a list of simple Stellar functions + +### `POST /api/v1/stellar/validate/rules` + * Description: Tests Stellar statements to ensure they are well-formed + * Input: + * statements - List of statements to validate + * Returns: + * 200 - Returns validation results + +### `GET /api/v1/storm` + * Description: Retrieves the status of all Storm topologies + * Returns: + * 200 - Returns a list of topologies with status information + +### `GET /api/v1/storm/client/status` + * Description: Retrieves information about the Storm command line client + * Returns: + * 200 - Returns storm command line client information + +### `GET /api/v1/storm/enrichment` + * Description: Retrieves the status of the Storm enrichment topology + * Returns: + * 200 - Returns topology status information + * 404 - Topology is missing + +### `GET /api/v1/storm/enrichment/activate` + * Description: Activates a Storm enrichment topology + * Returns: + * 200 - Returns activate response message + +### `GET /api/v1/storm/enrichment/deactivate` + * Description: Deactivates a Storm enrichment topology + * Returns: + * 200 - Returns deactivate response message + +### `GET /api/v1/storm/enrichment/start` + * Description: Starts a Storm enrichment topology + * Returns: + * 200 - Returns start response message + +### `GET /api/v1/storm/enrichment/stop` + * Description: Stops a Storm enrichment topology + * Input: + * stopNow - Stop the topology immediately + * Returns: + * 200 - Returns stop response message + +### `GET /api/v1/storm/indexing` + * Description: Retrieves the status of the Storm indexing topology + * Returns: + * 200 - Returns topology status information + * 404 - Topology is missing + +### `GET /api/v1/storm/indexing/activate` + * Description: Activates a Storm indexing topology + * Returns: + * 200 - Returns activate response message + +### `GET /api/v1/storm/indexing/deactivate` + * Description: Deactivates a Storm indexing topology + * Returns: + * 200 - Returns deactivate response message + +### `GET /api/v1/storm/indexing/start` + * Description: Starts a Storm indexing topology + * Returns: + * 200 - Returns start response message + +### `GET /api/v1/storm/indexing/stop` + * Description: Stops a Storm enrichment topology + * Input: + * stopNow - Stop the topology immediately + * Returns: + * 200 - Returns stop response message + +### `GET /api/v1/storm/parser/activate/{name}` + * Description: Activates a Storm parser topology + * Input: + * name - Parser name + * Returns: + * 200 - Returns activate response message + +### `GET /api/v1/storm/parser/deactivate/{name}` + * Description: Deactivates a Storm parser topology + * Input: + * name - Parser name + * Returns: + * 200 - Returns deactivate response message + +### `GET /api/v1/storm/parser/start/{name}` + * Description: Starts a Storm parser topology + * Input: + * name - Parser name + * Returns: + * 200 - Returns start response message + +### `GET /api/v1/storm/parser/stop/{name}` + * Description: Stops a Storm parser topology + * Input: + * name - Parser name + * stopNow - Stop the topology immediately + * Returns: + * 200 - Returns stop response message + +### `GET /api/v1/storm/{name}` + * Description: Retrieves the status of a Storm topology + * Input: + * name - Topology name + * Returns: + * 200 - Returns topology status information + * 404 - Topology is missing + +### `GET /api/v1/user` + * Description: Retrieves the current user + * Returns: + * 200 - Current user + +## Testing + +Profiles are includes for both the metron-docker and Quick Dev environments. + +### metron-docker + +Start the [metron-docker](../../metron-docker) environment. Build the metron-rest module and start it with the Spring Boot Maven plugin: +``` +mvn clean package +mvn spring-boot:run -Drun.profiles=docker,dev +``` +The metron-rest application will be available at http://localhost:8080/swagger-ui.html#/. + +### Quick Dev + +Start the [Quick Dev](../../metron-deployment/vagrant/quick-dev-platform) environment. Build the metron-rest module and start it with the Spring Boot Maven plugin: +``` +mvn clean package +mvn spring-boot:run -Drun.profiles=vagrant,dev +``` +The metron-rest application will be available at http://localhost:8080/swagger-ui.html#/. + +To run the application locally on the Quick Dev host, package the application and scp the archive to node1: +``` +mvn clean package +scp ./target/metron-rest-$METRON_VERSION-archive.tar.gz root@node1:~/ +``` +Login to node1 and unarchive the metron-rest application. Start the application on a different port to avoid conflicting with Ambari: +``` +java -jar ./lib/metron-rest-$METRON_VERSION.jar --spring.profiles.active=vagrant,dev --server.port=8082 +``` +The metron-rest application will be available at http://node1:8082/swagger-ui.html#/. + +## License + +This project depends on the Java Transaction API. See https://java.net/projects/jta-spec/ for more details. diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml new file mode 100644 index 0000000000..849455e67d --- /dev/null +++ b/metron-interface/metron-rest/pom.xml @@ -0,0 +1,327 @@ + + + + + 4.0.0 + + org.apache.metron + metron-interface + 0.3.1 + + metron-rest + + UTF-8 + UTF-8 + 1.8 + 4.5 + 2.7.1 + 1.6.4 + 1.4.1.RELEASE + 2.5.0 + 5.1.40 + 1.0-alpha-3 + + + + org.springframework.boot + spring-boot-starter-web + + + ch.qos.logback + logback-classic + + + org.slf4j + log4j-over-slf4j + + + org.hibernate + hibernate-validator + + + + + org.springframework.boot + spring-boot-starter-security + + + org.springframework.boot + spring-boot-starter-jdbc + + + org.apache.curator + curator-recipes + ${curator.version} + + + com.google.guava + guava + + + + + com.googlecode.json-simple + json-simple + ${global_json_simple_version} + + + org.antlr + antlr4-runtime + ${antlr.version} + + + com.fasterxml.jackson.core + jackson-databind + 2.8.3 + + + com.fasterxml.jackson.datatype + jackson-datatype-joda + 2.8.1 + + + org.apache.metron + metron-rest-client + ${project.parent.version} + + + org.apache.metron + metron-common + ${project.parent.version} + + + com.fasterxml.jackson.core + jackson-databind + + + org.reflections + reflections + + + + + org.apache.metron + metron-parsers + ${project.parent.version} + + + com.fasterxml.jackson.core + jackson-databind + + + org.apache.metron + metron-profiler-client + + + org.apache.metron + metron-statistics + + + org.apache.metron + metron-writer + + + com.google.guava + guava + + + + + io.springfox + springfox-swagger2 + ${swagger.version} + + + io.springfox + springfox-swagger-ui + ${swagger.version} + + + org.apache.kafka + kafka_2.10 + ${global_kafka_version} + + + io.thekraken + grok + 0.1.0 + + + org.springframework.boot + spring-boot-starter-test + test + + + com.h2database + h2 + + + org.springframework.security + spring-security-test + test + + + org.powermock + powermock-module-junit4 + ${powermock.version} + test + + + org.powermock + powermock-api-mockito + ${powermock.version} + test + + + org.apache.metron + metron-integration-test + ${project.parent.version} + test + + + org.apache.logging.log4j + log4j-slf4j-impl + + + org.slf4j + log4j-over-slf4j + + + javax.servlet + servlet-api + + + + + org.apache.kafka + kafka-clients + ${global_kafka_version} + test + test + + + log4j + log4j + + + + + org.apache.velocity + velocity + 1.7 + test + + + + + + + + org.springframework.boot + spring-boot-dependencies + ${spring.boot.version} + pom + import + + + org.apache.logging.log4j + log4j-slf4j-impl + + + ch.qos.logback + logback-classic + + + + + + + + + + org.codehaus.mojo + emma-maven-plugin + ${emma.version} + + + org.apache.maven.plugins + maven-pmd-plugin + + ${global_java_version} + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + ${spring.boot.version} + + + + repackage + + + + + ZIP + + + mysql + mysql-connector-java + + + org.hibernate.common + hibernate-commons-annotations + + + org.hibernate + hibernate-core + + + org.hibernate + hibernate-envers + + + org.hibernate + hibernate-entitymanager + + + org.hibernate.javax.persistence + hibernate-jpa-2.1-api + + + + + + maven-assembly-plugin + + src/main/assembly/assembly.xml + + + + make-assembly + package + + single + + + + + + + diff --git a/metron-interface/metron-rest/src/main/assembly/assembly.xml b/metron-interface/metron-rest/src/main/assembly/assembly.xml new file mode 100644 index 0000000000..7a5dbeeb4b --- /dev/null +++ b/metron-interface/metron-rest/src/main/assembly/assembly.xml @@ -0,0 +1,54 @@ + + + + archive + + tar.gz + + false + + + ${project.basedir}/src/main/config + /config + true + + **/*.formatted + **/*.filtered + + 0644 + unix + true + + + ${project.basedir}/src/main/scripts + /bin + true + + **/*.formatted + **/*.filtered + + 0755 + unix + true + + + ${project.basedir}/target + + ${project.artifactId}-${project.version}.jar + + /lib + true + + + diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestApplication.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestApplication.java new file mode 100644 index 0000000000..ac89b1dc67 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestApplication.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class MetronRestApplication { + + public static void main(String[] args) { + SpringApplication.run(MetronRestApplication.class, args); + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java new file mode 100644 index 0000000000..87862222e5 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest; + +import org.apache.metron.parsers.GrokParser; + +public class MetronRestConstants { + + public static final String DEV_PROFILE = "dev"; + public static final String TEST_PROFILE = "test"; + public static final String DOCKER_PROFILE = "docker"; + public static final String CSRF_ENABLE_PROFILE = "csrf-enable"; + + public static final String GROK_DEFAULT_PATH_SPRING_PROPERTY = "grok.path.default"; + public static final String GROK_TEMP_PATH_SPRING_PROPERTY = "grok.path.temp"; + public static final String GROK_CLASS_NAME = GrokParser.class.getName(); + public static final String GROK_PATH_KEY = "grokPath"; + public static final String GROK_STATEMENT_KEY = "grokStatement"; + public static final String GROK_PATTERN_LABEL_KEY = "patternLabel"; + + public static final String STORM_UI_SPRING_PROPERTY = "storm.ui.url"; + public static final String TOPOLOGY_SUMMARY_URL = "/api/v1/topology/summary"; + public static final String TOPOLOGY_URL = "/api/v1/topology"; + public static final String ENRICHMENT_TOPOLOGY_NAME = "enrichment"; + public static final String INDEXING_TOPOLOGY_NAME = "indexing"; + public static final String PARSER_SCRIPT_PATH_SPRING_PROPERTY = "storm.parser.script.path"; + public static final String ENRICHMENT_SCRIPT_PATH_SPRING_PROPERTY = "storm.enrichment.script.path"; + public static final String INDEXING_SCRIPT_PATH_SPRING_PROPERTY = "storm.indexing.script.path"; + + public static final String ZK_URL_SPRING_PROPERTY = "zookeeper.url"; + public static final String ZK_CLIENT_SESSION_TIMEOUT = "zookeeper.client.timeout.session"; + public static final String ZK_CLIENT_CONNECTION_TIMEOUT = "zookeeper.client.timeout.connection"; + public static final String CURATOR_SLEEP_TIME = "curator.sleep.time"; + public static final String CURATOR_MAX_RETRIES = "curator.max.retries"; + + public static final String KAFKA_BROKER_URL_SPRING_PROPERTY = "kafka.broker.url"; + + public static final String HDFS_URL_SPRING_PROPERTY = "hdfs.namenode.url"; + public static final String DEFAULT_HDFS_URL = "file:///"; +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/RestException.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/RestException.java new file mode 100644 index 0000000000..1f985f7b18 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/RestException.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest; + +public class RestException extends Exception { + + private String fullMessage; + + public RestException(Exception e) { super( e.getMessage(), e.getCause()); } + + public RestException(String message) { + super(message); + } + + public RestException(String message, Throwable cause) { + super(message, cause); + } + + public RestException(String message, String fullMessage, Throwable cause) { + super(message, cause); + this.fullMessage = fullMessage; + } + + public String getFullMessage() { + return this.fullMessage; + } + +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/GrokConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/GrokConfig.java new file mode 100644 index 0000000000..1166e9cd26 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/GrokConfig.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.config; + +import oi.thekraken.grok.api.Grok; +import oi.thekraken.grok.api.exception.GrokException; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.io.InputStreamReader; + +@Configuration +public class GrokConfig { + + @Bean + public Grok commonGrok() throws GrokException { + Grok grok = new Grok(); + grok.addPatternFromReader(new InputStreamReader(getClass().getResourceAsStream("/patterns/common"))); + return grok; + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java new file mode 100644 index 0000000000..f4b8bdd783 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/HadoopConfig.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.config; + +import org.apache.metron.rest.MetronRestConstants; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; + +@Configuration +public class HadoopConfig { + + @Autowired + private Environment environment; + + @Bean + public org.apache.hadoop.conf.Configuration configuration() { + org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration(); + configuration.set("fs.defaultFS", environment.getProperty(MetronRestConstants.HDFS_URL_SPRING_PROPERTY, MetronRestConstants.DEFAULT_HDFS_URL)); + return configuration; + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java new file mode 100644 index 0000000000..f6ff73c7db --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.config; + +import kafka.admin.AdminUtils$; +import kafka.utils.ZkUtils; +import org.I0Itec.zkclient.ZkClient; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.metron.rest.MetronRestConstants; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.core.env.Environment; + +import java.util.Properties; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; + +@Configuration +@Profile("!" + TEST_PROFILE) +public class KafkaConfig { + + @Autowired + private Environment environment; + + @Autowired + private ZkClient zkClient; + + @Bean + public ZkUtils zkUtils() { + return ZkUtils.apply(zkClient, false); + } + + @Bean(destroyMethod = "close") + public KafkaConsumer kafkaConsumer() { + Properties props = new Properties(); + props.put("bootstrap.servers", environment.getProperty(MetronRestConstants.KAFKA_BROKER_URL_SPRING_PROPERTY)); + props.put("group.id", "metron-config"); + props.put("enable.auto.commit", "false"); + props.put("auto.commit.interval.ms", "1000"); + props.put("session.timeout.ms", "30000"); + props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + return new KafkaConsumer<>(props); + } + + @Bean + public AdminUtils$ adminUtils() { + return AdminUtils$.MODULE$; + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/MvcConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/MvcConfig.java new file mode 100644 index 0000000000..ed8216460a --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/MvcConfig.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.config; + +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.CorsRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter; + +@Configuration +public class MvcConfig extends WebMvcConfigurerAdapter { + + @Override + public void addCorsMappings(CorsRegistry registry) { + registry.addMapping("/**"); + } +} \ No newline at end of file diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/RestTemplateConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/RestTemplateConfig.java new file mode 100644 index 0000000000..8fea90cf28 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/RestTemplateConfig.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.web.client.RestTemplate; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; + +@Configuration +@Profile("!" + TEST_PROFILE) +public class RestTemplateConfig { + + @Bean + public RestTemplate restTemplate() { + return new RestTemplate(); + } + +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/StormConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/StormConfig.java new file mode 100644 index 0000000000..7a61cbce08 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/StormConfig.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.config; + +import org.apache.metron.rest.service.impl.DockerStormCLIWrapper; +import org.apache.metron.rest.service.impl.StormCLIWrapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.core.env.Environment; + +import java.util.Arrays; + +import static org.apache.metron.rest.MetronRestConstants.DOCKER_PROFILE; +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; + +@Configuration +@Profile("!" + TEST_PROFILE) +public class StormConfig { + + @Autowired + private Environment environment; + + @Bean + public StormCLIWrapper stormCLIClientWrapper() { + if (Arrays.asList(environment.getActiveProfiles()).contains(DOCKER_PROFILE)) { + return new DockerStormCLIWrapper(environment); + } else { + return new StormCLIWrapper(); + } + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/SwaggerConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/SwaggerConfig.java new file mode 100644 index 0000000000..9e82196132 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/SwaggerConfig.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import springfox.documentation.builders.PathSelectors; +import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.spi.DocumentationType; +import springfox.documentation.spring.web.plugins.Docket; +import springfox.documentation.swagger2.annotations.EnableSwagger2; + +@Configuration +@EnableSwagger2 +public class SwaggerConfig { + @Bean + public Docket api() { + return new Docket(DocumentationType.SWAGGER_2) + .select() + .apis(RequestHandlerSelectors.any()) + .paths(PathSelectors.any()) + .build(); + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/WebSecurityConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/WebSecurityConfig.java new file mode 100644 index 0000000000..789098a0aa --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/WebSecurityConfig.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.config; + +import org.apache.metron.rest.MetronRestConstants; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; +import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; +import org.springframework.security.web.authentication.logout.HttpStatusReturningLogoutSuccessHandler; +import org.springframework.security.web.csrf.CookieCsrfTokenRepository; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; + +import javax.sql.DataSource; +import java.util.Arrays; +import java.util.List; + +@Configuration +@EnableWebSecurity +@Controller +public class WebSecurityConfig extends WebSecurityConfigurerAdapter { + + @Autowired + private Environment environment; + + @RequestMapping({"/login", "/logout", "/sensors", "/sensors*/**"}) + public String handleNGRequests() { + return "forward:/index.html"; + } + + @Override + protected void configure(HttpSecurity http) throws Exception { + http + .authorizeRequests() + .antMatchers("/", "/home", "/login").permitAll() + .antMatchers("/app/**").permitAll() + .antMatchers("/vendor/**").permitAll() + .antMatchers("/fonts/**").permitAll() + .antMatchers("/assets/images/**").permitAll() + .antMatchers("/*.js").permitAll() + .antMatchers("/*.ttf").permitAll() + .antMatchers("/*.woff2").permitAll() + .anyRequest().authenticated() + .and().httpBasic() + .and() + .logout() + .logoutSuccessHandler(new HttpStatusReturningLogoutSuccessHandler()) + .invalidateHttpSession(true) + .deleteCookies("JSESSIONID"); + if (Arrays.asList(environment.getActiveProfiles()).contains(MetronRestConstants.CSRF_ENABLE_PROFILE)) { + http.csrf().csrfTokenRepository(CookieCsrfTokenRepository.withHttpOnlyFalse()); + } else { + http.csrf().disable(); + } + } + + @Autowired + private DataSource dataSource; + + @Autowired + public void configureJdbc(AuthenticationManagerBuilder auth) throws Exception { + List activeProfiles = Arrays.asList(environment.getActiveProfiles()); + if (activeProfiles.contains(MetronRestConstants.DEV_PROFILE) || + activeProfiles.contains(MetronRestConstants.TEST_PROFILE)) { + auth.jdbcAuthentication().dataSource(dataSource) + .withUser("user").password("password").roles("USER").and() + .withUser("user1").password("password").roles("USER").and() + .withUser("user2").password("password").roles("USER").and() + .withUser("admin").password("password").roles("USER", "ADMIN"); + } else { + auth.jdbcAuthentication().dataSource(dataSource); + } + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/ZookeeperConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/ZookeeperConfig.java new file mode 100644 index 0000000000..5f58193210 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/ZookeeperConfig.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.config; + +import kafka.utils.ZKStringSerializer$; +import org.I0Itec.zkclient.ZkClient; +import org.apache.curator.RetryPolicy; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.metron.rest.MetronRestConstants; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.core.env.Environment; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; + +@Configuration +@Profile("!" + TEST_PROFILE) +public class ZookeeperConfig { + + @Bean(initMethod = "start", destroyMethod="close") + public CuratorFramework client(Environment environment) { + int sleepTime = Integer.parseInt(environment.getProperty(MetronRestConstants.CURATOR_SLEEP_TIME)); + int maxRetries = Integer.parseInt(environment.getProperty(MetronRestConstants.CURATOR_MAX_RETRIES)); + RetryPolicy retryPolicy = new ExponentialBackoffRetry(sleepTime, maxRetries); + return CuratorFrameworkFactory.newClient(environment.getProperty(MetronRestConstants.ZK_URL_SPRING_PROPERTY), retryPolicy); + } + + @Bean(destroyMethod="close") + public ZkClient zkClient(Environment environment) { + int sessionTimeout = Integer.parseInt(environment.getProperty(MetronRestConstants.ZK_CLIENT_SESSION_TIMEOUT)); + int connectionTimeout = Integer.parseInt(environment.getProperty(MetronRestConstants.ZK_CLIENT_CONNECTION_TIMEOUT)); + return new ZkClient(environment.getProperty(MetronRestConstants.ZK_URL_SPRING_PROPERTY), sessionTimeout, connectionTimeout, ZKStringSerializer$.MODULE$); + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GlobalConfigController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GlobalConfigController.java new file mode 100644 index 0000000000..739d68bc2b --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GlobalConfigController.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.GlobalConfigService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +@RestController +@RequestMapping("/api/v1/global/config") +public class GlobalConfigController { + + @Autowired + private GlobalConfigService globalConfigService; + + @ApiOperation(value = "Creates or updates the Global Config in Zookeeper") + @ApiResponses(value = { @ApiResponse(message = "Global Config updated. Returns saved Global Config JSON", code = 200), + @ApiResponse(message = "Global Config created. Returns saved Global Config JSON", code = 201) }) + @RequestMapping(method = RequestMethod.POST) + ResponseEntity> save(@ApiParam(name="globalConfig", value="The Global Config JSON to be saved", required=true)@RequestBody Map globalConfig) throws RestException { + if (globalConfigService.get() == null) { + return new ResponseEntity<>(globalConfigService.save(globalConfig), HttpStatus.CREATED); + } else { + return new ResponseEntity<>(globalConfigService.save(globalConfig), HttpStatus.OK); + } + } + + @ApiOperation(value = "Retrieves the current Global Config from Zookeeper") + @ApiResponses(value = { @ApiResponse(message = "Returns current Global Config JSON in Zookeeper", code = 200), + @ApiResponse(message = "Global Config JSON was not found in Zookeeper", code = 404) }) + @RequestMapping(method = RequestMethod.GET) + ResponseEntity> get() throws RestException { + Map globalConfig = globalConfigService.get(); + if (globalConfig != null) { + return new ResponseEntity<>(globalConfig, HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } + + @ApiOperation(value = "Deletes the current Global Config from Zookeeper") + @ApiResponses(value = { @ApiResponse(message = "Global Config JSON was deleted", code = 200), + @ApiResponse(message = "Global Config JSON was not found in Zookeeper", code = 404) }) + @RequestMapping(method = RequestMethod.DELETE) + ResponseEntity delete() throws RestException { + if (globalConfigService.delete()) { + return new ResponseEntity<>(HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GrokController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GrokController.java new file mode 100644 index 0000000000..d561897813 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/GrokController.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.GrokValidation; +import org.apache.metron.rest.service.GrokService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +@RestController +@RequestMapping("/api/v1/grok") +public class GrokController { + + @Autowired + private GrokService grokService; + + @ApiOperation(value = "Applies a Grok statement to a sample message") + @ApiResponse(message = "JSON results", code = 200) + @RequestMapping(value = "/validate", method = RequestMethod.POST) + ResponseEntity post(@ApiParam(name = "grokValidation", value = "Object containing Grok statement and sample message", required = true) @RequestBody GrokValidation grokValidation) throws RestException { + return new ResponseEntity<>(grokService.validateGrokStatement(grokValidation), HttpStatus.OK); + } + + @ApiOperation(value = "Lists the common Grok statements available in Metron") + @ApiResponse(message = "JSON object containing pattern label/Grok statements key value pairs", code = 200) + @RequestMapping(value = "/list", method = RequestMethod.GET) + ResponseEntity> list() throws RestException { + return new ResponseEntity<>(grokService.getCommonGrokPatterns(), HttpStatus.OK); + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java new file mode 100644 index 0000000000..bf7ae84b93 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.apache.hadoop.fs.Path; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.HdfsService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +import static java.nio.charset.StandardCharsets.UTF_8; + +@RestController +@RequestMapping("/api/v1/hdfs") +public class HdfsController { + + @Autowired + private HdfsService hdfsService; + + @ApiOperation(value = "Reads a file from HDFS and returns the contents") + @ApiResponse(message = "Returns file contents", code = 200) + @RequestMapping(value = "/list", method = RequestMethod.GET) + ResponseEntity> list(@ApiParam(name = "path", value = "Path to HDFS directory", required = true) @RequestParam String path) throws RestException { + return new ResponseEntity<>(hdfsService.list(new Path(path)), HttpStatus.OK); + } + + @ApiOperation(value = "Reads a file from HDFS and returns the contents") + @ApiResponse(message = "Returns file contents", code = 200) + @RequestMapping(method = RequestMethod.GET) + ResponseEntity read(@ApiParam(name = "path", value = "Path to HDFS file", required = true) @RequestParam String path) throws RestException { + String contents = hdfsService.read(new Path(path)); + if (contents != null) { + return new ResponseEntity<>(hdfsService.read(new Path(path)), HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + + } + + @ApiOperation(value = "Writes contents to an HDFS file. Warning: this will overwite the contents of a file if it already exists.") + @ApiResponse(message = "Contents were written", code = 200) + @RequestMapping(method = RequestMethod.POST) + ResponseEntity write(@ApiParam(name="path", value="Path to HDFS file", required=true) @RequestParam String path, + @ApiParam(name="contents", value="File contents", required=true) @RequestBody String contents) throws RestException { + hdfsService.write(new Path(path), contents.getBytes(UTF_8)); + return new ResponseEntity<>(HttpStatus.OK); + + } + + @ApiOperation(value = "Deletes a file from HDFS") + @ApiResponses(value = { @ApiResponse(message = "File was deleted", code = 200), + @ApiResponse(message = "File was not found in HDFS", code = 404) }) + @RequestMapping(method = RequestMethod.DELETE) + ResponseEntity delete(@ApiParam(name = "path", value = "Path to HDFS file", required = true) @RequestParam String path, + @ApiParam(name = "recursive", value = "Delete files recursively") @RequestParam(required = false, defaultValue = "false") boolean recursive) throws RestException { + if (hdfsService.delete(new Path(path), recursive)) { + return new ResponseEntity<>(HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java new file mode 100644 index 0000000000..876558b198 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.KafkaTopic; +import org.apache.metron.rest.service.KafkaService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Set; + +@RestController +@RequestMapping("/api/v1/kafka") +public class KafkaController { + + @Autowired + private KafkaService kafkaService; + + @ApiOperation(value = "Creates a new Kafka topic") + @ApiResponse(message = "Returns saved Kafka topic", code = 200) + @RequestMapping(value = "/topic", method = RequestMethod.POST) + ResponseEntity save(@ApiParam(name="topic", value="Kafka topic", required=true)@RequestBody KafkaTopic topic) throws RestException { + return new ResponseEntity<>(kafkaService.createTopic(topic), HttpStatus.CREATED); + } + + @ApiOperation(value = "Retrieves a Kafka topic") + @ApiResponses(value = { @ApiResponse(message = "Returns Kafka topic", code = 200), + @ApiResponse(message = "Kafka topic is missing", code = 404) }) + @RequestMapping(value = "/topic/{name}", method = RequestMethod.GET) + ResponseEntity get(@ApiParam(name="name", value="Kafka topic name", required=true)@PathVariable String name) throws RestException { + KafkaTopic kafkaTopic = kafkaService.getTopic(name); + if (kafkaTopic != null) { + return new ResponseEntity<>(kafkaTopic, HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } + + @ApiOperation(value = "Retrieves all Kafka topics") + @ApiResponse(message = "Returns a list of all Kafka topics", code = 200) + @RequestMapping(value = "/topic", method = RequestMethod.GET) + ResponseEntity> list() throws Exception { + return new ResponseEntity<>(kafkaService.listTopics(), HttpStatus.OK); + } + + @ApiOperation(value = "Delets a Kafka topic") + @ApiResponses(value = { @ApiResponse(message = "Kafka topic was deleted", code = 200), + @ApiResponse(message = "Kafka topic is missing", code = 404) }) + @RequestMapping(value = "/topic/{name}", method = RequestMethod.DELETE) + ResponseEntity delete(@ApiParam(name="name", value="Kafka topic name", required=true)@PathVariable String name) throws RestException { + if (kafkaService.deleteTopic(name)) { + return new ResponseEntity<>(HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } + + @ApiOperation(value = "Retrieves a sample message from a Kafka topic using the most recent offset") + @ApiResponses(value = { @ApiResponse(message = "Returns sample message", code = 200), + @ApiResponse(message = "Either Kafka topic is missing or contains no messages", code = 404) }) + @RequestMapping(value = "/topic/{name}/sample", method = RequestMethod.GET) + ResponseEntity getSample(@ApiParam(name="name", value="Kafka topic name", required=true)@PathVariable String name) throws RestException { + String sampleMessage = kafkaService.getSampleMessage(name); + if (sampleMessage != null) { + return new ResponseEntity<>(sampleMessage, HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/RestExceptionHandler.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/RestExceptionHandler.java new file mode 100644 index 0000000000..fa73dab6a5 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/RestExceptionHandler.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.RestError; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; + +import javax.servlet.http.HttpServletRequest; + +@ControllerAdvice(basePackages = "org.apache.metron.rest.controller") +public class RestExceptionHandler extends ResponseEntityExceptionHandler { + + @ExceptionHandler(RestException.class) + @ResponseBody + ResponseEntity handleControllerException(HttpServletRequest request, Throwable ex) { + HttpStatus status = getStatus(request); + return new ResponseEntity<>(new RestError(status.value(), ex.getMessage(), ex.getMessage()), status); + } + + private HttpStatus getStatus(HttpServletRequest request) { + Integer statusCode = (Integer) request.getAttribute("javax.servlet.error.status_code"); + if (statusCode == null) { + return HttpStatus.INTERNAL_SERVER_ERROR; + } + return HttpStatus.valueOf(statusCode); + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorEnrichmentConfigController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorEnrichmentConfigController.java new file mode 100644 index 0000000000..546384ac13 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorEnrichmentConfigController.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.SensorEnrichmentConfigService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.Map; + +@RestController +@RequestMapping("/api/v1/sensor/enrichment/config") +public class SensorEnrichmentConfigController { + + @Autowired + private SensorEnrichmentConfigService sensorEnrichmentConfigService; + + @ApiOperation(value = "Updates or creates a SensorEnrichmentConfig in Zookeeper") + @ApiResponses(value = { @ApiResponse(message = "SensorEnrichmentConfig updated. Returns saved SensorEnrichmentConfig", code = 200), + @ApiResponse(message = "SensorEnrichmentConfig created. Returns saved SensorEnrichmentConfig", code = 201) }) + @RequestMapping(value = "/{name}", method = RequestMethod.POST) + ResponseEntity save(@ApiParam(name="name", value="SensorEnrichmentConfig name", required=true)@PathVariable String name, + @ApiParam(name="sensorEnrichmentConfig", value="SensorEnrichmentConfig", required=true)@RequestBody SensorEnrichmentConfig sensorEnrichmentConfig) throws RestException { + if (sensorEnrichmentConfigService.findOne(name) == null) { + return new ResponseEntity<>(sensorEnrichmentConfigService.save(name, sensorEnrichmentConfig), HttpStatus.CREATED); + } else { + return new ResponseEntity<>(sensorEnrichmentConfigService.save(name, sensorEnrichmentConfig), HttpStatus.OK); + } + } + + @ApiOperation(value = "Retrieves a SensorEnrichmentConfig from Zookeeper") + @ApiResponses(value = { @ApiResponse(message = "Returns SensorEnrichmentConfig", code = 200), + @ApiResponse(message = "SensorEnrichmentConfig is missing", code = 404) }) + @RequestMapping(value = "/{name}", method = RequestMethod.GET) + ResponseEntity findOne(@ApiParam(name="name", value="SensorEnrichmentConfig name", required=true)@PathVariable String name) throws RestException { + SensorEnrichmentConfig sensorEnrichmentConfig = sensorEnrichmentConfigService.findOne(name); + if (sensorEnrichmentConfig != null) { + return new ResponseEntity<>(sensorEnrichmentConfig, HttpStatus.OK); + } + + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + + @ApiOperation(value = "Retrieves all SensorEnrichmentConfigs from Zookeeper") + @ApiResponse(message = "Returns all SensorEnrichmentConfigs", code = 200) + @RequestMapping(method = RequestMethod.GET) + ResponseEntity> getAll() throws Exception { + return new ResponseEntity<>(sensorEnrichmentConfigService.getAll(), HttpStatus.OK); + } + + @ApiOperation(value = "Deletes a SensorEnrichmentConfig from Zookeeper") + @ApiResponses(value = { @ApiResponse(message = "SensorEnrichmentConfig was deleted", code = 200), + @ApiResponse(message = "SensorEnrichmentConfig is missing", code = 404) }) + @RequestMapping(value = "/{name}", method = RequestMethod.DELETE) + ResponseEntity delete(@ApiParam(name="name", value="SensorEnrichmentConfig name", required=true)@PathVariable String name) throws RestException { + if (sensorEnrichmentConfigService.delete(name)) { + return new ResponseEntity<>(HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } + + @ApiOperation(value = "Lists the available enrichments") + @ApiResponse(message = "Returns a list of available enrichments", code = 200) + @RequestMapping(value = "/list/available", method = RequestMethod.GET) + ResponseEntity> getAvailable() throws RestException { + return new ResponseEntity<>(sensorEnrichmentConfigService.getAvailableEnrichments(), HttpStatus.OK); + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java new file mode 100644 index 0000000000..ffd80fb35f --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorIndexingConfigController.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.SensorIndexingConfigService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +@RestController +@RequestMapping("/api/v1/sensor/indexing/config") +public class SensorIndexingConfigController { + + @Autowired + private SensorIndexingConfigService sensorIndexingConfigService; + + @ApiOperation(value = "Updates or creates a SensorIndexingConfig in Zookeeper") + @ApiResponses(value = { @ApiResponse(message = "SensorIndexingConfig updated. Returns saved SensorIndexingConfig", code = 200), + @ApiResponse(message = "SensorIndexingConfig created. Returns saved SensorIndexingConfig", code = 201) }) + @RequestMapping(value = "/{name}", method = RequestMethod.POST) + ResponseEntity> save(@ApiParam(name="name", value="SensorIndexingConfig name", required=true)@PathVariable String name, + @ApiParam(name="sensorIndexingConfig", value="SensorIndexingConfig", required=true)@RequestBody Map sensorIndexingConfig) throws RestException { + if (sensorIndexingConfigService.findOne(name) == null) { + return new ResponseEntity<>(sensorIndexingConfigService.save(name, sensorIndexingConfig), HttpStatus.CREATED); + } else { + return new ResponseEntity<>(sensorIndexingConfigService.save(name, sensorIndexingConfig), HttpStatus.OK); + } + } + + @ApiOperation(value = "Retrieves a SensorIndexingConfig from Zookeeper") + @ApiResponses(value = { @ApiResponse(message = "Returns SensorIndexingConfig", code = 200), + @ApiResponse(message = "SensorIndexingConfig is missing", code = 404) }) + @RequestMapping(value = "/{name}", method = RequestMethod.GET) + ResponseEntity> findOne(@ApiParam(name="name", value="SensorIndexingConfig name", required=true)@PathVariable String name) throws RestException { + Map sensorIndexingConfig = sensorIndexingConfigService.findOne(name); + if (sensorIndexingConfig != null) { + return new ResponseEntity<>(sensorIndexingConfig, HttpStatus.OK); + } + + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + + @ApiOperation(value = "Retrieves all SensorIndexingConfigs from Zookeeper") + @ApiResponse(message = "Returns all SensorIndexingConfigs", code = 200) + @RequestMapping(method = RequestMethod.GET) + ResponseEntity>> getAll() throws Exception { + return new ResponseEntity<>(sensorIndexingConfigService.getAll(), HttpStatus.OK); + } + + @ApiOperation(value = "Deletes a SensorIndexingConfig from Zookeeper") + @ApiResponses(value = { @ApiResponse(message = "SensorIndexingConfig was deleted", code = 200), + @ApiResponse(message = "SensorIndexingConfig is missing", code = 404) }) + @RequestMapping(value = "/{name}", method = RequestMethod.DELETE) + ResponseEntity delete(@ApiParam(name="name", value="SensorIndexingConfig name", required=true)@PathVariable String name) throws RestException { + if (sensorIndexingConfigService.delete(name)) { + return new ResponseEntity<>(HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorParserConfigController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorParserConfigController.java new file mode 100644 index 0000000000..189e2afed6 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/SensorParserConfigController.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.ParseMessageRequest; +import org.apache.metron.rest.service.SensorParserConfigService; +import org.json.simple.JSONObject; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +@RestController +@RequestMapping("/api/v1/sensor/parser/config") +public class SensorParserConfigController { + + @Autowired + private SensorParserConfigService sensorParserConfigService; + + @ApiOperation(value = "Updates or creates a SensorParserConfig in Zookeeper") + @ApiResponses(value = { @ApiResponse(message = "SensorParserConfig updated. Returns saved SensorParserConfig", code = 200), + @ApiResponse(message = "SensorParserConfig created. Returns saved SensorParserConfig", code = 201) }) + @RequestMapping(method = RequestMethod.POST) + ResponseEntity save(@ApiParam(name="sensorParserConfig", value="SensorParserConfig", required=true)@RequestBody SensorParserConfig sensorParserConfig) throws RestException { + String name = sensorParserConfig.getSensorTopic(); + if (sensorParserConfigService.findOne(name) == null) { + return new ResponseEntity<>(sensorParserConfigService.save(sensorParserConfig), HttpStatus.CREATED); + } else { + return new ResponseEntity<>(sensorParserConfigService.save(sensorParserConfig), HttpStatus.OK); + } + } + + @ApiOperation(value = "Retrieves a SensorParserConfig from Zookeeper") + @ApiResponses(value = { @ApiResponse(message = "Returns SensorParserConfig", code = 200), + @ApiResponse(message = "SensorParserConfig is missing", code = 404) }) + @RequestMapping(value = "/{name}", method = RequestMethod.GET) + ResponseEntity findOne(@ApiParam(name="name", value="SensorParserConfig name", required=true)@PathVariable String name) throws RestException { + SensorParserConfig sensorParserConfig = sensorParserConfigService.findOne(name); + if (sensorParserConfig != null) { + return new ResponseEntity<>(sensorParserConfig, HttpStatus.OK); + } + + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + + @ApiOperation(value = "Retrieves all SensorParserConfigs from Zookeeper") + @ApiResponse(message = "Returns all SensorParserConfigs", code = 200) + @RequestMapping(method = RequestMethod.GET) + ResponseEntity> findAll() throws RestException { + return new ResponseEntity<>(sensorParserConfigService.getAll(), HttpStatus.OK); + } + + @ApiOperation(value = "Deletes a SensorParserConfig from Zookeeper") + @ApiResponses(value = { @ApiResponse(message = "SensorParserConfig was deleted", code = 200), + @ApiResponse(message = "SensorParserConfig is missing", code = 404) }) + @RequestMapping(value = "/{name}", method = RequestMethod.DELETE) + ResponseEntity delete(@ApiParam(name="name", value="SensorParserConfig name", required=true)@PathVariable String name) throws RestException { + if (sensorParserConfigService.delete(name)) { + return new ResponseEntity<>(HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } + + @ApiOperation(value = "Lists the available parser classes that can be found on the classpath") + @ApiResponse(message = "Returns a list of available parser classes", code = 200) + @RequestMapping(value = "/list/available", method = RequestMethod.GET) + ResponseEntity> getAvailable() throws RestException { + return new ResponseEntity<>(sensorParserConfigService.getAvailableParsers(), HttpStatus.OK); + } + + @ApiOperation(value = "Scans the classpath for available parser classes and reloads the cached parser class list") + @ApiResponse(message = "Returns a list of available parser classes", code = 200) + @RequestMapping(value = "/reload/available", method = RequestMethod.GET) + ResponseEntity> reloadAvailable() throws RestException { + return new ResponseEntity<>(sensorParserConfigService.reloadAvailableParsers(), HttpStatus.OK); + } + + @ApiOperation(value = "Parses a sample message given a SensorParserConfig") + @ApiResponse(message = "Returns parsed message", code = 200) + @RequestMapping(value = "/parseMessage", method = RequestMethod.POST) + ResponseEntity parseMessage(@ApiParam(name="parseMessageRequest", value="Object containing a sample message and SensorParserConfig", required=true) @RequestBody ParseMessageRequest parseMessageRequest) throws RestException { + return new ResponseEntity<>(sensorParserConfigService.parseMessage(parseMessageRequest), HttpStatus.OK); + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java new file mode 100644 index 0000000000..13327f9279 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StellarController.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import org.apache.metron.common.field.transformation.FieldTransformations; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.StellarFunctionDescription; +import org.apache.metron.rest.model.SensorParserContext; +import org.apache.metron.rest.service.StellarService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.Map; + +@RestController +@RequestMapping("/api/v1/stellar") +public class StellarController { + + @Autowired + private StellarService stellarService; + + @ApiOperation(value = "Tests Stellar statements to ensure they are well-formed") + @ApiResponse(message = "Returns validation results", code = 200) + @RequestMapping(value = "/validate/rules", method = RequestMethod.POST) + ResponseEntity> validateRules(@ApiParam(name="statements", value="List of statements to validate", required=true)@RequestBody List statements) throws RestException { + return new ResponseEntity<>(stellarService.validateRules(statements), HttpStatus.OK); + } + + @ApiOperation(value = "Executes transformations against a sample message") + @ApiResponse(message = "Returns transformation results", code = 200) + @RequestMapping(value = "/apply/transformations", method = RequestMethod.POST) + ResponseEntity> applyTransformations(@ApiParam(name="transformationValidation", value="Object containing SensorParserConfig and sample message", required=true)@RequestBody SensorParserContext sensorParserContext) throws RestException { + return new ResponseEntity<>(stellarService.applyTransformations(sensorParserContext), HttpStatus.OK); + } + + @ApiOperation(value = "Retrieves field transformations") + @ApiResponse(message = "Returns a list field transformations", code = 200) + @RequestMapping(value = "/list", method = RequestMethod.GET) + ResponseEntity list() throws RestException { + return new ResponseEntity<>(stellarService.getTransformations(), HttpStatus.OK); + } + + @ApiOperation(value = "Lists the Stellar functions that can be found on the classpath") + @ApiResponse(message = "Returns a list of Stellar functions", code = 200) + @RequestMapping(value = "/list/functions", method = RequestMethod.GET) + ResponseEntity> listFunctions() throws RestException { + return new ResponseEntity<>(stellarService.getStellarFunctions(), HttpStatus.OK); + } + + @ApiOperation(value = "Lists the simple Stellar functions (functions with only 1 input) that can be found on the classpath") + @ApiResponse(message = "Returns a list of simple Stellar functions", code = 200) + @RequestMapping(value = "/list/simple/functions", method = RequestMethod.GET) + ResponseEntity> listSimpleFunctions() throws RestException { + return new ResponseEntity<>(stellarService.getSimpleStellarFunctions(), HttpStatus.OK); + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java new file mode 100644 index 0000000000..915e304b53 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java @@ -0,0 +1,190 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.apache.metron.rest.MetronRestConstants; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.TopologyResponse; +import org.apache.metron.rest.model.TopologyStatus; +import org.apache.metron.rest.service.StormAdminService; +import org.apache.metron.rest.service.StormStatusService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.Map; + +@RestController +@RequestMapping("/api/v1/storm") +public class StormController { + + @Autowired + private StormStatusService stormStatusService; + + @Autowired + private StormAdminService stormAdminService; + + @ApiOperation(value = "Retrieves the status of all Storm topologies") + @ApiResponse(message = "Returns a list of topologies with status information", code = 200) + @RequestMapping(method = RequestMethod.GET) + ResponseEntity> getAll() throws RestException { + return new ResponseEntity<>(stormStatusService.getAllTopologyStatus(), HttpStatus.OK); + } + + @ApiOperation(value = "Retrieves the status of a Storm topology") + @ApiResponses(value = { @ApiResponse(message = "Returns topology status information", code = 200), + @ApiResponse(message = "Topology is missing", code = 404) }) + @RequestMapping(value = "/{name}", method = RequestMethod.GET) + ResponseEntity get(@ApiParam(name="name", value="Topology name", required=true)@PathVariable String name) throws RestException { + TopologyStatus topologyStatus = stormStatusService.getTopologyStatus(name); + if (topologyStatus != null) { + return new ResponseEntity<>(topologyStatus, HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } + + @ApiOperation(value = "Starts a Storm parser topology") + @ApiResponse(message = "Returns start response message", code = 200) + @RequestMapping(value = "/parser/start/{name}", method = RequestMethod.GET) + ResponseEntity start(@ApiParam(name="name", value="Parser name", required=true)@PathVariable String name) throws RestException { + return new ResponseEntity<>(stormAdminService.startParserTopology(name), HttpStatus.OK); + } + + @ApiOperation(value = "Stops a Storm parser topology") + @ApiResponse(message = "Returns stop response message", code = 200) + @RequestMapping(value = "/parser/stop/{name}", method = RequestMethod.GET) + ResponseEntity stop(@ApiParam(name="name", value="Parser name", required=true)@PathVariable String name, + @ApiParam(name="stopNow", value="Stop the topology immediately")@RequestParam(required = false, defaultValue = "false") boolean stopNow) throws RestException { + return new ResponseEntity<>(stormAdminService.stopParserTopology(name, stopNow), HttpStatus.OK); + } + + @ApiOperation(value = "Activates a Storm parser topology") + @ApiResponse(message = "Returns activate response message", code = 200) + @RequestMapping(value = "/parser/activate/{name}", method = RequestMethod.GET) + ResponseEntity activate(@ApiParam(name="name", value="Parser name", required=true)@PathVariable String name) throws RestException { + return new ResponseEntity<>(stormStatusService.activateTopology(name), HttpStatus.OK); + } + + @ApiOperation(value = "Deactivates a Storm parser topology") + @ApiResponse(message = "Returns deactivate response message", code = 200) + @RequestMapping(value = "/parser/deactivate/{name}", method = RequestMethod.GET) + ResponseEntity deactivate(@ApiParam(name="name", value="Parser name", required=true)@PathVariable String name) throws RestException { + return new ResponseEntity<>(stormStatusService.deactivateTopology(name), HttpStatus.OK); + } + + @ApiOperation(value = "Retrieves the status of the Storm enrichment topology") + @ApiResponses(value = { @ApiResponse(message = "Returns topology status information", code = 200), + @ApiResponse(message = "Topology is missing", code = 404) }) + @RequestMapping(value = "/enrichment", method = RequestMethod.GET) + ResponseEntity getEnrichment() throws RestException { + TopologyStatus sensorParserStatus = stormStatusService.getTopologyStatus(MetronRestConstants.ENRICHMENT_TOPOLOGY_NAME); + if (sensorParserStatus != null) { + return new ResponseEntity<>(sensorParserStatus, HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } + + @ApiOperation(value = "Starts a Storm enrichment topology") + @ApiResponse(message = "Returns start response message", code = 200) + @RequestMapping(value = "/enrichment/start", method = RequestMethod.GET) + ResponseEntity startEnrichment() throws RestException { + return new ResponseEntity<>(stormAdminService.startEnrichmentTopology(), HttpStatus.OK); + } + + @ApiOperation(value = "Stops a Storm enrichment topology") + @ApiResponse(message = "Returns stop response message", code = 200) + @RequestMapping(value = "/enrichment/stop", method = RequestMethod.GET) + ResponseEntity stopEnrichment(@ApiParam(name="stopNow", value="Stop the topology immediately")@RequestParam(required = false, defaultValue = "false") boolean stopNow) throws RestException { + return new ResponseEntity<>(stormAdminService.stopEnrichmentTopology(stopNow), HttpStatus.OK); + } + + @ApiOperation(value = "Activates a Storm enrichment topology") + @ApiResponse(message = "Returns activate response message", code = 200) + @RequestMapping(value = "/enrichment/activate", method = RequestMethod.GET) + ResponseEntity activateEnrichment() throws RestException { + return new ResponseEntity<>(stormStatusService.activateTopology(MetronRestConstants.ENRICHMENT_TOPOLOGY_NAME), HttpStatus.OK); + } + + @ApiOperation(value = "Deactivates a Storm enrichment topology") + @ApiResponse(message = "Returns deactivate response message", code = 200) + @RequestMapping(value = "/enrichment/deactivate", method = RequestMethod.GET) + ResponseEntity deactivateEnrichment() throws RestException { + return new ResponseEntity<>(stormStatusService.deactivateTopology(MetronRestConstants.ENRICHMENT_TOPOLOGY_NAME), HttpStatus.OK); + } + + @ApiOperation(value = "Retrieves the status of the Storm indexing topology") + @ApiResponses(value = { @ApiResponse(message = "Returns topology status information", code = 200), + @ApiResponse(message = "Topology is missing", code = 404) }) + @RequestMapping(value = "/indexing", method = RequestMethod.GET) + ResponseEntity getIndexing() throws RestException { + TopologyStatus topologyStatus = stormStatusService.getTopologyStatus(MetronRestConstants.INDEXING_TOPOLOGY_NAME); + if (topologyStatus != null) { + return new ResponseEntity<>(topologyStatus, HttpStatus.OK); + } else { + return new ResponseEntity<>(HttpStatus.NOT_FOUND); + } + } + + @ApiOperation(value = "Starts a Storm indexing topology") + @ApiResponse(message = "Returns start response message", code = 200) + @RequestMapping(value = "/indexing/start", method = RequestMethod.GET) + ResponseEntity startIndexing() throws RestException { + return new ResponseEntity<>(stormAdminService.startIndexingTopology(), HttpStatus.OK); + } + + @ApiOperation(value = "Stops a Storm enrichment topology") + @ApiResponse(message = "Returns stop response message", code = 200) + @RequestMapping(value = "/indexing/stop", method = RequestMethod.GET) + ResponseEntity stopIndexing(@ApiParam(name="stopNow", value="Stop the topology immediately")@RequestParam(required = false, defaultValue = "false") boolean stopNow) throws RestException { + return new ResponseEntity<>(stormAdminService.stopIndexingTopology(stopNow), HttpStatus.OK); + } + + @ApiOperation(value = "Activates a Storm indexing topology") + @ApiResponse(message = "Returns activate response message", code = 200) + @RequestMapping(value = "/indexing/activate", method = RequestMethod.GET) + ResponseEntity activateIndexing() throws RestException { + return new ResponseEntity<>(stormStatusService.activateTopology(MetronRestConstants.INDEXING_TOPOLOGY_NAME), HttpStatus.OK); + } + + @ApiOperation(value = "Deactivates a Storm indexing topology") + @ApiResponse(message = "Returns deactivate response message", code = 200) + @RequestMapping(value = "/indexing/deactivate", method = RequestMethod.GET) + ResponseEntity deactivateIndexing() throws RestException { + return new ResponseEntity<>(stormStatusService.deactivateTopology(MetronRestConstants.INDEXING_TOPOLOGY_NAME), HttpStatus.OK); + } + + @ApiOperation(value = "Retrieves information about the Storm command line client") + @ApiResponse(message = "Returns storm command line client information", code = 200) + @RequestMapping(value = "/client/status", method = RequestMethod.GET) + ResponseEntity> clientStatus() throws RestException { + return new ResponseEntity<>(stormAdminService.getStormClientStatus(), HttpStatus.OK); + } + +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UserController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UserController.java new file mode 100644 index 0000000000..594b51f427 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UserController.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.security.Principal; + +@RestController +public class UserController { + + @ApiOperation(value = "Retrieves the current user") + @ApiResponse(message = "Current user", code = 200) + @RequestMapping("/api/v1/user") + public String user(Principal user) { + return user.getName(); + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GlobalConfigService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GlobalConfigService.java new file mode 100644 index 0000000000..7b8cf450fd --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GlobalConfigService.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service; + +import org.apache.metron.rest.RestException; + +import java.util.Map; + +public interface GlobalConfigService { + + Map save(Map globalConfig) throws RestException; + + Map get() throws RestException; + + boolean delete() throws RestException; +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java new file mode 100644 index 0000000000..268d396077 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/GrokService.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service; + +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.GrokValidation; + +import java.io.File; +import java.util.Map; + +public interface GrokService { + + Map getCommonGrokPatterns(); + + GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException; + + File saveTemporary(String statement, String name) throws RestException; + +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java new file mode 100644 index 0000000000..d5932c7bab --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service; + +import org.apache.hadoop.fs.Path; +import org.apache.metron.rest.RestException; + +import java.util.List; + +public interface HdfsService { + + String read(Path path) throws RestException; + + void write(Path path, byte[] contents) throws RestException; + + List list(Path path) throws RestException; + + boolean delete(Path path, boolean recursive) throws RestException; + } diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java new file mode 100644 index 0000000000..f3cd9019ea --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service; + +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.KafkaTopic; + +import java.util.Set; + +public interface KafkaService { + String CONSUMER_OFFSETS_TOPIC = "__consumer_offsets"; + + KafkaTopic createTopic(KafkaTopic topic) throws RestException; + + boolean deleteTopic(String name); + + KafkaTopic getTopic(String name); + + Set listTopics(); + + String getSampleMessage(String topic); +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorEnrichmentConfigService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorEnrichmentConfigService.java new file mode 100644 index 0000000000..fe84802ba0 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorEnrichmentConfigService.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service; + +import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; +import org.apache.metron.rest.RestException; + +import java.util.List; +import java.util.Map; + +public interface SensorEnrichmentConfigService { + + SensorEnrichmentConfig save(String name, SensorEnrichmentConfig sensorEnrichmentConfig) throws RestException; + + SensorEnrichmentConfig findOne(String name) throws RestException; + + Map getAll() throws RestException; + + List getAllTypes() throws RestException; + + boolean delete(String name) throws RestException; + + List getAvailableEnrichments(); + +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java new file mode 100644 index 0000000000..06994317b4 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorIndexingConfigService.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service; + +import org.apache.metron.rest.RestException; + +import java.util.List; +import java.util.Map; + +public interface SensorIndexingConfigService { + + Map save(String name, Map sensorIndexingConfig) throws RestException; + + Map findOne(String name) throws RestException; + + Map> getAll() throws RestException; + + List getAllTypes() throws RestException; + + boolean delete(String name) throws RestException; + +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java new file mode 100644 index 0000000000..9b863b8f6a --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/SensorParserConfigService.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service; + +import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.ParseMessageRequest; +import org.json.simple.JSONObject; + +import java.util.List; +import java.util.Map; + +public interface SensorParserConfigService { + + SensorParserConfig save(SensorParserConfig sensorParserConfig) throws RestException; + + SensorParserConfig findOne(String name) throws RestException; + + Iterable getAll() throws RestException; + + List getAllTypes() throws RestException; + + boolean delete(String name) throws RestException; + + Map getAvailableParsers(); + + Map reloadAvailableParsers(); + + JSONObject parseMessage(ParseMessageRequest parseMessageRequest) throws RestException; +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java new file mode 100644 index 0000000000..14cd31fc9a --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StellarService.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service; + +import org.apache.metron.common.field.transformation.FieldTransformations; +import org.apache.metron.rest.model.StellarFunctionDescription; +import org.apache.metron.rest.model.SensorParserContext; + +import java.util.List; +import java.util.Map; + +public interface StellarService { + + Map validateRules(List rules); + + Map applyTransformations(SensorParserContext sensorParserContext); + + FieldTransformations[] getTransformations(); + + List getStellarFunctions(); + + List getSimpleStellarFunctions(); + +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormAdminService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormAdminService.java new file mode 100644 index 0000000000..8c1e228864 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormAdminService.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service; + +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.TopologyResponse; + +import java.util.Map; + +public interface StormAdminService { + + TopologyResponse startParserTopology(String name) throws RestException; + + TopologyResponse stopParserTopology(String name, boolean stopNow) throws RestException; + + TopologyResponse startEnrichmentTopology() throws RestException; + + TopologyResponse stopEnrichmentTopology(boolean stopNow) throws RestException; + + TopologyResponse startIndexingTopology() throws RestException; + + TopologyResponse stopIndexingTopology(boolean stopNow) throws RestException; + + Map getStormClientStatus() throws RestException; +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormStatusService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormStatusService.java new file mode 100644 index 0000000000..76216d0207 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormStatusService.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service; + +import org.apache.metron.rest.model.TopologyResponse; +import org.apache.metron.rest.model.TopologyStatus; +import org.apache.metron.rest.model.TopologySummary; + +import java.util.List; + +public interface StormStatusService { + + TopologySummary getTopologySummary(); + + TopologyStatus getTopologyStatus(String name); + + List getAllTopologyStatus(); + + TopologyResponse activateTopology(String name); + + TopologyResponse deactivateTopology(String name); +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapper.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapper.java new file mode 100644 index 0000000000..059afd781c --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapper.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.commons.lang3.ArrayUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Map; + +public class DockerStormCLIWrapper extends StormCLIWrapper { + + private static final Logger LOG = LoggerFactory.getLogger(DockerStormCLIWrapper.class); + + private Environment environment; + + @Autowired + public DockerStormCLIWrapper(final Environment environment) { + this.environment = environment; + } + + @Override + protected ProcessBuilder getProcessBuilder(final String... command) { + final String[] dockerCommand = {"docker-compose", "-f", environment.getProperty("docker.compose.path"), "-p", "metron", "exec", "storm"}; + final ProcessBuilder pb = new ProcessBuilder(ArrayUtils.addAll(dockerCommand, command)); + final Map pbEnvironment = pb.environment(); + pbEnvironment.put("METRON_VERSION", environment.getProperty("metron.version")); + setDockerEnvironment(pbEnvironment); + return pb; + } + + private void setDockerEnvironment(final Map environmentVariables) { + final ProcessBuilder pb = getDockerEnvironmentProcessBuilder(); + try { + final Process process = pb.start(); + final BufferedReader inputStream = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + while ((line = inputStream.readLine()) != null) { + if (line.startsWith("export")) { + final String[] parts = line.replaceFirst("export ", "").split("="); + environmentVariables.put(parts[0], parts[1].replaceAll("\"", "")); + } + } + process.waitFor(); + } catch (IOException | InterruptedException e) { + LOG.error(e.getMessage(), e); + } + } + + private ProcessBuilder getDockerEnvironmentProcessBuilder() { + String[] command = {"docker-machine", "env", "metron-machine"}; + return new ProcessBuilder(command); + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImpl.java new file mode 100644 index 0000000000..e80380bd14 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImpl.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.common.configuration.ConfigurationType; +import org.apache.metron.common.configuration.ConfigurationsUtils; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.GlobalConfigService; +import org.apache.zookeeper.KeeperException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.ByteArrayInputStream; +import java.util.Map; + +@Service +public class GlobalConfigServiceImpl implements GlobalConfigService { + private CuratorFramework client; + + @Autowired + public GlobalConfigServiceImpl(CuratorFramework client) { + this.client = client; + } + + @Override + public Map save(Map globalConfig) throws RestException { + try { + ConfigurationsUtils.writeGlobalConfigToZookeeper(globalConfig, client); + } catch (Exception e) { + throw new RestException(e); + } + return globalConfig; + } + + @Override + public Map get() throws RestException { + Map globalConfig; + try { + byte[] globalConfigBytes = ConfigurationsUtils.readGlobalConfigBytesFromZookeeper(client); + globalConfig = JSONUtils.INSTANCE.load(new ByteArrayInputStream(globalConfigBytes), new TypeReference>(){}); + } catch (KeeperException.NoNodeException e) { + return null; + } catch (Exception e) { + throw new RestException(e); + } + return globalConfig; + } + + @Override + public boolean delete() throws RestException { + try { + client.delete().forPath(ConfigurationType.GLOBAL.getZookeeperRoot()); + } catch (KeeperException.NoNodeException e) { + return false; + } catch (Exception e) { + throw new RestException(e); + } + return true; + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java new file mode 100644 index 0000000000..3f2de2f62a --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import oi.thekraken.grok.api.Grok; +import oi.thekraken.grok.api.Match; +import org.apache.directory.api.util.Strings; +import org.apache.hadoop.fs.Path; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.GrokValidation; +import org.apache.metron.rest.service.GrokService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.util.Map; + +import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY; + +@Service +public class GrokServiceImpl implements GrokService { + + private Environment environment; + + private Grok commonGrok; + + @Autowired + public GrokServiceImpl(Environment environment, Grok commonGrok) { + this.environment = environment; + this.commonGrok = commonGrok; + } + + @Override + public Map getCommonGrokPatterns() { + return commonGrok.getPatterns(); + } + + @Override + public GrokValidation validateGrokStatement(GrokValidation grokValidation) throws RestException { + Map results; + try { + if (grokValidation.getPatternLabel() == null) { + throw new RestException("Pattern label is required"); + } + if (Strings.isEmpty(grokValidation.getStatement())) { + throw new RestException("Grok statement is required"); + } + Grok grok = new Grok(); + grok.addPatternFromReader(new InputStreamReader(getClass().getResourceAsStream("/patterns/common"))); + grok.addPatternFromReader(new StringReader(grokValidation.getStatement())); + String grokPattern = "%{" + grokValidation.getPatternLabel() + "}"; + grok.compile(grokPattern); + Match gm = grok.match(grokValidation.getSampleData()); + gm.captures(); + results = gm.toMap(); + results.remove(grokValidation.getPatternLabel()); + } catch (Exception e) { + throw new RestException(e); + } + grokValidation.setResults(results); + return grokValidation; + } + + @Override + public File saveTemporary(String statement, String name) throws RestException { + if (statement != null) { + try { + File grokDirectory = new File(getTemporaryGrokRootPath()); + if (!grokDirectory.exists()) { + grokDirectory.mkdirs(); + } + File path = new File(grokDirectory, name); + FileWriter fileWriter = new FileWriter(new File(grokDirectory, name)); + fileWriter.write(statement); + fileWriter.close(); + return path; + } catch (IOException e) { + throw new RestException(e); + } + } else { + throw new RestException("A grokStatement must be provided"); + } + } + + private String getTemporaryGrokRootPath() { + String grokTempPath = environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY); + Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); + return new Path(grokTempPath, authentication.getName()).toString(); + } + +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java new file mode 100644 index 0000000000..789c4216b5 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.HdfsService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static java.nio.charset.StandardCharsets.UTF_8; + +@Service +public class HdfsServiceImpl implements HdfsService { + + private Configuration configuration; + + @Autowired + public HdfsServiceImpl(Configuration configuration) { + this.configuration = configuration; + } + + @Override + public List list(Path path) throws RestException { + try { + return Arrays.asList(FileSystem.get(configuration).listStatus(path)).stream().map(fileStatus -> fileStatus.getPath().getName()).collect(Collectors.toList()); + } catch (IOException e) { + throw new RestException(e); + } + } + + @Override + public String read(Path path) throws RestException { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + try { + IOUtils.copyBytes(FileSystem.get(configuration).open(path), byteArrayOutputStream, configuration); + } catch (FileNotFoundException e) { + return null; + } catch (IOException e) { + throw new RestException(e); + } + return new String(byteArrayOutputStream.toByteArray(), UTF_8); + } + + @Override + public void write(Path path, byte[] contents) throws RestException { + FSDataOutputStream fsDataOutputStream; + try { + fsDataOutputStream = FileSystem.get(configuration).create(path, true); + fsDataOutputStream.write(contents); + fsDataOutputStream.close(); + } catch (IOException e) { + throw new RestException(e); + } + } + + @Override + public boolean delete(Path path, boolean recursive) throws RestException { + try { + return FileSystem.get(configuration).delete(path, recursive); + } catch (IOException e) { + throw new RestException(e); + } + } + } diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java new file mode 100644 index 0000000000..7b10dc4ed4 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import kafka.admin.AdminOperationException; +import kafka.admin.AdminUtils$; +import kafka.admin.RackAwareMode; +import kafka.utils.ZkUtils; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.KafkaTopic; +import org.apache.metron.rest.service.KafkaService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Service +public class KafkaServiceImpl implements KafkaService { + private ZkUtils zkUtils; + private KafkaConsumer kafkaConsumer; + private AdminUtils$ adminUtils; + + @Autowired + public KafkaServiceImpl(ZkUtils zkUtils, KafkaConsumer kafkaConsumer, AdminUtils$ adminUtils) { + this.zkUtils = zkUtils; + this.kafkaConsumer = kafkaConsumer; + this.adminUtils = adminUtils; + } + + @Override + public KafkaTopic createTopic(KafkaTopic topic) throws RestException { + if (!listTopics().contains(topic.getName())) { + try { + adminUtils.createTopic(zkUtils, topic.getName(), topic.getNumPartitions(), topic.getReplicationFactor(), topic.getProperties(), RackAwareMode.Disabled$.MODULE$); + } catch (AdminOperationException e) { + throw new RestException(e); + } + } + return topic; + } + + @Override + public boolean deleteTopic(String name) { + Set topics = listTopics(); + if (topics != null && topics.contains(name)) { + adminUtils.deleteTopic(zkUtils, name); + return true; + } else { + return false; + } + } + + @Override + public KafkaTopic getTopic(String name) { + KafkaTopic kafkaTopic = null; + if (listTopics().contains(name)) { + List partitionInfos = kafkaConsumer.partitionsFor(name); + if (partitionInfos.size() > 0) { + PartitionInfo partitionInfo = partitionInfos.get(0); + kafkaTopic = new KafkaTopic(); + kafkaTopic.setName(name); + kafkaTopic.setNumPartitions(partitionInfos.size()); + kafkaTopic.setReplicationFactor(partitionInfo.replicas().length); + } + } + return kafkaTopic; + } + + @Override + public Set listTopics() { + Set topics; + synchronized (this) { + Map> topicsInfo = kafkaConsumer.listTopics(); + topics = topicsInfo == null ? new HashSet<>() : topicsInfo.keySet(); + topics.remove(CONSUMER_OFFSETS_TOPIC); + } + return topics; + } + + @Override + public String getSampleMessage(String topic) { + String message = null; + if (listTopics().contains(topic)) { + synchronized (this) { + kafkaConsumer.assign(kafkaConsumer.partitionsFor(topic).stream() + .map(partitionInfo -> new TopicPartition(topic, partitionInfo.partition())) + .collect(Collectors.toList())); + + kafkaConsumer.assignment().stream() + .filter(p -> (kafkaConsumer.position(p) -1) >= 0) + .forEach(p -> kafkaConsumer.seek(p, kafkaConsumer.position(p) - 1)); + + ConsumerRecords records = kafkaConsumer.poll(100); + message = records.isEmpty() ? null : records.iterator().next().value(); + kafkaConsumer.unsubscribe(); + } + } + return message; + } + +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java new file mode 100644 index 0000000000..2bfef89ce4 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImpl.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.common.configuration.ConfigurationType; +import org.apache.metron.common.configuration.ConfigurationsUtils; +import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.SensorEnrichmentConfigService; +import org.apache.zookeeper.KeeperException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +public class SensorEnrichmentConfigServiceImpl implements SensorEnrichmentConfigService { + + private ObjectMapper objectMapper; + + private CuratorFramework client; + + @Autowired + public SensorEnrichmentConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client) { + this.objectMapper = objectMapper; + this.client = client; + } + + @Override + public SensorEnrichmentConfig save(String name, SensorEnrichmentConfig sensorEnrichmentConfig) throws RestException { + try { + ConfigurationsUtils.writeSensorEnrichmentConfigToZookeeper(name, objectMapper.writeValueAsString(sensorEnrichmentConfig).getBytes(), client); + } catch (Exception e) { + throw new RestException(e); + } + return sensorEnrichmentConfig; + } + + @Override + public SensorEnrichmentConfig findOne(String name) throws RestException { + SensorEnrichmentConfig sensorEnrichmentConfig; + try { + sensorEnrichmentConfig = ConfigurationsUtils.readSensorEnrichmentConfigFromZookeeper(name, client); + } catch (KeeperException.NoNodeException e) { + return null; + } catch (Exception e) { + throw new RestException(e); + } + return sensorEnrichmentConfig; + } + + @Override + public Map getAll() throws RestException { + Map sensorEnrichmentConfigs = new HashMap<>(); + List sensorNames = getAllTypes(); + for (String name : sensorNames) { + sensorEnrichmentConfigs.put(name, findOne(name)); + } + return sensorEnrichmentConfigs; + } + + @Override + public List getAllTypes() throws RestException { + List types; + try { + types = client.getChildren().forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot()); + } catch (KeeperException.NoNodeException e) { + types = new ArrayList<>(); + } catch (Exception e) { + throw new RestException(e); + } + return types; + } + + @Override + public boolean delete(String name) throws RestException { + try { + client.delete().forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/" + name); + } catch (KeeperException.NoNodeException e) { + return false; + } catch (Exception e) { + throw new RestException(e); + } + return true; + } + + @Override + public List getAvailableEnrichments() { + return new ArrayList() {{ + add("geo"); + add("host"); + add("whois"); + }}; + } + +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java new file mode 100644 index 0000000000..9f984e0021 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImpl.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.common.configuration.ConfigurationType; +import org.apache.metron.common.configuration.ConfigurationsUtils; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.SensorIndexingConfigService; +import org.apache.zookeeper.KeeperException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.ByteArrayInputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +public class SensorIndexingConfigServiceImpl implements SensorIndexingConfigService { + + private ObjectMapper objectMapper; + + private CuratorFramework client; + + @Autowired + public SensorIndexingConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client) { + this.objectMapper = objectMapper; + this.client = client; + } + + @Override + public Map save(String name, Map sensorIndexingConfig) throws RestException { + try { + ConfigurationsUtils.writeSensorIndexingConfigToZookeeper(name, objectMapper.writeValueAsString(sensorIndexingConfig).getBytes(), client); + } catch (Exception e) { + throw new RestException(e); + } + return sensorIndexingConfig; + } + + @Override + public Map findOne(String name) throws RestException { + Map sensorIndexingConfig; + try { + byte[] sensorIndexingConfigBytes = ConfigurationsUtils.readSensorIndexingConfigBytesFromZookeeper(name, client); + sensorIndexingConfig = JSONUtils.INSTANCE.load(new ByteArrayInputStream(sensorIndexingConfigBytes), new TypeReference>(){}); + } catch (KeeperException.NoNodeException e) { + return null; + } catch (Exception e) { + throw new RestException(e); + } + return sensorIndexingConfig; + } + + @Override + public Map> getAll() throws RestException { + Map> sensorIndexingConfigs = new HashMap<>(); + List sensorNames = getAllTypes(); + for (String name : sensorNames) { + sensorIndexingConfigs.put(name, findOne(name)); + } + return sensorIndexingConfigs; + } + + @Override + public List getAllTypes() throws RestException { + List types; + try { + types = client.getChildren().forPath(ConfigurationType.INDEXING.getZookeeperRoot()); + } catch (KeeperException.NoNodeException e) { + types = new ArrayList<>(); + } catch (Exception e) { + throw new RestException(e); + } + return types; + } + + @Override + public boolean delete(String name) throws RestException { + try { + client.delete().forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/" + name); + } catch (KeeperException.NoNodeException e) { + return false; + } catch (Exception e) { + throw new RestException(e); + } + return true; + } + +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java new file mode 100644 index 0000000000..eddfc8d74d --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java @@ -0,0 +1,178 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.curator.framework.CuratorFramework; +import org.apache.metron.common.configuration.ConfigurationType; +import org.apache.metron.common.configuration.ConfigurationsUtils; +import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.parsers.interfaces.MessageParser; +import org.apache.metron.rest.MetronRestConstants; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.ParseMessageRequest; +import org.apache.metron.rest.service.GrokService; +import org.apache.metron.rest.service.SensorParserConfigService; +import org.apache.zookeeper.KeeperException; +import org.json.simple.JSONObject; +import org.reflections.Reflections; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME; + +@Service +public class SensorParserConfigServiceImpl implements SensorParserConfigService { + + private ObjectMapper objectMapper; + + private CuratorFramework client; + + private GrokService grokService; + + @Autowired + public SensorParserConfigServiceImpl(ObjectMapper objectMapper, CuratorFramework client, GrokService grokService) { + this.objectMapper = objectMapper; + this.client = client; + this.grokService = grokService; + } + + private Map availableParsers; + + @Override + public SensorParserConfig save(SensorParserConfig sensorParserConfig) throws RestException { + try { + ConfigurationsUtils.writeSensorParserConfigToZookeeper(sensorParserConfig.getSensorTopic(), objectMapper.writeValueAsString(sensorParserConfig).getBytes(), client); + } catch (Exception e) { + throw new RestException(e); + } + return sensorParserConfig; + } + + @Override + public SensorParserConfig findOne(String name) throws RestException { + SensorParserConfig sensorParserConfig; + try { + sensorParserConfig = ConfigurationsUtils.readSensorParserConfigFromZookeeper(name, client); + } catch (KeeperException.NoNodeException e) { + return null; + } catch (Exception e) { + throw new RestException(e); + } + return sensorParserConfig; + } + + @Override + public Iterable getAll() throws RestException { + List sensorParserConfigs = new ArrayList<>(); + List sensorNames = getAllTypes(); + for (String name : sensorNames) { + sensorParserConfigs.add(findOne(name)); + } + return sensorParserConfigs; + } + + @Override + public boolean delete(String name) throws RestException { + try { + client.delete().forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/" + name); + } catch (KeeperException.NoNodeException e) { + return false; + } catch (Exception e) { + throw new RestException(e); + } + return true; + } + + @Override + public List getAllTypes() throws RestException { + List types; + try { + types = client.getChildren().forPath(ConfigurationType.PARSER.getZookeeperRoot()); + } catch (KeeperException.NoNodeException e) { + types = new ArrayList<>(); + } catch (Exception e) { + throw new RestException(e); + } + return types; + } + + @Override + public Map getAvailableParsers() { + if (availableParsers == null) { + availableParsers = new HashMap<>(); + Set> parserClasses = getParserClasses(); + parserClasses.forEach(parserClass -> { + if (!"BasicParser".equals(parserClass.getSimpleName())) { + availableParsers.put(parserClass.getSimpleName().replaceAll("Basic|Parser", ""), parserClass.getName()); + } + }); + } + return availableParsers; + } + + @Override + public Map reloadAvailableParsers() { + availableParsers = null; + return getAvailableParsers(); + } + + private Set> getParserClasses() { + Reflections reflections = new Reflections("org.apache.metron.parsers"); + return reflections.getSubTypesOf(MessageParser.class); + } + + @Override + public JSONObject parseMessage(ParseMessageRequest parseMessageRequest) throws RestException { + SensorParserConfig sensorParserConfig = parseMessageRequest.getSensorParserConfig(); + if (sensorParserConfig == null) { + throw new RestException("SensorParserConfig is missing from ParseMessageRequest"); + } else if (sensorParserConfig.getParserClassName() == null) { + throw new RestException("SensorParserConfig must have a parserClassName"); + } else { + MessageParser parser; + try { + parser = (MessageParser) Class.forName(sensorParserConfig.getParserClassName()).newInstance(); + } catch (Exception e) { + throw new RestException(e.toString(), e.getCause()); + } + File temporaryGrokFile = null; + if (isGrokConfig(sensorParserConfig)) { + temporaryGrokFile = grokService.saveTemporary(parseMessageRequest.getGrokStatement(), parseMessageRequest.getSensorParserConfig().getSensorTopic()); + sensorParserConfig.getParserConfig().put(MetronRestConstants.GROK_PATH_KEY, temporaryGrokFile.toString()); + } + parser.configure(sensorParserConfig.getParserConfig()); + JSONObject results = parser.parse(parseMessageRequest.getSampleData().getBytes()).get(0); + if (isGrokConfig(sensorParserConfig) && temporaryGrokFile != null) { + temporaryGrokFile.delete(); + } + return results; + } + } + + private boolean isGrokConfig(SensorParserConfig sensorParserConfig) { + return GROK_CLASS_NAME.equals(sensorParserConfig.getParserClassName()); + } +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java new file mode 100644 index 0000000000..f5392bee2b --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StellarServiceImpl.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.metron.common.dsl.Context; +import org.apache.metron.common.dsl.ParseException; +import org.apache.metron.common.dsl.StellarFunctionInfo; +import org.apache.metron.common.dsl.functions.resolver.SingletonFunctionResolver; +import org.apache.metron.common.field.transformation.FieldTransformations; +import org.apache.metron.common.stellar.StellarProcessor; +import org.apache.metron.rest.model.StellarFunctionDescription; +import org.apache.metron.rest.model.SensorParserContext; +import org.apache.metron.rest.service.StellarService; +import org.json.simple.JSONObject; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Service +public class StellarServiceImpl implements StellarService { + + @Override + public Map validateRules(List rules) { + Map results = new HashMap<>(); + StellarProcessor stellarProcessor = new StellarProcessor(); + for(String rule: rules) { + try { + boolean result = stellarProcessor.validate(rule, Context.EMPTY_CONTEXT()); + results.put(rule, result); + } catch (ParseException e) { + results.put(rule, false); + } + } + return results; + } + + @Override + public Map applyTransformations(SensorParserContext sensorParserContext) { + JSONObject sampleJson = new JSONObject(sensorParserContext.getSampleData()); + sensorParserContext.getSensorParserConfig().getFieldTransformations().forEach(fieldTransformer -> { + fieldTransformer.transformAndUpdate(sampleJson, sensorParserContext.getSensorParserConfig().getParserConfig(), Context.EMPTY_CONTEXT()); + } + ); + return sampleJson; + } + + @Override + public FieldTransformations[] getTransformations() { + return FieldTransformations.values(); + } + + @Override + public List getStellarFunctions() { + List stellarFunctionDescriptions = new ArrayList<>(); + Iterable stellarFunctionsInfo = SingletonFunctionResolver.getInstance().getFunctionInfo(); + stellarFunctionsInfo.forEach(stellarFunctionInfo -> { + stellarFunctionDescriptions.add(new StellarFunctionDescription( + stellarFunctionInfo.getName(), + stellarFunctionInfo.getDescription(), + stellarFunctionInfo.getParams(), + stellarFunctionInfo.getReturns())); + }); + return stellarFunctionDescriptions; + } + + @Override + public List getSimpleStellarFunctions() { + List stellarFunctionDescriptions = getStellarFunctions(); + return stellarFunctionDescriptions.stream().filter(stellarFunctionDescription -> + stellarFunctionDescription.getParams().length == 1).sorted((o1, o2) -> o1.getName().compareTo(o2.getName())).collect(Collectors.toList()); + } + +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java new file mode 100644 index 0000000000..9bd368ff4c --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.TopologyResponse; +import org.apache.metron.rest.model.TopologyStatusCode; +import org.apache.metron.rest.service.GlobalConfigService; +import org.apache.metron.rest.service.SensorParserConfigService; +import org.apache.metron.rest.service.StormAdminService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Map; + +@Service +public class StormAdminServiceImpl implements StormAdminService { + + private StormCLIWrapper stormCLIClientWrapper; + + private GlobalConfigService globalConfigService; + + private SensorParserConfigService sensorParserConfigService; + + @Autowired + public StormAdminServiceImpl(StormCLIWrapper stormCLIClientWrapper, GlobalConfigService globalConfigService, SensorParserConfigService sensorParserConfigService) { + this.stormCLIClientWrapper = stormCLIClientWrapper; + this.globalConfigService = globalConfigService; + this.sensorParserConfigService = sensorParserConfigService; + } + + + @Override + public TopologyResponse startParserTopology(String name) throws RestException { + TopologyResponse topologyResponse = new TopologyResponse(); + if (globalConfigService.get() == null) { + topologyResponse.setErrorMessage(TopologyStatusCode.GLOBAL_CONFIG_MISSING.toString()); + } else if (sensorParserConfigService.findOne(name) == null) { + topologyResponse.setErrorMessage(TopologyStatusCode.SENSOR_PARSER_CONFIG_MISSING.toString()); + } else { + topologyResponse = createResponse(stormCLIClientWrapper.startParserTopology(name), TopologyStatusCode.STARTED, TopologyStatusCode.START_ERROR); + } + return topologyResponse; + } + + @Override + public TopologyResponse stopParserTopology(String name, boolean stopNow) throws RestException { + return createResponse(stormCLIClientWrapper.stopParserTopology(name, stopNow), TopologyStatusCode.STOPPED, TopologyStatusCode.STOP_ERROR); + } + + @Override + public TopologyResponse startEnrichmentTopology() throws RestException { + return createResponse(stormCLIClientWrapper.startEnrichmentTopology(), TopologyStatusCode.STARTED, TopologyStatusCode.START_ERROR); + } + + @Override + public TopologyResponse stopEnrichmentTopology(boolean stopNow) throws RestException { + return createResponse(stormCLIClientWrapper.stopEnrichmentTopology(stopNow), TopologyStatusCode.STOPPED, TopologyStatusCode.STOP_ERROR); + } + + @Override + public TopologyResponse startIndexingTopology() throws RestException { + return createResponse(stormCLIClientWrapper.startIndexingTopology(), TopologyStatusCode.STARTED, TopologyStatusCode.START_ERROR); + } + + @Override + public TopologyResponse stopIndexingTopology(boolean stopNow) throws RestException { + return createResponse(stormCLIClientWrapper.stopIndexingTopology(stopNow), TopologyStatusCode.STOPPED, TopologyStatusCode.STOP_ERROR); + } + + private TopologyResponse createResponse(int responseCode, TopologyStatusCode successMessage, TopologyStatusCode errorMessage) { + TopologyResponse topologyResponse = new TopologyResponse(); + if (responseCode == 0) { + topologyResponse.setSuccessMessage(successMessage.toString()); + } else { + topologyResponse.setErrorMessage(errorMessage.toString()); + } + return topologyResponse; + } + + @Override + public Map getStormClientStatus() throws RestException { + return stormCLIClientWrapper.getStormClientStatus(); + } + +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java new file mode 100644 index 0000000000..c5569c59c1 --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.metron.rest.MetronRestConstants; +import org.apache.metron.rest.RestException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.stream.Collectors.toList; +import static org.apache.metron.rest.MetronRestConstants.ENRICHMENT_TOPOLOGY_NAME; +import static org.apache.metron.rest.MetronRestConstants.INDEXING_TOPOLOGY_NAME; + +public class StormCLIWrapper { + + private Environment environment; + + @Autowired + public void setEnvironment(final Environment environment) { + this.environment = environment; + } + + public int startParserTopology(String name) throws RestException { + return runCommand(getParserStartCommand(name)); + } + + public int stopParserTopology(String name, boolean stopNow) throws RestException { + return runCommand(getStopCommand(name, stopNow)); + } + + public int startEnrichmentTopology() throws RestException { + return runCommand(getEnrichmentStartCommand()); + } + + public int stopEnrichmentTopology(boolean stopNow) throws RestException { + return runCommand(getStopCommand(ENRICHMENT_TOPOLOGY_NAME, stopNow)); + } + + public int startIndexingTopology() throws RestException { + return runCommand(getIndexingStartCommand()); + } + + public int stopIndexingTopology(boolean stopNow) throws RestException { + return runCommand(getStopCommand(INDEXING_TOPOLOGY_NAME, stopNow)); + } + + protected int runCommand(String[] command) throws RestException { + ProcessBuilder pb = getProcessBuilder(command); + pb.inheritIO(); + Process process = null; + try { + process = pb.start(); + process.waitFor(); + } catch (Exception e) { + throw new RestException(e); + } + return process.exitValue(); + } + + protected String[] getParserStartCommand(String name) { + String[] command = new String[7]; + command[0] = environment.getProperty(MetronRestConstants.PARSER_SCRIPT_PATH_SPRING_PROPERTY); + command[1] = "-k"; + command[2] = environment.getProperty(MetronRestConstants.KAFKA_BROKER_URL_SPRING_PROPERTY); + command[3] = "-z"; + command[4] = environment.getProperty(MetronRestConstants.ZK_URL_SPRING_PROPERTY); + command[5] = "-s"; + command[6] = name; + return command; + } + + protected String[] getEnrichmentStartCommand() { + String[] command = new String[1]; + command[0] = environment.getProperty(MetronRestConstants.ENRICHMENT_SCRIPT_PATH_SPRING_PROPERTY); + return command; + } + + protected String[] getIndexingStartCommand() { + String[] command = new String[1]; + command[0] = environment.getProperty(MetronRestConstants.INDEXING_SCRIPT_PATH_SPRING_PROPERTY); + return command; + } + + protected String[] getStopCommand(String name, boolean stopNow) { + String[] command; + if (stopNow) { + command = new String[5]; + command[3] = "-w"; + command[4] = "0"; + } else { + command = new String[3]; + } + command[0] = "storm"; + command[1] = "kill"; + command[2] = name; + return command; + } + + protected ProcessBuilder getProcessBuilder(String... command) { + return new ProcessBuilder(command); + } + + public Map getStormClientStatus() throws RestException { + Map status = new HashMap<>(); + status.put("parserScriptPath", environment.getProperty(MetronRestConstants.PARSER_SCRIPT_PATH_SPRING_PROPERTY)); + status.put("enrichmentScriptPath", environment.getProperty(MetronRestConstants.ENRICHMENT_SCRIPT_PATH_SPRING_PROPERTY)); + status.put("indexingScriptPath", environment.getProperty(MetronRestConstants.INDEXING_SCRIPT_PATH_SPRING_PROPERTY)); + status.put("stormClientVersionInstalled", stormClientVersionInstalled()); + return status; + } + + protected String stormClientVersionInstalled() throws RestException { + String stormClientVersionInstalled = "Storm client is not installed"; + ProcessBuilder pb = getProcessBuilder("storm", "version"); + pb.redirectErrorStream(true); + Process p; + try { + p = pb.start(); + } catch (IOException e) { + throw new RestException(e); + } + BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream())); + List lines = reader.lines().collect(toList()); + lines.forEach(System.out::println); + if (lines.size() > 1) { + stormClientVersionInstalled = lines.get(1).replaceFirst("Storm ", ""); + } + return stormClientVersionInstalled; + } + + +} diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java new file mode 100644 index 0000000000..a02ee755ca --- /dev/null +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormStatusServiceImpl.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.metron.rest.model.TopologyResponse; +import org.apache.metron.rest.model.TopologyStatus; +import org.apache.metron.rest.model.TopologyStatusCode; +import org.apache.metron.rest.model.TopologySummary; +import org.apache.metron.rest.service.StormStatusService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.apache.metron.rest.MetronRestConstants.STORM_UI_SPRING_PROPERTY; +import static org.apache.metron.rest.MetronRestConstants.TOPOLOGY_SUMMARY_URL; +import static org.apache.metron.rest.MetronRestConstants.TOPOLOGY_URL; + +@Service +public class StormStatusServiceImpl implements StormStatusService { + + private Environment environment; + + private RestTemplate restTemplate; + + @Autowired + public StormStatusServiceImpl(Environment environment, RestTemplate restTemplate) { + this.environment = environment; + this.restTemplate = restTemplate; + } + + @Override + public TopologySummary getTopologySummary() { + return restTemplate.getForObject("http://" + environment.getProperty(STORM_UI_SPRING_PROPERTY) + TOPOLOGY_SUMMARY_URL, TopologySummary.class); + } + + @Override + public TopologyStatus getTopologyStatus(String name) { + TopologyStatus topologyResponse = null; + String id = null; + for (TopologyStatus topology : getTopologySummary().getTopologies()) { + if (name.equals(topology.getName())) { + id = topology.getId(); + break; + } + } + if (id != null) { + topologyResponse = restTemplate.getForObject("http://" + environment.getProperty(STORM_UI_SPRING_PROPERTY) + TOPOLOGY_URL + "/" + id, TopologyStatus.class); + } + return topologyResponse; + } + + @Override + public List getAllTopologyStatus() { + List topologyStatus = new ArrayList<>(); + for (TopologyStatus topology : getTopologySummary().getTopologies()) { + topologyStatus.add(restTemplate.getForObject("http://" + environment.getProperty(STORM_UI_SPRING_PROPERTY) + TOPOLOGY_URL + "/" + topology.getId(), TopologyStatus.class)); + } + return topologyStatus; + } + + @Override + public TopologyResponse activateTopology(String name) { + TopologyResponse topologyResponse = new TopologyResponse(); + String id = null; + for (TopologyStatus topology : getTopologySummary().getTopologies()) { + if (name.equals(topology.getName())) { + id = topology.getId(); + break; + } + } + if (id != null) { + Map result = restTemplate.postForObject("http://" + environment.getProperty(STORM_UI_SPRING_PROPERTY) + TOPOLOGY_URL + "/" + id + "/activate", null, Map.class); + if("success".equals(result.get("status"))) { + topologyResponse.setSuccessMessage(TopologyStatusCode.ACTIVE.toString()); + } else { + topologyResponse.setErrorMessage((String) result.get("status")); + } + } else { + topologyResponse.setErrorMessage(TopologyStatusCode.TOPOLOGY_NOT_FOUND.toString()); + } + return topologyResponse; + } + + @Override + public TopologyResponse deactivateTopology(String name) { + TopologyResponse topologyResponse = new TopologyResponse(); + String id = null; + for (TopologyStatus topology : getTopologySummary().getTopologies()) { + if (name.equals(topology.getName())) { + id = topology.getId(); + break; + } + } + if (id != null) { + Map result = restTemplate.postForObject("http://" + environment.getProperty(STORM_UI_SPRING_PROPERTY) + TOPOLOGY_URL + "/" + id + "/deactivate", null, Map.class); + if("success".equals(result.get("status"))) { + topologyResponse.setSuccessMessage(TopologyStatusCode.INACTIVE.toString()); + } else { + topologyResponse.setErrorMessage((String) result.get("status")); + } + } else { + topologyResponse.setErrorMessage(TopologyStatusCode.TOPOLOGY_NOT_FOUND.toString()); + } + return topologyResponse; + } +} diff --git a/metron-interface/metron-rest/src/main/resources/META-INF/LICENSE b/metron-interface/metron-rest/src/main/resources/META-INF/LICENSE new file mode 100644 index 0000000000..c9d4b53cad --- /dev/null +++ b/metron-interface/metron-rest/src/main/resources/META-INF/LICENSE @@ -0,0 +1,692 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------------ +------------------------------------------------------------------------------------ + DOM4J +------------------------------------------------------------------------------------ + +This product bundles dom4j 1.6.1. For details, see https://github.com/dom4j/dom4j. + +Copyright 2001-2016 (C) MetaStuff, Ltd. and DOM4J contributors. All Rights Reserved. + +Redistribution and use of this software and associated documentation +("Software"), with or without modification, are permitted provided +that the following conditions are met: + +1. Redistributions of source code must retain copyright + statements and notices. Redistributions must also contain a + copy of this document. + +2. Redistributions in binary form must reproduce the + above copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + +3. The name "DOM4J" must not be used to endorse or promote + products derived from this Software without prior written + permission of MetaStuff, Ltd. For written permission, + please contact dom4j-info@metastuff.com. + +4. Products derived from this Software may not be called "DOM4J" + nor may "DOM4J" appear in their names without prior written + permission of MetaStuff, Ltd. DOM4J is a registered + trademark of MetaStuff, Ltd. + +5. Due credit should be given to the DOM4J Project - https://dom4j.github.io/ + +THIS SOFTWARE IS PROVIDED BY METASTUFF, LTD. AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT +NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +METASTUFF, LTD. OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------------ + EPL +------------------------------------------------------------------------------------ + +This product bundles aspectjweaver 1.8.9, which is available under a "Eclipse Public License - v 1.0" license. For details, see https://eclipse.org/aspectj. + +Eclipse Public License - v 1.0 + +THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT’S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + +a) in the case of the initial Contributor, the initial code and documentation distributed under this Agreement, and +b) in the case of each subsequent Contributor: + +i)changes to the Program, and + +ii)additions to the Program; + +where such changes and/or additions to the Program originate from and are distributed by that particular Contributor. A Contribution 'originates' from a Contributor if it was added to the Program by such Contributor itself or anyone acting on such Contributor’s behalf. Contributions do not include additions to the Program which: (i) are separate modules of software distributed in conjunction with the Program under their own license agreement, and (ii) are not derivative works of the Program. +"Contributor" means any person or entity that distributes the Program. + +"Licensed Patents " mean patent claims licensable by a Contributor which are necessarily infringed by the use or sale of its Contribution alone or when combined with the Program. + +"Program" means the Contributions distributed in accordance with this Agreement. + +"Recipient" means anyone who receives the Program under this Agreement, including all Contributors. + +2. GRANT OF RIGHTS + +a) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, distribute and sublicense the Contribution of such Contributor, if any, and such derivative works, in source code and object code form. + +b) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed Patents to make, use, sell, offer to sell, import and otherwise transfer the Contribution of such Contributor, if any, in source code and object code form. This patent license shall apply to the combination of the Contribution and the Program if, at the time the Contribution is added by the Contributor, such addition of the Contribution causes such combination to be covered by the Licensed Patents. The patent license shall not apply to any other combinations which include the Contribution. No hardware per se is licensed hereunder. + +c) Recipient understands that although each Contributor grants the licenses to its Contributions set forth herein, no assurances are provided by any Contributor that the Program does not infringe the patent or other intellectual property rights of any other entity. Each Contributor disclaims any liability to Recipient for claims brought by any other entity based on infringement of intellectual property rights or otherwise. As a condition to exercising the rights and licenses granted hereunder, each Recipient hereby assumes sole responsibility to secure any other intellectual property rights needed, if any. For example, if a third party patent license is required to allow Recipient to distribute the Program, it is Recipient’s responsibility to acquire that license before distributing the Program. + +d) Each Contributor represents that to its knowledge it has sufficient copyright rights in its Contribution, if any, to grant the copyright license set forth in this Agreement. + +3. REQUIREMENTS + +A Contributor may choose to distribute the Program in object code form under its own license agreement, provided that: + +a) it complies with the terms and conditions of this Agreement; and + +b) its license agreement: + +i) effectively disclaims on behalf of all Contributors all warranties and conditions, express and implied, including warranties or conditions of title and non-infringement, and implied warranties or conditions of merchantability and fitness for a particular purpose; + +ii) effectively excludes on behalf of all Contributors all liability for damages, including direct, indirect, special, incidental and consequential damages, such as lost profits; + +iii) states that any provisions which differ from this Agreement are offered by that Contributor alone and not by any other party; and + +iv) states that source code for the Program is available from such Contributor, and informs licensees how to obtain it in a reasonable manner on or through a medium customarily used for software exchange. +When the Program is made available in source code form: + +a) it must be made available under this Agreement; and + +b) a copy of this Agreement must be included with each copy of the Program. +Contributors may not remove or alter any copyright notices contained within the Program. + +Each Contributor must identify itself as the originator of its Contribution, if any, in a manner that reasonably allows subsequent Recipients to identify the originator of the Contribution. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities with respect to end users, business partners and the like. While this license is intended to facilitate the commercial use of the Program, the Contributor who includes the Program in a commercial product offering should do so in a manner which does not create potential liability for other Contributors. Therefore, if a Contributor includes the Program in a commercial product offering, such Contributor ("Commercial Contributor") hereby agrees to defend and indemnify every other Contributor ("Indemnified Contributor") against any losses, damages and costs (collectively "Losses") arising from claims, lawsuits and other legal actions brought by a third party against the Indemnified Contributor to the extent caused by the acts or omissions of such Commercial Contributor in connection with its distribution of the Program in a commercial product offering. The obligations in this section do not apply to any claims or Losses relating to any actual or alleged intellectual property infringement. In order to qualify, an Indemnified Contributor must: a) promptly notify the Commercial Contributor in writing of such claim, and b) allow the Commercial Contributor to control, and cooperate with the Commercial Contributor in, the defense and any related settlement negotiations. The Indemnified Contributor may participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial product offering, Product X. That Contributor is then a Commercial Contributor. If that Commercial Contributor then makes performance claims, or offers warranties related to Product X, those performance claims and warranties are such Commercial Contributor’s responsibility alone. Under this section, the Commercial Contributor would have to defend claims against the other Contributors related to those performance claims and warranties, and if a court requires any other Contributor to pay any damages as a result, the Commercial Contributor must pay those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the appropriateness of using and distributing the Program and assumes all risks associated with its exercise of rights under this Agreement , including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this Agreement, and without further action by the parties hereto, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Program itself (excluding combinations of the Program with other software or hardware) infringes such Recipient’s patent(s), then such Recipient’s rights granted under Section 2(b) shall terminate as of the date such litigation is filed. + +All Recipient’s rights under this Agreement shall terminate if it fails to comply with any of the material terms or conditions of this Agreement and does not cure such failure in a reasonable period of time after becoming aware of such noncompliance. If all Recipient’s rights under this Agreement terminate, Recipient agrees to cease use and distribution of the Program as soon as reasonably practicable. However, Recipient’s obligations under this Agreement and any licenses granted by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, but in order to avoid inconsistency the Agreement is copyrighted and may only be modified in the following manner. The Agreement Steward reserves the right to publish new versions (including revisions) of this Agreement from time to time. No one other than the Agreement Steward has the right to modify this Agreement. The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation may assign the responsibility to serve as the Agreement Steward to a suitable separate entity. Each new version of the Agreement will be given a distinguishing version number. The Program (including Contributions) may always be distributed subject to the version of the Agreement under which it was received. In addition, after a new version of the Agreement is published, Contributor may elect to distribute the Program (including its Contributions) under the new version. Except as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no rights or licenses to the intellectual property of any Contributor under this Agreement, whether expressly, by implication, estoppel or otherwise. All rights in the Program not expressly granted under this Agreement are reserved. + +This Agreement is governed by the laws of the State of New York and the intellectual property laws of the United States of America. No party to this Agreement will bring a legal action under this Agreement more than one year after the cause of action arose. Each party waives its rights to a jury trial in any resulting litigation. + +------------------------------------------------------------------------------------ + BSD +------------------------------------------------------------------------------------ + +This product bundles antlr 2.7.7, which is available under a "BSD Software License" license. For details, see http://www.antlr2.org + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------------ + CDDL v1.0 +------------------------------------------------------------------------------------ + +This product bundles javax.transaction-api 1.2, which is available under a "Common Development and Distribution License v1.0" license. For details, see https://java.net/projects/jta-spec/ + +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 + 1. Definitions. + 1.1. "Contributor" means each individual or entity that + creates or contributes to the creation of Modifications. + 1.2. "Contributor Version" means the combination of the + Original Software, prior Modifications used by a + Contributor (if any), and the Modifications made by that + particular Contributor. + 1.3. "Covered Software" means (a) the Original Software, or + (b) Modifications, or (c) the combination of files + containing Original Software with files containing + Modifications, in each case including portions thereof. + 1.4. "Executable" means the Covered Software in any form + other than Source Code. + 1.5. "Initial Developer" means the individual or entity + that first makes Original Software available under this + License. + 1.6. "Larger Work" means a work which combines Covered + Software or portions thereof with code not governed by the + terms of this License. + 1.7. "License" means this document. + 1.8. "Licensable" means having the right to grant, to the + maximum extent possible, whether at the time of the initial + grant or subsequently acquired, any and all of the rights + conveyed herein. + 1.9. "Modifications" means the Source Code and Executable + form of any of the following: + A. Any file that results from an addition to, + deletion from or modification of the contents of a + file containing Original Software or previous + Modifications; + B. Any new file that contains any part of the + Original Software or previous Modification; or + C. Any new file that is contributed or otherwise made + available under the terms of this License. + 1.10. "Original Software" means the Source Code and + Executable form of computer software code that is + originally released under this License. + 1.11. "Patent Claims" means any patent claim(s), now owned + or hereafter acquired, including without limitation, + method, process, and apparatus claims, in any patent + Licensable by grantor. + 1.12. "Source Code" means (a) the common form of computer + software code in which modifications are made and (b) + associated documentation included in or with such code. + 1.13. "You" (or "Your") means an individual or a legal + entity exercising rights under, and complying with all of + the terms of, this License. For legal entities, "You" + includes any entity which controls, is controlled by, or is + under common control with You. For purposes of this + definition, "control" means (a) the power, direct or + indirect, to cause the direction or management of such + entity, whether by contract or otherwise, or (b) ownership + of more than fifty percent (50%) of the outstanding shares + or beneficial ownership of such entity. + 2. License Grants. + 2.1. The Initial Developer Grant. + Conditioned upon Your compliance with Section 3.1 below and + subject to third party intellectual property claims, the + Initial Developer hereby grants You a world-wide, + royalty-free, non-exclusive license: + (a) under intellectual property rights (other than + patent or trademark) Licensable by Initial Developer, + to use, reproduce, modify, display, perform, + sublicense and distribute the Original Software (or + portions thereof), with or without Modifications, + and/or as part of a Larger Work; and + (b) under Patent Claims infringed by the making, + using or selling of Original Software, to make, have + made, use, practice, sell, and offer for sale, and/or + otherwise dispose of the Original Software (or + portions thereof). + (c) The licenses granted in Sections 2.1(a) and (b) + are effective on the date Initial Developer first + distributes or otherwise makes the Original Software + available to a third party under the terms of this + License. + (d) Notwithstanding Section 2.1(b) above, no patent + license is granted: (1) for code that You delete from + the Original Software, or (2) for infringements + caused by: (i) the modification of the Original + Software, or (ii) the combination of the Original + Software with other software or devices. + 2.2. Contributor Grant. + Conditioned upon Your compliance with Section 3.1 below and + subject to third party intellectual property claims, each + Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + (a) under intellectual property rights (other than + patent or trademark) Licensable by Contributor to + use, reproduce, modify, display, perform, sublicense + and distribute the Modifications created by such + Contributor (or portions thereof), either on an + unmodified basis, with other Modifications, as + Covered Software and/or as part of a Larger Work; and + (b) under Patent Claims infringed by the making, + using, or selling of Modifications made by that + Contributor either alone and/or in combination with + its Contributor Version (or portions of such + combination), to make, use, sell, offer for sale, + have made, and/or otherwise dispose of: (1) + Modifications made by that Contributor (or portions + thereof); and (2) the combination of Modifications + made by that Contributor with its Contributor Version + (or portions of such combination). + (c) The licenses granted in Sections 2.2(a) and + 2.2(b) are effective on the date Contributor first + distributes or otherwise makes the Modifications + available to a third party. + (d) Notwithstanding Section 2.2(b) above, no patent + license is granted: (1) for any code that Contributor + has deleted from the Contributor Version; (2) for + infringements caused by: (i) third party + modifications of Contributor Version, or (ii) the + combination of Modifications made by that Contributor + with other software (except as part of the + Contributor Version) or other devices; or (3) under + Patent Claims infringed by Covered Software in the + absence of Modifications made by that Contributor. + 3. Distribution Obligations. + 3.1. Availability of Source Code. + Any Covered Software that You distribute or otherwise make + available in Executable form must also be made available in + Source Code form and that Source Code form must be + distributed only under the terms of this License. You must + include a copy of this License with every copy of the + Source Code form of the Covered Software You distribute or + otherwise make available. You must inform recipients of any + such Covered Software in Executable form as to how they can + obtain such Covered Software in Source Code form in a + reasonable manner on or through a medium customarily used + for software exchange. + 3.2. Modifications. + The Modifications that You create or to which You + contribute are governed by the terms of this License. You + represent that You believe Your Modifications are Your + original creation(s) and/or You have sufficient rights to + grant the rights conveyed by this License. + 3.3. Required Notices. + You must include a notice in each of Your Modifications + that identifies You as the Contributor of the Modification. + You may not remove or alter any copyright, patent or + trademark notices contained within the Covered Software, or + any notices of licensing or any descriptive text giving + attribution to any Contributor or the Initial Developer. + 3.4. Application of Additional Terms. + You may not offer or impose any terms on any Covered + Software in Source Code form that alters or restricts the + applicable version of this License or the recipients' + rights hereunder. You may choose to offer, and to charge a + fee for, warranty, support, indemnity or liability + obligations to one or more recipients of Covered Software. + However, you may do so only on Your own behalf, and not on + behalf of the Initial Developer or any Contributor. You + must make it absolutely clear that any such warranty, + support, indemnity or liability obligation is offered by + You alone, and You hereby agree to indemnify the Initial + Developer and every Contributor for any liability incurred + by the Initial Developer or such Contributor as a result of + warranty, support, indemnity or liability terms You offer. + 3.5. Distribution of Executable Versions. + You may distribute the Executable form of the Covered + Software under the terms of this License or under the terms + of a license of Your choice, which may contain terms + different from this License, provided that You are in + compliance with the terms of this License and that the + license for the Executable form does not attempt to limit + or alter the recipient's rights in the Source Code form + from the rights set forth in this License. If You + distribute the Covered Software in Executable form under a + different license, You must make it absolutely clear that + any terms which differ from this License are offered by You + alone, not by the Initial Developer or Contributor. You + hereby agree to indemnify the Initial Developer and every + Contributor for any liability incurred by the Initial + Developer or such Contributor as a result of any such terms + You offer. + 3.6. Larger Works. + You may create a Larger Work by combining Covered Software + with other code not governed by the terms of this License + and distribute the Larger Work as a single product. In such + a case, You must make sure the requirements of this License + are fulfilled for the Covered Software. + 4. Versions of the License. + 4.1. New Versions. + Sun Microsystems, Inc. is the initial license steward and + may publish revised and/or new versions of this License + from time to time. Each version will be given a + distinguishing version number. Except as provided in + Section 4.3, no one other than the license steward has the + right to modify this License. + 4.2. Effect of New Versions. + You may always continue to use, distribute or otherwise + make the Covered Software available under the terms of the + version of the License under which You originally received + the Covered Software. If the Initial Developer includes a + notice in the Original Software prohibiting it from being + distributed or otherwise made available under any + subsequent version of the License, You must distribute and + make the Covered Software available under the terms of the + version of the License under which You originally received + the Covered Software. Otherwise, You may also choose to + use, distribute or otherwise make the Covered Software + available under the terms of any subsequent version of the + License published by the license steward. + 4.3. Modified Versions. + When You are an Initial Developer and You want to create a + new license for Your Original Software, You may create and + use a modified version of this License if You: (a) rename + the license and remove any references to the name of the + license steward (except to note that the license differs + from this License); and (b) otherwise make it clear that + the license contains terms which differ from this License. + 5. DISCLAIMER OF WARRANTY. + COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" + BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, + INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED + SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR + PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND + PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY + COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE + INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF + ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF + WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF + ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS + DISCLAIMER. + 6. TERMINATION. + 6.1. This License and the rights granted hereunder will + terminate automatically if You fail to comply with terms + herein and fail to cure such breach within 30 days of + becoming aware of the breach. Provisions which, by their + nature, must remain in effect beyond the termination of + this License shall survive. + 6.2. If You assert a patent infringement claim (excluding + declaratory judgment actions) against Initial Developer or + a Contributor (the Initial Developer or Contributor against + whom You assert such claim is referred to as "Participant") + alleging that the Participant Software (meaning the + Contributor Version where the Participant is a Contributor + or the Original Software where the Participant is the + Initial Developer) directly or indirectly infringes any + patent, then any and all rights granted directly or + indirectly to You by such Participant, the Initial + Developer (if the Initial Developer is not the Participant) + and all Contributors under Sections 2.1 and/or 2.2 of this + License shall, upon 60 days notice from Participant + terminate prospectively and automatically at the expiration + of such 60 day notice period, unless if within such 60 day + period You withdraw Your claim with respect to the + Participant Software against such Participant either + unilaterally or pursuant to a written agreement with + Participant. + 6.3. In the event of termination under Sections 6.1 or 6.2 + above, all end user licenses that have been validly granted + by You or any distributor hereunder prior to termination + (excluding licenses granted to You by any distributor) + shall survive termination. + 7. LIMITATION OF LIABILITY. + UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT + (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE + INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF + COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE + LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR + CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT + LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF GOODWILL, WORK + STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER + COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN + INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF + LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL + INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT + APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO + NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR + CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT + APPLY TO YOU. + 8. U.S. GOVERNMENT END USERS. + The Covered Software is a "commercial item," as that term is + defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial + computer software" (as that term is defined at 48 C.F.R. ¤ + 252.227-7014(a)(1)) and "commercial computer software + documentation" as such terms are used in 48 C.F.R. 12.212 (Sept. + 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 + through 227.7202-4 (June 1995), all U.S. Government End Users + acquire Covered Software with only those rights set forth herein. + This U.S. Government Rights clause is in lieu of, and supersedes, + any other FAR, DFAR, or other clause or provision that addresses + Government rights in computer software under this License. + 9. MISCELLANEOUS. + This License represents the complete agreement concerning subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the + extent necessary to make it enforceable. This License shall be + governed by the law of the jurisdiction specified in a notice + contained within the Original Software (except to the extent + applicable law, if any, provides otherwise), excluding such + jurisdiction's conflict-of-law provisions. Any litigation + relating to this License shall be subject to the jurisdiction of + the courts located in the jurisdiction and venue specified in a + notice contained within the Original Software, with the losing + party responsible for costs, including, without limitation, court + costs and reasonable attorneys' fees and expenses. The + application of the United Nations Convention on Contracts for the + International Sale of Goods is expressly excluded. Any law or + regulation which provides that the language of a contract shall + be construed against the drafter shall not apply to this License. + You agree that You alone are responsible for compliance with the + United States export administration regulations (and the export + control laws and regulation of any other countries) when You use, + distribute or otherwise make available any Covered Software. + 10. RESPONSIBILITY FOR CLAIMS. + As between Initial Developer and the Contributors, each party is + responsible for claims and damages arising, directly or + indirectly, out of its utilization of rights under this License + and You agree to work with Initial Developer and Contributors to + distribute such responsibility on an equitable basis. Nothing + herein is intended or shall be deemed to constitute any admission + of liability. + +------------------------------------------------------------------------------------ + MIT +------------------------------------------------------------------------------------ + +This product bundles slf4j-api 1.7.21, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org +This product bundles jcl-over-slf4j 1.7.21, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org +This product bundles jul-to-slf4j 1.7.21, which is available under a "MIT Software License" license. For details, see http://www.slf4j.org +This product bundles lodash, which is available under a "MIT Software License" license. For details, see https://lodash.com/. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/metron-interface/metron-rest/src/main/resources/META-INF/NOTICE b/metron-interface/metron-rest/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000..aba445de14 --- /dev/null +++ b/metron-interface/metron-rest/src/main/resources/META-INF/NOTICE @@ -0,0 +1,6 @@ + +metron-rest +Copyright 2006-2016 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/metron-interface/metron-rest/src/main/resources/application-docker.yml b/metron-interface/metron-rest/src/main/resources/application-docker.yml new file mode 100644 index 0000000000..5891417364 --- /dev/null +++ b/metron-interface/metron-rest/src/main/resources/application-docker.yml @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +docker: + host: + address: 192.168.99.100 + compose: + path: ../../metron-docker/compose/docker-compose.yml + +spring: + datasource: + driverClassName: org.h2.Driver + url: jdbc:h2:file:./metrondb + username: root + password: root + platform: h2 + jpa: + hibernate: + ddl-auto: update + +zookeeper: + url: ${docker.host.address}:2181 + +kafka: + broker: + url: ${docker.host.address}:9092 + +hdfs: + namenode: + url: file:/// + +grok: + path: + temp: target/patterns/temp + default: target/patterns + +storm: + ui: + url: ${docker.host.address}:8080 + parser: + script.path: /usr/metron/${metron.version}/bin/start_parser_topology.sh + enrichment: + script.path: /usr/metron/${metron.version}/bin/start_enrichment_topology.sh + indexing: + script.path: /usr/metron/${metron.version}/bin/start_elasticsearch_topology.sh diff --git a/metron-interface/metron-rest/src/main/resources/application-test.yml b/metron-interface/metron-rest/src/main/resources/application-test.yml new file mode 100644 index 0000000000..2872e95665 --- /dev/null +++ b/metron-interface/metron-rest/src/main/resources/application-test.yml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +spring: + datasource: + driverClassName: org.h2.Driver + url: jdbc:h2:mem:testdb + username: sa + password: + platform: h2 + jpa: + hibernate: + ddl-auto: create-drop + main: + banner-mode: off + +grok: + path: + temp: target/patterns/temp + default: target/patterns + +storm: + ui: + url: stormUITestUrl + parser: + script.path: /usr/metron/${metron.version}/bin/start_parser_topology.sh + enrichment: + script.path: /usr/metron/${metron.version}/bin/start_enrichment_topology.sh + indexing: + script.path: /usr/metron/${metron.version}/bin/start_elasticsearch_topology.sh \ No newline at end of file diff --git a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml new file mode 100644 index 0000000000..76bef56748 --- /dev/null +++ b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +spring: + datasource: + driverClassName: org.h2.Driver + url: jdbc:h2:file:./metrondb + username: root + password: root + platform: h2 + jpa: + hibernate: + ddl-auto: update + +zookeeper: + url: node1:2181 + +kafka: + broker: + url: node1:6667 + +hdfs: + namenode: + url: hdfs://node1:8020 + +grok: + path: + temp: ./patterns/temp + default: /apps/metron/patterns + +storm: + ui: + url: node1:8744 + parser: + script.path: /usr/metron/${metron.version}/bin/start_parser_topology.sh + enrichment: + script.path: /usr/metron/${metron.version}/bin/start_enrichment_topology.sh + indexing: + script.path: /usr/metron/${metron.version}/bin/start_elasticsearch_topology.sh diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml new file mode 100644 index 0000000000..2bb1462030 --- /dev/null +++ b/metron-interface/metron-rest/src/main/resources/application.yml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +metron: + version: 0.3.1 + +logging: + level: + root: ERROR + +spring: + jackson: + date-format: yyyy-MM-dd HH:mm:ss + +grok: + path: + temp: ./ + default: /apps/metron/patterns + +zookeeper: + client: + timeout: + session: 10000 + connection: 10000 + +curator: + sleep: + time: 1000 + max: + retries: 3 diff --git a/metron-interface/metron-rest/src/main/resources/log4j.properties b/metron-interface/metron-rest/src/main/resources/log4j.properties new file mode 100644 index 0000000000..492cecf9d7 --- /dev/null +++ b/metron-interface/metron-rest/src/main/resources/log4j.properties @@ -0,0 +1,16 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=ERROR, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd'T'HH:mm:ss.SSS} %-5p [%c] - %m%n \ No newline at end of file diff --git a/metron-interface/metron-rest/src/main/resources/schema-h2.sql b/metron-interface/metron-rest/src/main/resources/schema-h2.sql new file mode 100644 index 0000000000..986b652cf2 --- /dev/null +++ b/metron-interface/metron-rest/src/main/resources/schema-h2.sql @@ -0,0 +1,30 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +drop table if exists authorities; +drop table if exists users; + +create table if not exists users( + username varchar(50) not null primary key, + password varchar(50) not null, + enabled boolean not null +); +create table authorities ( + username varchar(50) not null, + authority varchar(50) not null, + constraint fk_authorities_users foreign key(username) references users(username) +); +create unique index ix_auth_username on authorities (username,authority); \ No newline at end of file diff --git a/metron-interface/metron-rest/src/main/resources/schema-mysql.sql b/metron-interface/metron-rest/src/main/resources/schema-mysql.sql new file mode 100644 index 0000000000..986b652cf2 --- /dev/null +++ b/metron-interface/metron-rest/src/main/resources/schema-mysql.sql @@ -0,0 +1,30 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +drop table if exists authorities; +drop table if exists users; + +create table if not exists users( + username varchar(50) not null primary key, + password varchar(50) not null, + enabled boolean not null +); +create table authorities ( + username varchar(50) not null, + authority varchar(50) not null, + constraint fk_authorities_users foreign key(username) references users(username) +); +create unique index ix_auth_username on authorities (username,authority); \ No newline at end of file diff --git a/metron-interface/metron-rest/src/main/scripts/start_metron_rest.sh b/metron-interface/metron-rest/src/main/scripts/start_metron_rest.sh new file mode 100755 index 0000000000..ea5d6c903d --- /dev/null +++ b/metron-interface/metron-rest/src/main/scripts/start_metron_rest.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +METRON_VERSION=${project.version} +SCRIPTS_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +if [ "$#" -lt 1 ]; then + echo "Usage: start.sh " + echo "Path can be absolute or relative to $SCRIPTS_ROOT" +else + java -jar $SCRIPTS_ROOT/../lib/metron-rest-$METRON_VERSION.jar --spring.config.location=$@ +fi \ No newline at end of file diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/MetronRestApplicationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/MetronRestApplicationTest.java new file mode 100644 index 0000000000..3025932094 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/MetronRestApplicationTest.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.springframework.boot.SpringApplication; + +import static org.mockito.Mockito.times; +import static org.powermock.api.mockito.PowerMockito.mockStatic; +import static org.powermock.api.mockito.PowerMockito.verifyStatic; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({SpringApplication.class}) +public class MetronRestApplicationTest { + + @Test + public void test() { + mockStatic(SpringApplication.class); + String[] args = {"arg1", "arg2"}; + MetronRestApplication.main(args); + verifyStatic(times(1)); + SpringApplication.run(MetronRestApplication.class, args); + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java new file mode 100644 index 0000000000..edfd5427e4 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.config; + +import kafka.admin.AdminUtils$; +import kafka.utils.ZKStringSerializer$; +import kafka.utils.ZkUtils; +import org.I0Itec.zkclient.ZkClient; +import org.apache.curator.RetryPolicy; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.metron.integration.ComponentRunner; +import org.apache.metron.integration.UnableToStartException; +import org.apache.metron.integration.components.KafkaComponent; +import org.apache.metron.integration.components.ZKServerComponent; +import org.apache.metron.rest.mock.MockStormCLIClientWrapper; +import org.apache.metron.rest.mock.MockStormRestTemplate; +import org.apache.metron.rest.service.impl.StormCLIWrapper; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.web.client.RestTemplate; + +import java.util.Properties; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; + +@Configuration +@Profile(TEST_PROFILE) +public class TestConfig { + + @Bean + public Properties zkProperties() { + return new Properties(); + } + + @Bean + public ZKServerComponent zkServerComponent(Properties zkProperties) { + return new ZKServerComponent() + .withPostStartCallback((zkComponent) -> zkProperties.setProperty(ZKServerComponent.ZOOKEEPER_PROPERTY, zkComponent.getConnectionString())); + } + + @Bean + public KafkaComponent kafkaWithZKComponent(Properties zkProperties) { + return new KafkaComponent().withTopologyProperties(zkProperties); + } + + //@Bean(destroyMethod = "stop") + @Bean + public ComponentRunner componentRunner(ZKServerComponent zkServerComponent, KafkaComponent kafkaWithZKComponent) { + ComponentRunner runner = new ComponentRunner.Builder() + .withComponent("zk", zkServerComponent) + .withComponent("kafka", kafkaWithZKComponent) + .withCustomShutdownOrder(new String[] {"kafka","zk"}) + .build(); + try { + runner.start(); + } catch (UnableToStartException e) { + e.printStackTrace(); + } + return runner; + } + + @Bean(initMethod = "start", destroyMethod="close") + public CuratorFramework client(ComponentRunner componentRunner) { + RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3); + ZKServerComponent zkServerComponent = componentRunner.getComponent("zk", ZKServerComponent.class); + return CuratorFrameworkFactory.newClient(zkServerComponent.getConnectionString(), retryPolicy); + } + + @Bean(destroyMethod="close") + public ZkClient zkClient(ComponentRunner componentRunner) { + ZKServerComponent zkServerComponent = componentRunner.getComponent("zk", ZKServerComponent.class); + return new ZkClient(zkServerComponent.getConnectionString(), 10000, 10000, ZKStringSerializer$.MODULE$); + } + + @Bean + public ZkUtils zkUtils(ZkClient zkClient) { + return ZkUtils.apply(zkClient, false); + } + + @Bean(destroyMethod="close") + public KafkaConsumer kafkaConsumer(KafkaComponent kafkaWithZKComponent) { + Properties props = new Properties(); + props.put("bootstrap.servers", kafkaWithZKComponent.getBrokerList()); + props.put("group.id", "metron-config"); + props.put("enable.auto.commit", "false"); + props.put("auto.commit.interval.ms", "1000"); + props.put("session.timeout.ms", "30000"); + props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + return new KafkaConsumer<>(props); + } + + @Bean + public StormCLIWrapper stormCLIClientWrapper() { + return new MockStormCLIClientWrapper(); + } + + @Bean + public RestTemplate restTemplate(StormCLIWrapper stormCLIClientWrapper) { + MockStormRestTemplate restTemplate = new MockStormRestTemplate(); + restTemplate.setMockStormCLIClientWrapper((MockStormCLIClientWrapper) stormCLIClientWrapper); + return restTemplate; + } + + @Bean + public AdminUtils$ adminUtils() { + return AdminUtils$.MODULE$; + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java new file mode 100644 index 0000000000..7796c8b2bf --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GlobalConfigControllerIntegrationTest.java @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import org.adrianwalker.multilinestring.Multiline; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(TEST_PROFILE) +public class GlobalConfigControllerIntegrationTest { + + /** + { + "solr.zookeeper": "solr:2181", + "solr.collection": "metron", + "solr.numShards": 1, + "solr.replicationFactor": 1 + } + */ + @Multiline + public static String globalJson; + + @Autowired + private WebApplicationContext wac; + + private MockMvc mockMvc; + + private String globalConfigUrl = "/api/v1/global/config"; + private String user = "user"; + private String password = "password"; + + @Before + public void setup() throws Exception { + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); + } + + @Test + public void testSecurity() throws Exception { + this.mockMvc.perform(post(globalConfigUrl).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(globalJson)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(globalConfigUrl)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(delete(globalConfigUrl).with(csrf())) + .andExpect(status().isUnauthorized()); + } + + @Test + public void test() throws Exception { + this.mockMvc.perform(get(globalConfigUrl).with(httpBasic(user,password))) + .andExpect(status().isNotFound()); + + this.mockMvc.perform(post(globalConfigUrl).with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(globalJson)) + .andExpect(status().isCreated()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))); + + this.mockMvc.perform(post(globalConfigUrl).with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(globalJson)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))); + + this.mockMvc.perform(get(globalConfigUrl).with(httpBasic(user,password))) + .andExpect(status().isOk()); + + this.mockMvc.perform(delete(globalConfigUrl).with(httpBasic(user,password)).with(csrf())) + .andExpect(status().isOk()); + + this.mockMvc.perform(delete(globalConfigUrl).with(httpBasic(user,password)).with(csrf())) + .andExpect(status().isNotFound()); + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java new file mode 100644 index 0000000000..e0a9c5baf6 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/GrokControllerIntegrationTest.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import org.adrianwalker.multilinestring.Multiline; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(TEST_PROFILE) +public class GrokControllerIntegrationTest { + + /** + { + "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html", + "patternLabel":"SQUID", + "statement":"SQUID %{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}" + } + */ + @Multiline + public static String grokValidationJson; + + /** + { + "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html", + "statement":"SQUID %{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}" + } + */ + @Multiline + public static String missingPatternLabelGrokValidationJson; + + /** + { + "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html", + "patternLabel":"SQUID" + } + */ + @Multiline + public static String missingStatementGrokValidationJson; + + @Autowired + private WebApplicationContext wac; + + private MockMvc mockMvc; + + private String grokUrl = "/api/v1/grok"; + private String user = "user"; + private String password = "password"; + + @Before + public void setup() throws Exception { + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); + } + + @Test + public void testSecurity() throws Exception { + this.mockMvc.perform(post(grokUrl + "/validate").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(grokValidationJson)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(grokUrl + "/list")) + .andExpect(status().isUnauthorized()); + } + + @Test + public void test() throws Exception { + this.mockMvc.perform(post(grokUrl + "/validate").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(grokValidationJson)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.results.action").value("TCP_MISS")) + .andExpect(jsonPath("$.results.bytes").value(337891)) + .andExpect(jsonPath("$.results.code").value(200)) + .andExpect(jsonPath("$.results.elapsed").value(415)) + .andExpect(jsonPath("$.results.ip_dst_addr").value("207.109.73.154")) + .andExpect(jsonPath("$.results.ip_src_addr").value("127.0.0.1")) + .andExpect(jsonPath("$.results.method").value("GET")) + .andExpect(jsonPath("$.results.timestamp").value("1467011157.401")) + .andExpect(jsonPath("$.results.url").value("http://www.aliexpress.com/af/shoes.html?")); + + this.mockMvc.perform(post(grokUrl + "/validate").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(missingPatternLabelGrokValidationJson)) + .andExpect(status().isInternalServerError()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.responseCode").value(500)) + .andExpect(jsonPath("$.message").value("Pattern label is required")); + + this.mockMvc.perform(post(grokUrl + "/validate").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(missingStatementGrokValidationJson)) + .andExpect(status().isInternalServerError()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.responseCode").value(500)) + .andExpect(jsonPath("$.message").value("Grok statement is required")); + + this.mockMvc.perform(get(grokUrl + "/list").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$").isNotEmpty()); + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/HdfsControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/HdfsControllerIntegrationTest.java new file mode 100644 index 0000000000..a6c3469bdc --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/HdfsControllerIntegrationTest.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import org.apache.hadoop.fs.Path; +import org.apache.metron.rest.service.HdfsService; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(TEST_PROFILE) +public class HdfsControllerIntegrationTest { + + @Autowired + private HdfsService hdfsService; + + @Autowired + private WebApplicationContext wac; + + private MockMvc mockMvc; + + private String hdfsUrl = "/api/v1/hdfs"; + private String user = "user"; + private String password = "password"; + private String path = "./target/hdfsTest.txt"; + private String fileContents = "file contents"; + + @Before + public void setup() throws Exception { + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); + } + + @Test + public void testSecurity() throws Exception { + this.mockMvc.perform(post(hdfsUrl).with(csrf()).contentType(MediaType.parseMediaType("text/plain;charset=UTF-8")).content(fileContents)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(hdfsUrl)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(delete(hdfsUrl).with(csrf())) + .andExpect(status().isUnauthorized()); + } + + @Test + public void test() throws Exception { + this.hdfsService.delete(new Path(path), false); + + this.mockMvc.perform(get(hdfsUrl + "?path=" + path).with(httpBasic(user,password))) + .andExpect(status().isNotFound()); + + this.mockMvc.perform(post(hdfsUrl + "?path=" + path).with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("text/plain;charset=UTF-8")).content(fileContents)) + .andExpect(status().isOk()); + + this.mockMvc.perform(get(hdfsUrl + "?path=" + path).with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("text/plain;charset=UTF-8"))) + .andExpect(content().bytes(fileContents.getBytes())); + + this.mockMvc.perform(delete(hdfsUrl + "?path=" + path).with(httpBasic(user,password)).with(csrf())) + .andExpect(status().isOk()); + + this.mockMvc.perform(delete(hdfsUrl + "?path=" + path).with(httpBasic(user,password)).with(csrf())) + .andExpect(status().isNotFound()); + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java new file mode 100644 index 0000000000..02997082d0 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/KafkaControllerIntegrationTest.java @@ -0,0 +1,188 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.integration.components.KafkaComponent; +import org.apache.metron.rest.generator.SampleDataGenerator; +import org.apache.metron.rest.service.KafkaService; +import org.hamcrest.Matchers; +import org.json.simple.parser.ParseException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +import java.io.IOException; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(TEST_PROFILE) +public class KafkaControllerIntegrationTest { + + @Autowired + private KafkaComponent kafkaWithZKComponent; + + class SampleDataRunner implements Runnable { + + private boolean stop = false; + private String path = "../../metron-platform/metron-integration-test/src/main/sample/data/bro/raw/BroExampleOutput"; + + @Override + public void run() { + SampleDataGenerator broSampleDataGenerator = new SampleDataGenerator(); + broSampleDataGenerator.setBrokerUrl(kafkaWithZKComponent.getBrokerList()); + broSampleDataGenerator.setNum(1); + broSampleDataGenerator.setSelectedSensorType("bro"); + broSampleDataGenerator.setDelay(0); + try { + while(!stop) { + broSampleDataGenerator.generateSampleData(path); + } + } catch (IOException e) { + e.printStackTrace(); + } catch (ParseException e) { + e.printStackTrace(); + } + } + + public void stop() { + stop = true; + } + } + + private SampleDataRunner sampleDataRunner = new SampleDataRunner(); + private Thread sampleDataThread = new Thread(sampleDataRunner); + + /** + { + "name": "bro", + "numPartitions": 1, + "properties": {}, + "replicationFactor": 1 + } + */ + @Multiline + public static String broTopic; + + @Autowired + private WebApplicationContext wac; + + @Autowired + private KafkaService kafkaService; + + private MockMvc mockMvc; + + private String kafkaUrl = "/api/v1/kafka"; + private String user = "user"; + private String password = "password"; + + @Before + public void setup() throws Exception { + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); + } + + @Test + public void testSecurity() throws Exception { + this.mockMvc.perform(post(kafkaUrl + "/topic").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broTopic)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(kafkaUrl + "/topic/bro")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(kafkaUrl + "/topic")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(kafkaUrl + "/topic/bro/sample")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(delete(kafkaUrl + "/topic/bro").with(csrf())) + .andExpect(status().isUnauthorized()); + } + + @Test + public void test() throws Exception { + this.kafkaService.deleteTopic("bro"); + this.kafkaService.deleteTopic("someTopic"); + Thread.sleep(1000); + + this.mockMvc.perform(delete(kafkaUrl + "/topic/bro").with(httpBasic(user,password)).with(csrf())) + .andExpect(status().isNotFound()); + + this.mockMvc.perform(post(kafkaUrl + "/topic").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broTopic)) + .andExpect(status().isCreated()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.name").value("bro")) + .andExpect(jsonPath("$.numPartitions").value(1)) + .andExpect(jsonPath("$.replicationFactor").value(1)); + + sampleDataThread.start(); + Thread.sleep(1000); + + this.mockMvc.perform(get(kafkaUrl + "/topic/bro").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.name").value("bro")) + .andExpect(jsonPath("$.numPartitions").value(1)) + .andExpect(jsonPath("$.replicationFactor").value(1)); + + this.mockMvc.perform(get(kafkaUrl + "/topic/someTopic").with(httpBasic(user,password))) + .andExpect(status().isNotFound()); + + this.mockMvc.perform(get(kafkaUrl + "/topic").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$", Matchers.hasItem("bro"))); + + + this.mockMvc.perform(get(kafkaUrl + "/topic/bro/sample").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("text/plain;charset=UTF-8"))) + .andExpect(jsonPath("$").isNotEmpty()); + + this.mockMvc.perform(get(kafkaUrl + "/topic/someTopic/sample").with(httpBasic(user,password))) + .andExpect(status().isNotFound()); + + this.mockMvc.perform(delete(kafkaUrl + "/topic/bro").with(httpBasic(user,password)).with(csrf())) + .andExpect(status().isOk()); + } + + @After + public void tearDown() { + sampleDataRunner.stop(); + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java new file mode 100644 index 0000000000..0c9e18c430 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorEnrichmentConfigControllerIntegrationTest.java @@ -0,0 +1,247 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.rest.service.SensorEnrichmentConfigService; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(TEST_PROFILE) +public class SensorEnrichmentConfigControllerIntegrationTest { + + /** + { + "enrichment": { + "fieldMap": { + "geo": [ + "ip_dst_addr" + ], + "host": [ + "ip_dst_addr" + ], + "hbaseEnrichment": [ + "ip_src_addr" + ], + "stellar": { + "config": { + "group1": { + "foo": "1 + 1", + "bar": "foo" + }, + "group2": { + "ALL_CAPS": "TO_UPPER(source.type)" + } + } + } + }, + "fieldToTypeMap": { + "ip_src_addr": [ + "sample" + ] + } + }, + "threatIntel": { + "fieldMap": { + "hbaseThreatIntel": [ + "ip_src_addr", + "ip_dst_addr" + ] + }, + "fieldToTypeMap": { + "ip_src_addr": [ + "malicious_ip" + ], + "ip_dst_addr": [ + "malicious_ip" + ] + }, + "triageConfig": { + "riskLevelRules": [ + { + "rule": "ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'", + "score": 10 + } + ], + "aggregator": "MAX" + } + } + } + */ + @Multiline + public static String broJson; + + @Autowired + private SensorEnrichmentConfigService sensorEnrichmentConfigService; + + @Autowired + private WebApplicationContext wac; + + private MockMvc mockMvc; + + private String sensorEnrichmentConfigUrl = "/api/v1/sensor/enrichment/config"; + private String user = "user"; + private String password = "password"; + + @Before + public void setup() throws Exception { + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); + } + + @Test + public void testSecurity() throws Exception { + this.mockMvc.perform(post(sensorEnrichmentConfigUrl).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(sensorEnrichmentConfigUrl + "/broTest")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(sensorEnrichmentConfigUrl)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(delete(sensorEnrichmentConfigUrl + "/broTest").with(csrf())) + .andExpect(status().isUnauthorized()); + } + + @Test + public void test() throws Exception { + sensorEnrichmentConfigService.delete("broTest"); + + this.mockMvc.perform(get(sensorEnrichmentConfigUrl).with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(content().bytes("{}".getBytes())); + + this.mockMvc.perform(post(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) + .andExpect(status().isCreated()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.enrichment.fieldMap.geo[0]").value("ip_dst_addr")) + .andExpect(jsonPath("$.enrichment.fieldMap.host[0]").value("ip_dst_addr")) + .andExpect(jsonPath("$.enrichment.fieldMap.hbaseEnrichment[0]").value("ip_src_addr")) + .andExpect(jsonPath("$.enrichment.fieldToTypeMap.ip_src_addr[0]").value("sample")) + .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group1.foo").value("1 + 1")) + .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group1.bar").value("foo")) + .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group2.ALL_CAPS").value("TO_UPPER(source.type)")) + .andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[0]").value("ip_src_addr")) + .andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[1]").value("ip_dst_addr")) + .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_src_addr[0]").value("malicious_ip")) + .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_dst_addr[0]").value("malicious_ip")) + .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].rule").value("ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'")) + .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].score").value(10)) + .andExpect(jsonPath("$.threatIntel.triageConfig.aggregator").value("MAX")); + + this.mockMvc.perform(post(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.enrichment.fieldMap.geo[0]").value("ip_dst_addr")) + .andExpect(jsonPath("$.enrichment.fieldMap.host[0]").value("ip_dst_addr")) + .andExpect(jsonPath("$.enrichment.fieldMap.hbaseEnrichment[0]").value("ip_src_addr")) + .andExpect(jsonPath("$.enrichment.fieldToTypeMap.ip_src_addr[0]").value("sample")) + .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group1.foo").value("1 + 1")) + .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group1.bar").value("foo")) + .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group2.ALL_CAPS").value("TO_UPPER(source.type)")) + .andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[0]").value("ip_src_addr")) + .andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[1]").value("ip_dst_addr")) + .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_src_addr[0]").value("malicious_ip")) + .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_dst_addr[0]").value("malicious_ip")) + .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].rule").value("ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'")) + .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].score").value(10)) + .andExpect(jsonPath("$.threatIntel.triageConfig.aggregator").value("MAX")); + + this.mockMvc.perform(get(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.enrichment.fieldMap.geo[0]").value("ip_dst_addr")) + .andExpect(jsonPath("$.enrichment.fieldMap.host[0]").value("ip_dst_addr")) + .andExpect(jsonPath("$.enrichment.fieldMap.hbaseEnrichment[0]").value("ip_src_addr")) + .andExpect(jsonPath("$.enrichment.fieldToTypeMap.ip_src_addr[0]").value("sample")) + .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group1.foo").value("1 + 1")) + .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group1.bar").value("foo")) + .andExpect(jsonPath("$.enrichment.fieldMap.stellar.config.group2.ALL_CAPS").value("TO_UPPER(source.type)")) + .andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[0]").value("ip_src_addr")) + .andExpect(jsonPath("$.threatIntel.fieldMap.hbaseThreatIntel[1]").value("ip_dst_addr")) + .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_src_addr[0]").value("malicious_ip")) + .andExpect(jsonPath("$.threatIntel.fieldToTypeMap.ip_dst_addr[0]").value("malicious_ip")) + .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].rule").value("ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'")) + .andExpect(jsonPath("$.threatIntel.triageConfig.riskLevelRules[0].score").value(10)) + .andExpect(jsonPath("$.threatIntel.triageConfig.aggregator").value("MAX")); + + this.mockMvc.perform(get(sensorEnrichmentConfigUrl).with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$[?(@.broTest.enrichment.fieldMap.geo[0] == 'ip_dst_addr' &&" + + "@.broTest.enrichment.fieldMap.host[0] == 'ip_dst_addr' &&" + + "@.broTest.enrichment.fieldMap.hbaseEnrichment[0] == 'ip_src_addr' &&" + + "@.broTest.enrichment.fieldToTypeMap.ip_src_addr[0] == 'sample' &&" + + "@.broTest.enrichment.fieldMap.stellar.config.group1.foo == '1 + 1' &&" + + "@.broTest.enrichment.fieldMap.stellar.config.group1.bar == 'foo' &&" + + "@.broTest.enrichment.fieldMap.stellar.config.group2.ALL_CAPS == 'TO_UPPER(source.type)' &&" + + "@.broTest.threatIntel.fieldMap.hbaseThreatIntel[0] == 'ip_src_addr' &&" + + "@.broTest.threatIntel.fieldMap.hbaseThreatIntel[1] == 'ip_dst_addr' &&" + + "@.broTest.threatIntel.fieldToTypeMap.ip_src_addr[0] == 'malicious_ip' &&" + + "@.broTest.threatIntel.fieldToTypeMap.ip_dst_addr[0] == 'malicious_ip' &&" + + "@.broTest.threatIntel.triageConfig.riskLevelRules[0].rule == \"ip_src_addr == '10.122.196.204' or ip_dst_addr == '10.122.196.204'\" &&" + + "@.broTest.threatIntel.triageConfig.riskLevelRules[0].score == 10 &&" + + "@.broTest.threatIntel.triageConfig.aggregator == 'MAX'" + + ")]").exists()); + + this.mockMvc.perform(delete(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf())) + .andExpect(status().isOk()); + + this.mockMvc.perform(get(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user,password))) + .andExpect(status().isNotFound()); + + this.mockMvc.perform(delete(sensorEnrichmentConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf())) + .andExpect(status().isNotFound()); + + this.mockMvc.perform(get(sensorEnrichmentConfigUrl).with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$[?(@.sensorTopic == 'broTest')]").doesNotExist()); + + this.mockMvc.perform(get(sensorEnrichmentConfigUrl + "/list/available").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$[0]").value("geo")) + .andExpect(jsonPath("$[1]").value("host")) + .andExpect(jsonPath("$[2]").value("whois")); + } +} + diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java new file mode 100644 index 0000000000..10edadcb76 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.rest.service.SensorIndexingConfigService; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(TEST_PROFILE) +public class SensorIndexingConfigControllerIntegrationTest { + + /** + { + "index": "broTest", + "batchSize": 1 + } + */ + @Multiline + public static String broJson; + + @Autowired + private SensorIndexingConfigService sensorIndexingConfigService; + + @Autowired + private WebApplicationContext wac; + + private MockMvc mockMvc; + + private String sensorIndexingConfigUrl = "/api/v1/sensor/indexing/config"; + private String user = "user"; + private String password = "password"; + + @Before + public void setup() throws Exception { + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); + } + + @Test + public void testSecurity() throws Exception { + this.mockMvc.perform(post(sensorIndexingConfigUrl).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(sensorIndexingConfigUrl + "/broTest")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(sensorIndexingConfigUrl)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(delete(sensorIndexingConfigUrl + "/broTest").with(csrf())) + .andExpect(status().isUnauthorized()); + } + + @Test + public void test() throws Exception { + sensorIndexingConfigService.delete("broTest"); + + this.mockMvc.perform(get(sensorIndexingConfigUrl).with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(content().bytes("{}".getBytes())); + + this.mockMvc.perform(post(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) + .andExpect(status().isCreated()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.index").value("broTest")) + .andExpect(jsonPath("$.batchSize").value(1)); + + this.mockMvc.perform(post(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.index").value("broTest")) + .andExpect(jsonPath("$.batchSize").value(1)); + + this.mockMvc.perform(get(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.index").value("broTest")) + .andExpect(jsonPath("$.batchSize").value(1)); + + this.mockMvc.perform(get(sensorIndexingConfigUrl).with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$[?(@.broTest.index == 'broTest' &&" + + "@.broTest.batchSize == 1" + + ")]").exists()); + + this.mockMvc.perform(delete(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf())) + .andExpect(status().isOk()); + + this.mockMvc.perform(get(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user,password))) + .andExpect(status().isNotFound()); + + this.mockMvc.perform(delete(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf())) + .andExpect(status().isNotFound()); + + this.mockMvc.perform(get(sensorIndexingConfigUrl).with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$[?(@.sensorTopic == 'broTest')]").doesNotExist()); + } +} + diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java new file mode 100644 index 0000000000..fad751117c --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorParserConfigControllerIntegrationTest.java @@ -0,0 +1,351 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import org.adrianwalker.multilinestring.Multiline; +import org.apache.commons.io.FileUtils; +import org.apache.metron.rest.MetronRestConstants; +import org.apache.metron.rest.service.GrokService; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.core.env.Environment; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +import java.io.File; +import java.io.IOException; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(TEST_PROFILE) +public class SensorParserConfigControllerIntegrationTest { + + /** + { + "parserClassName": "org.apache.metron.parsers.GrokParser", + "sensorTopic": "squidTest", + "parserConfig": { + "patternLabel": "SQUIDTEST", + "grokPath": "target/patterns/squidTest", + "timestampField": "timestamp" + }, + "fieldTransformations" : [ + { + "transformation" : "STELLAR" + ,"output" : [ "full_hostname", "domain_without_subdomains" ] + ,"config" : { + "full_hostname" : "URL_TO_HOST(url)" + ,"domain_without_subdomains" : "DOMAIN_REMOVE_SUBDOMAINS(full_hostname)" + } + } + ] + } + */ + @Multiline + public static String squidJson; + + /** + { + "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser", + "sensorTopic":"broTest", + "parserConfig": {} + } + */ + @Multiline + public static String broJson; + + /** + { + "sensorParserConfig": + { + "parserClassName": "org.apache.metron.parsers.GrokParser", + "sensorTopic": "squidTest", + "parserConfig": { + "patternLabel": "SQUID_DELIMITED", + "grokPath":"./squidTest", + "timestampField": "timestamp" + } + }, + "grokStatement":"SQUID_DELIMITED %{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})?", + "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html" + } + */ + @Multiline + public static String parseRequest; + + /** + { + "sensorParserConfig": null, + "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html" + } + */ + @Multiline + public static String missingConfigParseRequest; + + /** + { + "sensorParserConfig": + { + "sensorTopic": "squidTest", + "parserConfig": { + "grokStatement": "%{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}", + "patternLabel": "SQUIDTEST", + "grokPath":"./squidTest", + "timestampField": "timestamp" + } + }, + "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html" + } + */ + @Multiline + public static String missingClassParseRequest; + + /** + { + "sensorParserConfig": + { + "parserClassName": "badClass", + "sensorTopic": "squidTest", + "parserConfig": { + "grokStatement": "%{NUMBER:timestamp} %{INT:elapsed} %{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url} - %{WORD:UNWANTED}\/%{IPV4:ip_dst_addr} %{WORD:UNWANTED}\/%{WORD:UNWANTED}", + "patternLabel": "SQUIDTEST", + "grokPath":"./squidTest", + "timestampField": "timestamp" + } + }, + "sampleData":"1467011157.401 415 127.0.0.1 TCP_MISS/200 337891 GET http://www.aliexpress.com/af/shoes.html? - DIRECT/207.109.73.154 text/html" + } + */ + @Multiline + public static String badClassParseRequest; + + @Autowired + private Environment environment; + + @Autowired + private GrokService grokService; + + @Autowired + private WebApplicationContext wac; + + private MockMvc mockMvc; + + private String sensorParserConfigUrl = "/api/v1/sensor/parser/config"; + private String user = "user"; + private String password = "password"; + + @Before + public void setup() throws Exception { + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); + } + + @Test + public void testSecurity() throws Exception { + this.mockMvc.perform(post(sensorParserConfigUrl).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(squidJson)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(sensorParserConfigUrl + "/squidTest")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(sensorParserConfigUrl)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(delete(sensorParserConfigUrl + "/squidTest").with(csrf())) + .andExpect(status().isUnauthorized()); + } + + @Test + public void test() throws Exception { + cleanFileSystem(); + + this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(squidJson)) + .andExpect(status().isCreated()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.GrokParser")) + .andExpect(jsonPath("$.sensorTopic").value("squidTest")) + .andExpect(jsonPath("$.parserConfig.grokPath").value("target/patterns/squidTest")) + .andExpect(jsonPath("$.parserConfig.patternLabel").value("SQUIDTEST")) + .andExpect(jsonPath("$.parserConfig.timestampField").value("timestamp")) + .andExpect(jsonPath("$.fieldTransformations[0].transformation").value("STELLAR")) + .andExpect(jsonPath("$.fieldTransformations[0].output[0]").value("full_hostname")) + .andExpect(jsonPath("$.fieldTransformations[0].output[1]").value("domain_without_subdomains")) + .andExpect(jsonPath("$.fieldTransformations[0].config.full_hostname").value("URL_TO_HOST(url)")) + .andExpect(jsonPath("$.fieldTransformations[0].config.domain_without_subdomains").value("DOMAIN_REMOVE_SUBDOMAINS(full_hostname)")); + + this.mockMvc.perform(get(sensorParserConfigUrl + "/squidTest").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.GrokParser")) + .andExpect(jsonPath("$.sensorTopic").value("squidTest")) + .andExpect(jsonPath("$.parserConfig.grokPath").value("target/patterns/squidTest")) + .andExpect(jsonPath("$.parserConfig.patternLabel").value("SQUIDTEST")) + .andExpect(jsonPath("$.parserConfig.timestampField").value("timestamp")) + .andExpect(jsonPath("$.fieldTransformations[0].transformation").value("STELLAR")) + .andExpect(jsonPath("$.fieldTransformations[0].output[0]").value("full_hostname")) + .andExpect(jsonPath("$.fieldTransformations[0].output[1]").value("domain_without_subdomains")) + .andExpect(jsonPath("$.fieldTransformations[0].config.full_hostname").value("URL_TO_HOST(url)")) + .andExpect(jsonPath("$.fieldTransformations[0].config.domain_without_subdomains").value("DOMAIN_REMOVE_SUBDOMAINS(full_hostname)")); + + this.mockMvc.perform(get(sensorParserConfigUrl).with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$[?(@.parserClassName == 'org.apache.metron.parsers.GrokParser' &&" + + "@.sensorTopic == 'squidTest' &&" + + "@.parserConfig.grokPath == 'target/patterns/squidTest' &&" + + "@.parserConfig.patternLabel == 'SQUIDTEST' &&" + + "@.parserConfig.timestampField == 'timestamp' &&" + + "@.fieldTransformations[0].transformation == 'STELLAR' &&" + + "@.fieldTransformations[0].output[0] == 'full_hostname' &&" + + "@.fieldTransformations[0].output[1] == 'domain_without_subdomains' &&" + + "@.fieldTransformations[0].config.full_hostname == 'URL_TO_HOST(url)' &&" + + "@.fieldTransformations[0].config.domain_without_subdomains == 'DOMAIN_REMOVE_SUBDOMAINS(full_hostname)')]").exists()); + + this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) + .andExpect(status().isCreated()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.bro.BasicBroParser")) + .andExpect(jsonPath("$.sensorTopic").value("broTest")) + .andExpect(jsonPath("$.parserConfig").isEmpty()); + + this.mockMvc.perform(post(sensorParserConfigUrl).with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.parserClassName").value("org.apache.metron.parsers.bro.BasicBroParser")) + .andExpect(jsonPath("$.sensorTopic").value("broTest")) + .andExpect(jsonPath("$.parserConfig").isEmpty()); + + this.mockMvc.perform(get(sensorParserConfigUrl).with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$[?(@.parserClassName == 'org.apache.metron.parsers.GrokParser' &&" + + "@.sensorTopic == 'squidTest' &&" + + "@.parserConfig.grokPath == 'target/patterns/squidTest' &&" + + "@.parserConfig.patternLabel == 'SQUIDTEST' &&" + + "@.parserConfig.timestampField == 'timestamp' &&" + + "@.fieldTransformations[0].transformation == 'STELLAR' &&" + + "@.fieldTransformations[0].output[0] == 'full_hostname' &&" + + "@.fieldTransformations[0].output[1] == 'domain_without_subdomains' &&" + + "@.fieldTransformations[0].config.full_hostname == 'URL_TO_HOST(url)' &&" + + "@.fieldTransformations[0].config.domain_without_subdomains == 'DOMAIN_REMOVE_SUBDOMAINS(full_hostname)')]").exists()) + .andExpect(jsonPath("$[?(@.parserClassName == 'org.apache.metron.parsers.bro.BasicBroParser' && " + + "@.sensorTopic == 'broTest')]").exists()); + + this.mockMvc.perform(delete(sensorParserConfigUrl + "/squidTest").with(httpBasic(user,password)).with(csrf())) + .andExpect(status().isOk()); + + this.mockMvc.perform(get(sensorParserConfigUrl + "/squidTest").with(httpBasic(user,password))) + .andExpect(status().isNotFound()); + + this.mockMvc.perform(delete(sensorParserConfigUrl + "/squidTest").with(httpBasic(user,password)).with(csrf())) + .andExpect(status().isNotFound()); + + this.mockMvc.perform(get(sensorParserConfigUrl).with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$[?(@.sensorTopic == 'squidTest')]").doesNotExist()) + .andExpect(jsonPath("$[?(@.sensorTopic == 'broTest')]").exists()); + + this.mockMvc.perform(delete(sensorParserConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf())) + .andExpect(status().isOk()); + + this.mockMvc.perform(delete(sensorParserConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf())) + .andExpect(status().isNotFound()); + + this.mockMvc.perform(get(sensorParserConfigUrl).with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$[?(@.sensorTopic == 'squidTest')]").doesNotExist()) + .andExpect(jsonPath("$[?(@.sensorTopic == 'broTest')]").doesNotExist()); + + this.mockMvc.perform(get(sensorParserConfigUrl + "/list/available").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.Bro").value("org.apache.metron.parsers.bro.BasicBroParser")) + .andExpect(jsonPath("$.Grok").value("org.apache.metron.parsers.GrokParser")); + + this.mockMvc.perform(get(sensorParserConfigUrl + "/reload/available").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.Bro").value("org.apache.metron.parsers.bro.BasicBroParser")) + .andExpect(jsonPath("$.Grok").value("org.apache.metron.parsers.GrokParser")); + + this.mockMvc.perform(post(sensorParserConfigUrl + "/parseMessage").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(parseRequest)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.elapsed").value(415)) + .andExpect(jsonPath("$.code").value(200)) + .andExpect(jsonPath("$.ip_dst_addr").value("207.109.73.154")) + .andExpect(jsonPath("$.method").value("GET")) + .andExpect(jsonPath("$.bytes").value(337891)) + .andExpect(jsonPath("$.action").value("TCP_MISS")) + .andExpect(jsonPath("$.ip_src_addr").value("127.0.0.1")) + .andExpect(jsonPath("$.url").value("http://www.aliexpress.com/af/shoes.html?")) + .andExpect(jsonPath("$.timestamp").value(1467011157401L)); + + this.mockMvc.perform(post(sensorParserConfigUrl + "/parseMessage").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(missingConfigParseRequest)) + .andExpect(status().isInternalServerError()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.responseCode").value(500)) + .andExpect(jsonPath("$.message").value("SensorParserConfig is missing from ParseMessageRequest")); + + this.mockMvc.perform(post(sensorParserConfigUrl + "/parseMessage").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(missingClassParseRequest)) + .andExpect(status().isInternalServerError()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.responseCode").value(500)) + .andExpect(jsonPath("$.message").value("SensorParserConfig must have a parserClassName")); + + this.mockMvc.perform(post(sensorParserConfigUrl + "/parseMessage").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(badClassParseRequest)) + .andExpect(status().isInternalServerError()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.responseCode").value(500)) + .andExpect(jsonPath("$.message").value("java.lang.ClassNotFoundException: badClass")); + + } + + private void cleanFileSystem() throws IOException { + File grokTempPath = new File(environment.getProperty(MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY)); + if (grokTempPath.exists()) { + FileUtils.cleanDirectory(grokTempPath); + FileUtils.deleteDirectory(grokTempPath); + } + File grokPath = new File(environment.getProperty(MetronRestConstants.GROK_DEFAULT_PATH_SPRING_PROPERTY)); + if (grokPath.exists()) { + FileUtils.cleanDirectory(grokPath); + FileUtils.deleteDirectory(grokPath); + } + } +} + diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StellarControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StellarControllerIntegrationTest.java new file mode 100644 index 0000000000..7a57173426 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StellarControllerIntegrationTest.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import org.adrianwalker.multilinestring.Multiline; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(TEST_PROFILE) +public class StellarControllerIntegrationTest { + + private String valid = "TO_LOWER(test)"; + private String invalid = "BAD_FUNCTION(test)"; + private String rulesJson = "[\"" + valid + "\",\"" + invalid + "\"]"; + + /** + { + "sensorParserConfig": { "fieldTransformations" : [{"transformation" : "STELLAR","output" : ["url_host"],"config" : {"url_host" : "TO_LOWER(URL_TO_HOST(url))"}}]}, + "sampleData": {"url": "https://caseystella.com/blog"} + } + */ + @Multiline + public static String sensorParseContext; + + + @Autowired + private WebApplicationContext wac; + + private MockMvc mockMvc; + + private String stellarUrl = "/api/v1/stellar"; + private String user = "user"; + private String password = "password"; + + @Before + public void setup() throws Exception { + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); + } + + @Test + public void testSecurity() throws Exception { + this.mockMvc.perform(post(stellarUrl + "/validate/rules").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(rulesJson)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(post(stellarUrl + "/apply/transformations").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(sensorParseContext)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(stellarUrl + "/list")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(stellarUrl + "/list/functions")) + .andExpect(status().isUnauthorized()); + } + + @Test + public void test() throws Exception { + this.mockMvc.perform(post(stellarUrl + "/validate/rules").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(rulesJson)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.['" + valid + "']").value(Boolean.TRUE)) + .andExpect(jsonPath("$.['" + invalid + "']").value(Boolean.FALSE)); + + this.mockMvc.perform(post(stellarUrl + "/apply/transformations").with(httpBasic(user,password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(sensorParseContext)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.url").value("https://caseystella.com/blog")) + .andExpect(jsonPath("$.url_host").value("caseystella.com")); + + this.mockMvc.perform(get(stellarUrl + "/list").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$", hasSize(greaterThan(0)))); + + this.mockMvc.perform(get(stellarUrl + "/list/functions").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$", hasSize(greaterThan(0)))); + + this.mockMvc.perform(get(stellarUrl + "/list/simple/functions").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$", hasSize(greaterThan(0)))); + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java new file mode 100644 index 0000000000..c592159330 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java @@ -0,0 +1,320 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.rest.model.TopologyStatusCode; +import org.apache.metron.rest.service.GlobalConfigService; +import org.apache.metron.rest.service.SensorParserConfigService; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.core.env.Environment; +import org.springframework.http.MediaType; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +import java.util.HashMap; +import java.util.Map; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasSize; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(TEST_PROFILE) +public class StormControllerIntegrationTest { + + @Autowired + private Environment environment; + + @Autowired + private WebApplicationContext wac; + + @Autowired + private GlobalConfigService globalConfigService; + + @Autowired + private SensorParserConfigService sensorParserConfigService; + + private MockMvc mockMvc; + + private String stormUrl = "/api/v1/storm"; + private String user = "user"; + private String password = "password"; + + private String metronVersion; + + @Before + public void setup() throws Exception { + this.metronVersion = this.environment.getProperty("metron.version"); + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); + } + + @Test + public void testSecurity() throws Exception { + this.mockMvc.perform(get(stormUrl)) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(stormUrl + "/broTest")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(stormUrl + "/parser/start/broTest")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(stormUrl + "/parser/stop/broTest")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(stormUrl + "/parser/activate/broTest")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(stormUrl + "/parser/deactivate/broTest")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get("/enrichment")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(stormUrl + "/enrichment/start")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(stormUrl + "/enrichment/stop")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(stormUrl + "/enrichment/activate")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(stormUrl + "/enrichment/deactivate")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get("/indexing")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(stormUrl + "/indexing/start")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(stormUrl + "/indexing/stop")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(stormUrl + "/indexing/activate")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get(stormUrl + "/indexing/deactivate")) + .andExpect(status().isUnauthorized()); + } + + @Test + public void test() throws Exception { + this.mockMvc.perform(get(stormUrl).with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$", hasSize(0))); + + this.mockMvc.perform(get(stormUrl + "/broTest").with(httpBasic(user,password))) + .andExpect(status().isNotFound()); + + Map globalConfig = globalConfigService.get(); + if (globalConfig == null) { + globalConfig = new HashMap<>(); + } + globalConfigService.delete(); + sensorParserConfigService.delete("broTest"); + + this.mockMvc.perform(get(stormUrl + "/parser/stop/broTest?stopNow=true").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("ERROR")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.STOP_ERROR.toString())); + + this.mockMvc.perform(get(stormUrl + "/parser/activate/broTest").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("ERROR")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name())); + + this.mockMvc.perform(get(stormUrl + "/parser/deactivate/broTest").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("ERROR")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name())); + + this.mockMvc.perform(get(stormUrl + "/parser/start/broTest").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("ERROR")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.GLOBAL_CONFIG_MISSING.name())); + + globalConfigService.save(globalConfig); + + this.mockMvc.perform(get(stormUrl + "/parser/start/broTest").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("ERROR")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.SENSOR_PARSER_CONFIG_MISSING.name())); + + SensorParserConfig sensorParserConfig = new SensorParserConfig(); + sensorParserConfig.setParserClassName("org.apache.metron.parsers.bro.BasicBroParser"); + sensorParserConfig.setSensorTopic("broTest"); + sensorParserConfigService.save(sensorParserConfig); + + this.mockMvc.perform(get(stormUrl + "/parser/start/broTest").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("SUCCESS")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.STARTED.name())); + + this.mockMvc.perform(get(stormUrl + "/broTest").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.name").value("broTest")) + .andExpect(jsonPath("$.id", containsString("broTest"))) + .andExpect(jsonPath("$.status").value("ACTIVE")) + .andExpect(jsonPath("$.latency").exists()) + .andExpect(jsonPath("$.throughput").exists()); + + this.mockMvc.perform(get(stormUrl).with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$[?(@.name == 'broTest' && @.status == 'ACTIVE')]").exists()); + + this.mockMvc.perform(get(stormUrl + "/parser/stop/broTest?stopNow=true").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("SUCCESS")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.STOPPED.name())); + + this.mockMvc.perform(get(stormUrl + "/enrichment").with(httpBasic(user,password))) + .andExpect(status().isNotFound()); + + this.mockMvc.perform(get(stormUrl + "/enrichment/activate").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("ERROR")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name())); + + this.mockMvc.perform(get(stormUrl + "/enrichment/deactivate").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("ERROR")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name())); + + this.mockMvc.perform(get(stormUrl + "/enrichment/stop?stopNow=true").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("ERROR")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.STOP_ERROR.toString())); + + this.mockMvc.perform(get(stormUrl + "/enrichment/start").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("SUCCESS")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.STARTED.toString())); + + this.mockMvc.perform(get(stormUrl + "/enrichment/deactivate").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("SUCCESS")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.INACTIVE.name())); + + this.mockMvc.perform(get(stormUrl + "/enrichment/deactivate").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("SUCCESS")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.INACTIVE.name())); + + this.mockMvc.perform(get(stormUrl + "/enrichment/activate").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("SUCCESS")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.ACTIVE.name())); + + this.mockMvc.perform(get(stormUrl + "/enrichment").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.name").value("enrichment")) + .andExpect(jsonPath("$.id", containsString("enrichment"))) + .andExpect(jsonPath("$.status").value("ACTIVE")) + .andExpect(jsonPath("$.latency").exists()) + .andExpect(jsonPath("$.throughput").exists()); + + this.mockMvc.perform(get(stormUrl).with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$[?(@.name == 'enrichment' && @.status == 'ACTIVE')]").exists()); + + this.mockMvc.perform(get(stormUrl + "/enrichment/stop").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("SUCCESS")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.STOPPED.name())); + + this.mockMvc.perform(get(stormUrl + "/indexing").with(httpBasic(user,password))) + .andExpect(status().isNotFound()); + + this.mockMvc.perform(get(stormUrl + "/indexing/activate").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("ERROR")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name())); + + this.mockMvc.perform(get(stormUrl + "/indexing/deactivate").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("ERROR")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name())); + + this.mockMvc.perform(get(stormUrl + "/indexing/stop?stopNow=true").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("ERROR")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.STOP_ERROR.toString())); + + this.mockMvc.perform(get(stormUrl + "/indexing/start").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("SUCCESS")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.STARTED.toString())); + + this.mockMvc.perform(get(stormUrl + "/indexing/deactivate").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("SUCCESS")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.INACTIVE.name())); + + this.mockMvc.perform(get(stormUrl + "/indexing/activate").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("SUCCESS")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.ACTIVE.name())); + + this.mockMvc.perform(get(stormUrl + "/indexing").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$.name").value("indexing")) + .andExpect(jsonPath("$.id", containsString("indexing"))) + .andExpect(jsonPath("$.status").value("ACTIVE")) + .andExpect(jsonPath("$.latency").exists()) + .andExpect(jsonPath("$.throughput").exists()); + + this.mockMvc.perform(get(stormUrl).with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8"))) + .andExpect(jsonPath("$[?(@.name == 'indexing' && @.status == 'ACTIVE')]").exists()); + + this.mockMvc.perform(get(stormUrl + "/indexing/stop").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.status").value("SUCCESS")) + .andExpect(jsonPath("$.message").value(TopologyStatusCode.STOPPED.name())); + + this.mockMvc.perform(get(stormUrl + "/client/status").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.stormClientVersionInstalled").value("1.0.1")) + .andExpect(jsonPath("$.parserScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_parser_topology.sh")) + .andExpect(jsonPath("$.enrichmentScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_enrichment_topology.sh")) + .andExpect(jsonPath("$.indexingScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_elasticsearch_topology.sh")); + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UserControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UserControllerIntegrationTest.java new file mode 100644 index 0000000000..9bdd439759 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UserControllerIntegrationTest.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.controller; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.web.context.WebApplicationContext; + +import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE; +import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic; +import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles(TEST_PROFILE) +public class UserControllerIntegrationTest { + + private MockMvc mockMvc; + + @Autowired + private WebApplicationContext wac; + + private String user = "user"; + private String password = "password"; + + @Before + public void setup() throws Exception { + this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build(); + } + + @Test + public void test() throws Exception { + this.mockMvc.perform(get("/api/v1/user")) + .andExpect(status().isUnauthorized()); + + this.mockMvc.perform(get("/api/v1/user").with(httpBasic(user,password))) + .andExpect(status().isOk()) + .andExpect(content().string(user)); + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/generator/SampleDataGenerator.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/generator/SampleDataGenerator.java new file mode 100644 index 0000000000..8557035f5c --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/generator/SampleDataGenerator.java @@ -0,0 +1,170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.generator; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.apache.commons.io.FileUtils; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.json.simple.parser.ParseException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class SampleDataGenerator { + + private Logger LOG = LoggerFactory.getLogger(SampleDataGenerator.class); + + private Map> sampleData = new HashMap<>(); + private Map indexes = new HashMap<>(); + private KafkaProducer kafkaProducer; + private String brokerUrl; + private Integer num = -1; + private String selectedSensorType = null; + private Integer delay = 1000; + + public void setBrokerUrl(String brokerUrl) { + this.brokerUrl = brokerUrl; + } + + public void setNum(Integer num) { + this.num = num; + } + + public void setSelectedSensorType(String selectedSensorType) { + this.selectedSensorType = selectedSensorType; + } + + public void setDelay(Integer delay) { + this.delay = delay; + } + + public static void main(String[] args) throws org.apache.commons.cli.ParseException, IOException, ParseException { + CommandLineParser parser = new PosixParser(); + CommandLine cli = parser.parse(getOptions(), args); + Integer num = Integer.parseInt(cli.getOptionValue("n", "-1")); + String selectedSensorType = cli.getOptionValue("s"); + Integer delay = Integer.parseInt(cli.getOptionValue("d", "1000")); + String path = cli.getOptionValue("p"); + if (selectedSensorType == null || path == null) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp( "sample_data_generator", getOptions()); + } else { + SampleDataGenerator sampleDataGenerator = new SampleDataGenerator(); + sampleDataGenerator.setNum(num); + sampleDataGenerator.setSelectedSensorType(selectedSensorType); + sampleDataGenerator.setDelay(delay); + sampleDataGenerator.generateSampleData(path); + } + + } + + private static Options getOptions() { + Options options = new Options(); + options.addOption("b", "brokerUrl", true, "Kafka Broker Url"); + options.addOption("n", "num", false, "Number of messages to emit"); + options.addOption("s", "sensorType", true, "Emit messages to this topic"); + options.addOption("d", "delay", false, "Number of milliseconds to wait between each message. Defaults to 1 second"); + options.addOption("p", "path", true, "Local path to data file"); + return options; + } + + public void generateSampleData(String path) throws IOException, ParseException { + loadData(path); + startClients(); + try { + emitData(num, selectedSensorType, delay); + } finally { + stopClients(); + } + } + + private void loadData(String sampleDataPath) throws IOException, ParseException { + sampleData.put(selectedSensorType, FileUtils.readLines(new File(sampleDataPath))); + indexes.put(selectedSensorType, 0); + } + + private void emitData(int num, String selectedSensorType, int delay) { + int count = 0; + boolean continueEmitting = false; + do { + for (String sensorType : sampleData.keySet()) { + if (selectedSensorType == null || selectedSensorType.equals(sensorType)) { + List sensorData = sampleData.get(sensorType); + int index = indexes.get(sensorType); + String message = sensorData.get(index++); + emitSensorData(sensorType, message, delay); + if (num != -1 && ++count >= num) { + continueEmitting = false; + break; + } + continueEmitting = true; + if (index == sensorData.size()) { + index = 0; + } + indexes.put(sensorType, index); + } + } + } while (continueEmitting); + } + + private void emitSensorData(String sensorType, String message, int delay) { + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + e.printStackTrace(); + } + LOG.info("Emitting " + sensorType + " message " + message); + emitToKafka(sensorType, message); + } + + private void startClients() { + startKafka(); + } + + private void startKafka() { + Map producerConfig = new HashMap<>(); + producerConfig.put("bootstrap.servers", brokerUrl); + producerConfig.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); + producerConfig.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); + kafkaProducer = new KafkaProducer<>(producerConfig); + } + + private void stopClients() { + stopKafka(); + } + + private void stopKafka() { + LOG.info("Stopping Kafka producer"); + kafkaProducer.close(); + } + + private void emitToKafka(String topic, String message) { + kafkaProducer.send(new ProducerRecord(topic, message)); + } + +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormCLIClientWrapper.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormCLIClientWrapper.java new file mode 100644 index 0000000000..dd210958de --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormCLIClientWrapper.java @@ -0,0 +1,177 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.mock; + +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.TopologyStatusCode; +import org.apache.metron.rest.service.impl.StormCLIWrapper; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class MockStormCLIClientWrapper extends StormCLIWrapper { + + private final Map parsersStatus = new HashMap<>(); + private TopologyStatusCode enrichmentStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND; + private TopologyStatusCode indexingStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND; + + public Set getParserTopologyNames() { + return parsersStatus.keySet(); + } + + public TopologyStatusCode getParserStatus(String name) { + TopologyStatusCode parserStatus = parsersStatus.get(name); + if (parserStatus == null) { + return TopologyStatusCode.TOPOLOGY_NOT_FOUND; + } else { + return parserStatus; + } + } + + @Override + public int startParserTopology(String name) throws RestException { + TopologyStatusCode parserStatus = parsersStatus.get(name); + if (parserStatus == null || parserStatus == TopologyStatusCode.TOPOLOGY_NOT_FOUND) { + parsersStatus.put(name, TopologyStatusCode.ACTIVE); + return 0; + } else { + return 1; + } + } + + @Override + public int stopParserTopology(String name, boolean stopNow) throws RestException { + TopologyStatusCode parserStatus = parsersStatus.get(name); + if (parserStatus == TopologyStatusCode.ACTIVE) { + parsersStatus.put(name, TopologyStatusCode.TOPOLOGY_NOT_FOUND); + return 0; + } else { + return 1; + } + } + + public int activateParserTopology(String name) { + TopologyStatusCode parserStatus = parsersStatus.get(name); + if (parserStatus == TopologyStatusCode.INACTIVE || parserStatus == TopologyStatusCode.ACTIVE) { + parsersStatus.put(name, TopologyStatusCode.ACTIVE); + return 0; + } else { + return 1; + } + } + + public int deactivateParserTopology(String name) { + TopologyStatusCode parserStatus = parsersStatus.get(name); + if (parserStatus == TopologyStatusCode.INACTIVE || parserStatus == TopologyStatusCode.ACTIVE) { + parsersStatus.put(name, TopologyStatusCode.INACTIVE); + return 0; + } else { + return 1; + } + } + + public TopologyStatusCode getEnrichmentStatus() { + return enrichmentStatus; + } + + @Override + public int startEnrichmentTopology() throws RestException { + if (enrichmentStatus == TopologyStatusCode.TOPOLOGY_NOT_FOUND) { + enrichmentStatus = TopologyStatusCode.ACTIVE; + return 0; + } else { + return 1; + } + } + + @Override + public int stopEnrichmentTopology(boolean stopNow) throws RestException { + if (enrichmentStatus == TopologyStatusCode.ACTIVE) { + enrichmentStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND; + return 0; + } else { + return 1; + } + } + + public int activateEnrichmentTopology() { + if (enrichmentStatus == TopologyStatusCode.INACTIVE || enrichmentStatus == TopologyStatusCode.ACTIVE) { + enrichmentStatus = TopologyStatusCode.ACTIVE; + return 0; + } else { + return 1; + } + } + + public int deactivateEnrichmentTopology() { + if (enrichmentStatus == TopologyStatusCode.INACTIVE || enrichmentStatus == TopologyStatusCode.ACTIVE) { + enrichmentStatus = TopologyStatusCode.INACTIVE; + return 0; + } else { + return 1; + } + } + + public TopologyStatusCode getIndexingStatus() { + return indexingStatus; + } + + @Override + public int startIndexingTopology() throws RestException { + if (indexingStatus == TopologyStatusCode.TOPOLOGY_NOT_FOUND) { + indexingStatus = TopologyStatusCode.ACTIVE; + return 0; + } else { + return 1; + } + } + + @Override + public int stopIndexingTopology(boolean stopNow) throws RestException { + if (indexingStatus == TopologyStatusCode.ACTIVE) { + indexingStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND; + return 0; + } else { + return 1; + } + } + + public int activateIndexingTopology() { + if (indexingStatus == TopologyStatusCode.INACTIVE || indexingStatus == TopologyStatusCode.ACTIVE) { + indexingStatus = TopologyStatusCode.ACTIVE; + return 0; + } else { + return 1; + } + } + + public int deactivateIndexingTopology() { + if (indexingStatus == TopologyStatusCode.INACTIVE || indexingStatus == TopologyStatusCode.ACTIVE) { + indexingStatus = TopologyStatusCode.INACTIVE; + return 0; + } else { + return 1; + } + } + + @Override + protected String stormClientVersionInstalled() throws RestException { + return "1.0.1"; + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormRestTemplate.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormRestTemplate.java new file mode 100644 index 0000000000..ca5c1c9ad2 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormRestTemplate.java @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.mock; + +import org.apache.metron.rest.MetronRestConstants; +import org.apache.metron.rest.model.TopologyStatus; +import org.apache.metron.rest.model.TopologyStatusCode; +import org.apache.metron.rest.model.TopologySummary; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; +import org.springframework.web.client.RestClientException; +import org.springframework.web.client.RestTemplate; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class MockStormRestTemplate extends RestTemplate { + + @Autowired + private Environment environment; + + private MockStormCLIClientWrapper mockStormCLIClientWrapper; + + public void setMockStormCLIClientWrapper(MockStormCLIClientWrapper mockStormCLIClientWrapper) { + this.mockStormCLIClientWrapper = mockStormCLIClientWrapper; + } + + @Override + public Object getForObject(String url, Class responseType, Object... urlVariables) throws RestClientException { + Object response = null; + if (url.equals("http://" + environment.getProperty(MetronRestConstants.STORM_UI_SPRING_PROPERTY) + MetronRestConstants.TOPOLOGY_SUMMARY_URL)) { + TopologySummary topologySummary = new TopologySummary(); + List topologyStatusList = new ArrayList<>(); + for(String name: mockStormCLIClientWrapper.getParserTopologyNames()) { + topologyStatusList.add(getTopologyStatus(name)); + } + TopologyStatusCode enrichmentStatus = mockStormCLIClientWrapper.getEnrichmentStatus(); + if (enrichmentStatus != TopologyStatusCode.TOPOLOGY_NOT_FOUND) { + topologyStatusList.add(getTopologyStatus("enrichment")); + } + TopologyStatusCode indexingStatus = mockStormCLIClientWrapper.getIndexingStatus(); + if (indexingStatus != TopologyStatusCode.TOPOLOGY_NOT_FOUND) { + topologyStatusList.add(getTopologyStatus("indexing")); + } + topologySummary.setTopologies(topologyStatusList.toArray(new TopologyStatus[topologyStatusList.size()])); + response = topologySummary; + } else if (url.startsWith("http://" + environment.getProperty(MetronRestConstants.STORM_UI_SPRING_PROPERTY) + MetronRestConstants.TOPOLOGY_URL + "/")){ + String name = url.substring(url.lastIndexOf('/') + 1, url.length()).replaceFirst("-id", ""); + response = getTopologyStatus(name); + } + return response; + } + + private TopologyStatus getTopologyStatus(String name) { + TopologyStatus topologyStatus = new TopologyStatus(); + topologyStatus.setName(name); + topologyStatus.setId(name + "-id"); + if ("enrichment".equals(name)) { + topologyStatus.setStatus(mockStormCLIClientWrapper.getEnrichmentStatus()); + } else if ("indexing".equals(name)) { + topologyStatus.setStatus(mockStormCLIClientWrapper.getIndexingStatus()); + } else { + topologyStatus.setStatus(mockStormCLIClientWrapper.getParserStatus(name)); + } + return topologyStatus; + } + + @Override + public Object postForObject(String url, Object request, Class responseType, Object... uriVariables) throws RestClientException { + Map result = new HashMap<>(); + String[] urlParts = url.split("/"); + String name = urlParts[urlParts.length - 2].replaceFirst("-id", ""); + String action = urlParts[urlParts.length - 1]; + int returnCode = 0; + if (action.equals("activate")) { + if (name.equals("enrichment")) { + returnCode = mockStormCLIClientWrapper.activateEnrichmentTopology(); + } else if (name.equals("indexing")) { + returnCode = mockStormCLIClientWrapper.activateIndexingTopology(); + } else { + returnCode = mockStormCLIClientWrapper.activateParserTopology(name); + } + } else if (action.equals("deactivate")){ + if (name.equals("enrichment")) { + returnCode = mockStormCLIClientWrapper.deactivateEnrichmentTopology(); + } else if (name.equals("indexing")) { + returnCode = mockStormCLIClientWrapper.deactivateIndexingTopology(); + } else { + returnCode = mockStormCLIClientWrapper.deactivateParserTopology(name); + } + } + if (returnCode == 0) { + result.put("status", "success"); + } else { + result.put("status", "error"); + } + return result; + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java new file mode 100644 index 0000000000..bc91b91a89 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/DockerStormCLIWrapperTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.springframework.core.env.Environment; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.anyVararg; +import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.mock; +import static org.powermock.api.mockito.PowerMockito.when; +import static org.powermock.api.mockito.PowerMockito.whenNew; + +@SuppressWarnings("unchecked") +@RunWith(PowerMockRunner.class) +@PrepareForTest({DockerStormCLIWrapper.class, ProcessBuilder.class}) +public class DockerStormCLIWrapperTest { + private ProcessBuilder processBuilder; + private Environment environment; + private DockerStormCLIWrapper dockerStormCLIWrapper; + + @Before + public void setUp() throws Exception { + processBuilder = mock(ProcessBuilder.class); + environment = mock(Environment.class); + + dockerStormCLIWrapper = new DockerStormCLIWrapper(environment); + } + + @Test + public void getProcessBuilderShouldProperlyGenerateProcessorBuilder() throws Exception { + whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder); + + when(processBuilder.environment()).thenReturn(new HashMap<>()); + when(processBuilder.command()).thenReturn(new ArrayList<>()); + + Process process = mock(Process.class); + InputStream inputStream = new ByteArrayInputStream("export DOCKER_HOST=\"tcp://192.168.99.100:2376\"".getBytes()); + + when(processBuilder.start()).thenReturn(process); + when(process.getInputStream()).thenReturn(inputStream); + when(environment.getProperty("docker.compose.path")).thenReturn("/test"); + when(environment.getProperty("metron.version")).thenReturn("1"); + + + ProcessBuilder actualBuilder = dockerStormCLIWrapper.getProcessBuilder("oo", "ooo"); + + assertEquals(new HashMap() {{ + put("METRON_VERSION", "1"); + put("DOCKER_HOST", "tcp://192.168.99.100:2376"); + }}, actualBuilder.environment()); + assertEquals(new ArrayList<>(), actualBuilder.command()); + + verify(process).waitFor(); + } +} \ No newline at end of file diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImplTest.java new file mode 100644 index 0000000000..59d595739d --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GlobalConfigServiceImplTest.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.api.DeleteBuilder; +import org.apache.curator.framework.api.GetDataBuilder; +import org.apache.curator.framework.api.SetDataBuilder; +import org.apache.metron.common.configuration.ConfigurationType; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.GlobalConfigService; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.data.Stat; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@SuppressWarnings("ALL") +public class GlobalConfigServiceImplTest { + @Rule + public final ExpectedException exception = ExpectedException.none(); + + CuratorFramework curatorFramework; + GlobalConfigService globalConfigService; + + @Before + public void setUp() throws Exception { + curatorFramework = mock(CuratorFramework.class); + globalConfigService = new GlobalConfigServiceImpl(curatorFramework); + } + + + @Test + public void deleteShouldProperlyCatchNoNodeExceptionAndReturnFalse() throws Exception { + DeleteBuilder builder = mock(DeleteBuilder.class); + + when(curatorFramework.delete()).thenReturn(builder); + when(builder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot())).thenThrow(KeeperException.NoNodeException.class); + + assertFalse(globalConfigService.delete()); + } + + @Test + public void deleteShouldProperlyCatchNonNoNodeExceptionAndThrowRestException() throws Exception { + exception.expect(RestException.class); + + DeleteBuilder builder = mock(DeleteBuilder.class); + + when(curatorFramework.delete()).thenReturn(builder); + when(builder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot())).thenThrow(Exception.class); + + assertFalse(globalConfigService.delete()); + } + + @Test + public void deleteShouldReturnTrueWhenClientSuccessfullyCallsDelete() throws Exception { + DeleteBuilder builder = mock(DeleteBuilder.class); + + when(curatorFramework.delete()).thenReturn(builder); + when(builder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot())).thenReturn(null); + + assertTrue(globalConfigService.delete()); + + verify(curatorFramework).delete(); + } + + @Test + public void getShouldProperlyReturnGlobalConfig() throws Exception { + final String config = "{\"k\":\"v\"}"; + final Map configMap = new HashMap() {{ + put("k", "v"); + }}; + + GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); + when(getDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot())).thenReturn(config.getBytes()); + + when(curatorFramework.getData()).thenReturn(getDataBuilder); + + assertEquals(configMap, globalConfigService.get()); + } + + @Test + public void getShouldReturnNullWhenNoNodeExceptionIsThrown() throws Exception { + GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); + when(getDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot())).thenThrow(KeeperException.NoNodeException.class); + + when(curatorFramework.getData()).thenReturn(getDataBuilder); + + assertNull(globalConfigService.get()); + } + + @Test + public void getShouldWrapNonNoNodeExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); + when(getDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot())).thenThrow(Exception.class); + + when(curatorFramework.getData()).thenReturn(getDataBuilder); + + globalConfigService.get(); + } + + @Test + public void saveShouldWrapExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + SetDataBuilder setDataBuilder = mock(SetDataBuilder.class); + when(setDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot(), "{}".getBytes())).thenThrow(Exception.class); + + when(curatorFramework.setData()).thenReturn(setDataBuilder); + + globalConfigService.save(new HashMap<>()); + } + + @Test + public void saveShouldReturnSameConfigThatIsPassedOnSuccessfulSave() throws Exception { + SetDataBuilder setDataBuilder = mock(SetDataBuilder.class); + when(setDataBuilder.forPath(ConfigurationType.GLOBAL.getZookeeperRoot(), "{}".getBytes())).thenReturn(new Stat()); + + when(curatorFramework.setData()).thenReturn(setDataBuilder); + + assertEquals(new HashMap<>(), globalConfigService.save(new HashMap<>())); + verify(setDataBuilder).forPath(eq(ConfigurationType.GLOBAL.getZookeeperRoot()), eq("{}".getBytes())); + } +} \ No newline at end of file diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java new file mode 100644 index 0000000000..6dd95c5808 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/GrokServiceImplTest.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import oi.thekraken.grok.api.Grok; +import org.apache.commons.io.FileUtils; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.GrokValidation; +import org.apache.metron.rest.service.GrokService; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.springframework.core.env.Environment; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.metron.rest.MetronRestConstants.GROK_TEMP_PATH_SPRING_PROPERTY; +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.when; +import static org.powermock.api.mockito.PowerMockito.whenNew; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({GrokServiceImpl.class, FileWriter.class}) +public class GrokServiceImplTest { + @Rule + public final ExpectedException exception = ExpectedException.none(); + + private Environment environment; + private Grok grok; + private GrokService grokService; + + @Before + public void setUp() throws Exception { + environment = mock(Environment.class); + grok = mock(Grok.class); + grokService = new GrokServiceImpl(environment, grok); + } + + @Test + public void getCommonGrokPattersShouldCallGrokToGetPatterns() throws Exception { + grokService.getCommonGrokPatterns(); + + verify(grok).getPatterns(); + } + + @Test + public void getCommonGrokPattersShouldCallGrokToGetPatternsAndNotAlterValue() throws Exception { + final Map actual = new HashMap() {{ + put("k", "v"); + put("k1", "v1"); + }}; + + when(grok.getPatterns()).thenReturn(actual); + + Map expected = new HashMap() {{ + put("k", "v"); + put("k1", "v1"); + }}; + assertEquals(expected, grokService.getCommonGrokPatterns()); + } + + @Test + public void validateGrokStatementShouldThrowExceptionWithNullStringAsPatternLabel() throws Exception { + exception.expect(RestException.class); + exception.expectMessage("Pattern label is required"); + + GrokValidation grokValidation = new GrokValidation(); + grokValidation.setResults(new HashMap<>()); + grokValidation.setSampleData("asdf asdf"); + grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}"); + + grokService.validateGrokStatement(grokValidation); + } + + @Test + public void validateGrokStatementShouldThrowExceptionWithEmptyStringAsStatement() throws Exception { + exception.expect(RestException.class); + exception.expectMessage("Grok statement is required"); + + GrokValidation grokValidation = new GrokValidation(); + grokValidation.setResults(new HashMap<>()); + grokValidation.setSampleData("asdf asdf"); + grokValidation.setPatternLabel("LABEL"); + grokValidation.setStatement(""); + + grokService.validateGrokStatement(grokValidation); + } + + @Test + public void validateGrokStatementShouldThrowExceptionWithNullStringAsStatement() throws Exception { + exception.expect(RestException.class); + exception.expectMessage("Grok statement is required"); + + GrokValidation grokValidation = new GrokValidation(); + grokValidation.setResults(new HashMap<>()); + grokValidation.setSampleData("asdf asdf"); + grokValidation.setPatternLabel("LABEL"); + grokValidation.setStatement(null); + + grokService.validateGrokStatement(grokValidation); + } + + @Test + public void validateGrokStatementShouldProperlyMatchSampleDataAgainstGivenStatement() throws Exception { + final GrokValidation grokValidation = new GrokValidation(); + grokValidation.setResults(new HashMap<>()); + grokValidation.setSampleData("asdf asdf"); + grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}"); + grokValidation.setPatternLabel("LABEL"); + + GrokValidation expected = new GrokValidation(); + expected.setResults(new HashMap() {{ put("word1", "asdf"); put("word2", "asdf"); }}); + expected.setSampleData("asdf asdf"); + expected.setStatement("LABEL %{WORD:word1} %{WORD:word2}"); + expected.setPatternLabel("LABEL"); + + GrokValidation actual = grokService.validateGrokStatement(grokValidation); + assertEquals(expected, actual); + assertEquals(expected.hashCode(), actual.hashCode()); + } + + @Test + public void validateGrokStatementShouldProperlyMatchNothingAgainstEmptyString() throws Exception { + final GrokValidation grokValidation = new GrokValidation(); + grokValidation.setResults(new HashMap<>()); + grokValidation.setSampleData(""); + grokValidation.setPatternLabel("LABEL"); + grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}"); + + GrokValidation expected = new GrokValidation(); + expected.setResults(new HashMap<>()); + expected.setSampleData(""); + expected.setPatternLabel("LABEL"); + expected.setStatement("LABEL %{WORD:word1} %{WORD:word2}"); + + assertEquals(expected, grokService.validateGrokStatement(grokValidation)); + } + + @Test + public void validateGrokStatementShouldProperlyMatchNothingAgainstNullString() throws Exception { + final GrokValidation grokValidation = new GrokValidation(); + grokValidation.setResults(new HashMap<>()); + grokValidation.setSampleData(null); + grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2}"); + grokValidation.setPatternLabel("LABEL"); + + GrokValidation expected = new GrokValidation(); + expected.setResults(new HashMap<>()); + expected.setSampleData(null); + expected.setStatement("LABEL %{WORD:word1} %{WORD:word2}"); + expected.setPatternLabel("LABEL"); + + assertEquals(expected, grokService.validateGrokStatement(grokValidation)); + } + + @Test + public void invalidGrokStatementShouldThrowRestException() throws Exception { + exception.expect(RestException.class); + + GrokValidation grokValidation = new GrokValidation(); + grokValidation.setResults(new HashMap<>()); + grokValidation.setSampleData(null); + grokValidation.setStatement("LABEL %{WORD:word1} %{WORD:word2"); + + grokService.validateGrokStatement(grokValidation); + } + + @Test + public void saveTemporaryShouldProperlySaveFile() throws Exception { + new File("./target/user1").delete(); + String statement = "grok statement"; + + Authentication authentication = mock(Authentication.class); + when(authentication.getName()).thenReturn("user1"); + SecurityContextHolder.getContext().setAuthentication(authentication); + when(environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY)).thenReturn("./target"); + + grokService.saveTemporary(statement, "squid"); + + File testFile = new File("./target/user1/squid"); + assertEquals(statement, FileUtils.readFileToString(testFile)); + testFile.delete(); + } + + @Test + public void saveTemporaryShouldWrapExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + String statement = "grok statement"; + + Authentication authentication = mock(Authentication.class); + when(authentication.getName()).thenReturn("user1"); + SecurityContextHolder.getContext().setAuthentication(authentication); + when(environment.getProperty(GROK_TEMP_PATH_SPRING_PROPERTY)).thenReturn("./target"); + whenNew(FileWriter.class).withParameterTypes(File.class).withArguments(any()).thenThrow(new IOException()); + + grokService.saveTemporary(statement, "squid"); + } + + @Test + public void missingGrokStatementShouldThrowRestException() throws Exception { + exception.expect(RestException.class); + exception.expectMessage("A grokStatement must be provided"); + + grokService.saveTemporary(null, "squid"); + } +} \ No newline at end of file diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java new file mode 100644 index 0000000000..d11f5a4b19 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.HdfsService; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.io.IOException; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.powermock.api.mockito.PowerMockito.mock; +import static org.powermock.api.mockito.PowerMockito.mockStatic; +import static org.powermock.api.mockito.PowerMockito.when; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({HdfsServiceImpl.class, FileSystem.class}) +public class HdfsServiceImplExceptionTest { + @Rule + public final ExpectedException exception = ExpectedException.none(); + + private Configuration configuration; + private HdfsService hdfsService; + private String testDir = "./target/hdfsUnitTest"; + + @Before + public void setup() throws IOException { + configuration = new Configuration(); + hdfsService = new HdfsServiceImpl(configuration); + + mockStatic(FileSystem.class); + } + + @Test + public void listShouldWrapExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + FileSystem fileSystem = mock(FileSystem.class); + when(FileSystem.get(configuration)).thenReturn(fileSystem); + when(fileSystem.listStatus(new Path(testDir))).thenThrow(new IOException()); + + hdfsService.list(new Path(testDir)); + } + + @Test + public void readShouldWrapExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + FileSystem fileSystem = mock(FileSystem.class); + when(FileSystem.get(configuration)).thenReturn(fileSystem); + when(fileSystem.open(new Path(testDir))).thenThrow(new IOException()); + + hdfsService.read(new Path(testDir)); + } + + @Test + public void writeShouldWrapExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + FileSystem fileSystem = mock(FileSystem.class); + when(FileSystem.get(configuration)).thenReturn(fileSystem); + when(fileSystem.create(new Path(testDir), true)).thenThrow(new IOException()); + + hdfsService.write(new Path(testDir), "contents".getBytes(UTF_8)); + } + + @Test + public void deleteShouldWrapExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + FileSystem fileSystem = mock(FileSystem.class); + when(FileSystem.get(configuration)).thenReturn(fileSystem); + when(fileSystem.delete(new Path(testDir), false)).thenThrow(new IOException()); + + hdfsService.delete(new Path(testDir), false); + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java new file mode 100644 index 0000000000..9c18dc6b8e --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.metron.rest.service.HdfsService; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; + +public class HdfsServiceImplTest { + @Rule + public final ExpectedException exception = ExpectedException.none(); + + private Configuration configuration; + private HdfsService hdfsService; + private String testDir = "./target/hdfsUnitTest"; + + @Before + public void setup() throws IOException { + configuration = new Configuration(); + hdfsService = new HdfsServiceImpl(configuration); + File file = new File(testDir); + if (!file.exists()) { + file.mkdirs(); + } + FileUtils.cleanDirectory(file); + } + + @After + public void teardown() throws IOException { + File file = new File(testDir); + FileUtils.cleanDirectory(file); + } + + @Test + public void listShouldListFiles() throws Exception { + FileUtils.writeStringToFile(new File(testDir, "file1.txt"), "value1"); + FileUtils.writeStringToFile(new File(testDir, "file2.txt"), "value2"); + + List paths = hdfsService.list(new Path(testDir)); + assertEquals(2, paths.size()); + assertEquals("file1.txt", paths.get(0)); + assertEquals("file2.txt", paths.get(1)); + } + + + @Test + public void readShouldProperlyReadContents() throws Exception { + String contents = "contents"; + FileUtils.writeStringToFile(new File(testDir, "readTest.txt"), contents); + + assertEquals("contents", hdfsService.read(new Path(testDir, "readTest.txt"))); + } + + @Test + public void writeShouldProperlyWriteContents() throws Exception { + String contents = "contents"; + hdfsService.write(new Path(testDir, "writeTest.txt"), contents.getBytes(UTF_8)); + + assertEquals("contents", FileUtils.readFileToString(new File(testDir, "writeTest.txt"))); + } + + @Test + public void deleteShouldProperlyDeleteFile() throws Exception { + String contents = "contents"; + FileUtils.writeStringToFile(new File(testDir, "deleteTest.txt"), contents); + + List paths = hdfsService.list(new Path(testDir)); + assertEquals(1, paths.size()); + assertEquals("deleteTest.txt", paths.get(0)); + + hdfsService.delete(new Path(testDir, "deleteTest.txt"), false); + + paths = hdfsService.list(new Path(testDir)); + assertEquals(0, paths.size()); + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java new file mode 100644 index 0000000000..c7d42b3164 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import kafka.admin.AdminOperationException; +import kafka.admin.AdminUtils$; +import kafka.admin.RackAwareMode; +import kafka.utils.ZkUtils; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.KafkaTopic; +import org.apache.metron.rest.service.KafkaService; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PowerMockIgnore; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.mock; + +@SuppressWarnings("unchecked") +@RunWith(PowerMockRunner.class) +@PowerMockIgnore("javax.management.*") // resolve classloader conflict +@PrepareForTest({AdminUtils$.class}) +public class KafkaServiceImplTest { + @Rule + public final ExpectedException exception = ExpectedException.none(); + + private ZkUtils zkUtils; + private KafkaConsumer kafkaConsumer; + private AdminUtils$ adminUtils; + + private KafkaService kafkaService; + + private static final KafkaTopic VALID_KAFKA_TOPIC = new KafkaTopic() {{ + setReplicationFactor(2); + setNumPartitions(1); + setName("t"); + setProperties(new Properties()); + }}; + + @SuppressWarnings("unchecked") + @Before + public void setUp() throws Exception { + zkUtils = mock(ZkUtils.class); + kafkaConsumer = mock(KafkaConsumer.class); + adminUtils = mock(AdminUtils$.class); + + kafkaService = new KafkaServiceImpl(zkUtils, kafkaConsumer, adminUtils); + } + + @Test + public void listTopicsHappyPathWithListTopicsReturningNull() throws Exception { + final Map> topics = null; + + when(kafkaConsumer.listTopics()).thenReturn(topics); + + final Set listedTopics = kafkaService.listTopics(); + + assertEquals(Sets.newHashSet(), listedTopics); + + verifyZeroInteractions(zkUtils); + verify(kafkaConsumer).listTopics(); + verifyNoMoreInteractions(kafkaConsumer, zkUtils, adminUtils); + } + + @Test + public void listTopicsHappyPathWithListTopicsReturningEmptyMap() throws Exception { + final Map> topics = new HashMap<>(); + + when(kafkaConsumer.listTopics()).thenReturn(topics); + + final Set listedTopics = kafkaService.listTopics(); + + assertEquals(Sets.newHashSet(), listedTopics); + + verifyZeroInteractions(zkUtils); + verify(kafkaConsumer).listTopics(); + verifyNoMoreInteractions(kafkaConsumer, zkUtils); + } + + @Test + public void listTopicsHappyPath() throws Exception { + final Map> topics = new HashMap<>(); + topics.put("topic1", Lists.newArrayList()); + topics.put("topic2", Lists.newArrayList()); + topics.put("topic3", Lists.newArrayList()); + + when(kafkaConsumer.listTopics()).thenReturn(topics); + + final Set listedTopics = kafkaService.listTopics(); + + assertEquals(Sets.newHashSet("topic1", "topic2", "topic3"), listedTopics); + + verifyZeroInteractions(zkUtils); + verify(kafkaConsumer).listTopics(); + verifyNoMoreInteractions(kafkaConsumer, zkUtils); + } + + @Test + public void listTopicsShouldProperlyRemoveOffsetTopic() throws Exception { + final Map> topics = new HashMap<>(); + topics.put("topic1", Lists.newArrayList()); + topics.put("topic2", Lists.newArrayList()); + topics.put("topic3", Lists.newArrayList()); + topics.put("__consumer_offsets", Lists.newArrayList()); + + when(kafkaConsumer.listTopics()).thenReturn(topics); + + final Set listedTopics = kafkaService.listTopics(); + + assertEquals(Sets.newHashSet("topic1", "topic2", "topic3"), listedTopics); + + verifyZeroInteractions(zkUtils); + verify(kafkaConsumer).listTopics(); + verifyNoMoreInteractions(kafkaConsumer, zkUtils); + } + + @Test + public void deletingTopicThatDoesNotExistShouldReturnFalse() throws Exception { + when(kafkaConsumer.listTopics()).thenReturn(Maps.newHashMap()); + + assertFalse(kafkaService.deleteTopic("non_existent_topic")); + + verifyZeroInteractions(zkUtils); + verify(kafkaConsumer).listTopics(); + verifyNoMoreInteractions(kafkaConsumer, zkUtils); + } + + @Test + public void deletingTopicThatExistShouldReturnTrue() throws Exception { + final Map> topics = new HashMap<>(); + topics.put("non_existent_topic", Lists.newArrayList()); + + when(kafkaConsumer.listTopics()).thenReturn(topics); + + assertTrue(kafkaService.deleteTopic("non_existent_topic")); + + verify(kafkaConsumer).listTopics(); + verify(adminUtils).deleteTopic(zkUtils, "non_existent_topic"); + verifyNoMoreInteractions(kafkaConsumer); + } + + @Test + public void makeSureDeletingTopicReturnsFalseWhenNoTopicsExist() throws Exception { + final Map> topics = null; + + when(kafkaConsumer.listTopics()).thenReturn(topics); + + assertFalse(kafkaService.deleteTopic("non_existent_topic")); + + verify(kafkaConsumer).listTopics(); + verifyNoMoreInteractions(kafkaConsumer); + } + + @Test + public void getTopicShouldProperlyMapTopicToKafkaTopic() throws Exception { + final PartitionInfo partitionInfo = mock(PartitionInfo.class); + when(partitionInfo.replicas()).thenReturn(new Node[] {new Node(1, "host", 8080)}); + + final Map> topics = new HashMap<>(); + topics.put("t", Lists.newArrayList(partitionInfo)); + topics.put("t1", Lists.newArrayList()); + + final KafkaTopic expected = new KafkaTopic(); + expected.setName("t"); + expected.setNumPartitions(1); + expected.setReplicationFactor(1); + + when(kafkaConsumer.listTopics()).thenReturn(topics); + when(kafkaConsumer.partitionsFor("t")).thenReturn(Lists.newArrayList(partitionInfo)); + + KafkaTopic actual = kafkaService.getTopic("t"); + assertEquals(expected, actual); + assertEquals(expected.hashCode(), actual.hashCode()); + } + + @Test + public void getTopicShouldProperlyHandleTopicsThatDontExist() throws Exception { + final Map> topics = new HashMap<>(); + topics.put("t1", Lists.newArrayList()); + + when(kafkaConsumer.listTopics()).thenReturn(topics); + when(kafkaConsumer.partitionsFor("t")).thenReturn(Lists.newArrayList()); + + assertEquals(null, kafkaService.getTopic("t")); + + verify(kafkaConsumer).listTopics(); + verify(kafkaConsumer, times(0)).partitionsFor("t"); + verifyZeroInteractions(zkUtils); + verifyNoMoreInteractions(kafkaConsumer); + } + + @Test + public void createTopicShouldFailIfReplicationFactorIsGreaterThanAvailableBrokers() throws Exception { + final Map> topics = new HashMap<>(); + + when(kafkaConsumer.listTopics()).thenReturn(topics); + + kafkaService.createTopic(VALID_KAFKA_TOPIC); + + verify(adminUtils).createTopic(eq(zkUtils), eq("t"), eq(1), eq(2), eq(new Properties()), eq(RackAwareMode.Disabled$.MODULE$)); + verify(kafkaConsumer).listTopics(); + verifyZeroInteractions(zkUtils); + } + + @Test + public void whenAdminUtilsThrowsAdminOperationExceptionCreateTopicShouldProperlyWrapExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + final Map> topics = new HashMap<>(); + topics.put("1", new ArrayList<>()); + + when(kafkaConsumer.listTopics()).thenReturn(topics); + + doThrow(AdminOperationException.class).when(adminUtils).createTopic(eq(zkUtils), eq("t"), eq(1), eq(2), eq(new Properties()), eq(RackAwareMode.Disabled$.MODULE$)); + + kafkaService.createTopic(VALID_KAFKA_TOPIC); + } + + @Test + public void getSampleMessageProperlyReturnsAMessageFromAGivenKafkaTopic() throws Exception { + final String topicName = "t"; + final Node host = new Node(1, "host", 8080); + final Node[] replicas = {host}; + final List partitionInfo = Lists.newArrayList(new PartitionInfo(topicName, 1, host, replicas, replicas)); + final TopicPartition topicPartition = new TopicPartition(topicName, 1); + final List topicPartitions = Lists.newArrayList(topicPartition); + final Set topicPartitionsSet = Sets.newHashSet(topicPartitions); + final ConsumerRecords records = new ConsumerRecords<>(new HashMap>>() {{ + put(topicPartition, Lists.newArrayList(new ConsumerRecord<>(topicName, 1, 1, "k", "message"))); + }}); + + when(kafkaConsumer.listTopics()).thenReturn(new HashMap>() {{ put(topicName, Lists.newArrayList()); }}); + when(kafkaConsumer.partitionsFor(eq(topicName))).thenReturn(partitionInfo); + when(kafkaConsumer.assignment()).thenReturn(topicPartitionsSet); + when(kafkaConsumer.position(topicPartition)).thenReturn(1L); + when(kafkaConsumer.poll(100)).thenReturn(records); + + assertEquals("message", kafkaService.getSampleMessage(topicName)); + + verify(kafkaConsumer).assign(eq(topicPartitions)); + verify(kafkaConsumer).assignment(); + verify(kafkaConsumer).poll(100); + verify(kafkaConsumer).unsubscribe(); + verify(kafkaConsumer, times(2)).position(topicPartition); + verify(kafkaConsumer).seek(topicPartition, 0); + + verifyZeroInteractions(zkUtils, adminUtils); + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImplTest.java new file mode 100644 index 0000000000..d2929487cc --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorEnrichmentConfigServiceImplTest.java @@ -0,0 +1,256 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.api.DeleteBuilder; +import org.apache.curator.framework.api.GetChildrenBuilder; +import org.apache.curator.framework.api.GetDataBuilder; +import org.apache.curator.framework.api.SetDataBuilder; +import org.apache.metron.common.configuration.ConfigurationType; +import org.apache.metron.common.configuration.enrichment.EnrichmentConfig; +import org.apache.metron.common.configuration.enrichment.SensorEnrichmentConfig; +import org.apache.metron.common.configuration.enrichment.threatintel.ThreatIntelConfig; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.SensorEnrichmentConfigService; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.data.Stat; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@SuppressWarnings("ALL") +public class SensorEnrichmentConfigServiceImplTest { + @Rule + public final ExpectedException exception = ExpectedException.none(); + + ObjectMapper objectMapper; + CuratorFramework curatorFramework; + SensorEnrichmentConfigService sensorEnrichmentConfigService; + + /** + { + "enrichment" : { + "fieldMap": { + "geo": ["ip_dst_addr"] + } + }, + "threatIntel": { + "fieldMap": { + "hbaseThreatIntel": ["ip_src_addr"] + }, + "fieldToTypeMap": { + "ip_src_addr" : ["malicious_ip"] + } + } + } + */ + @Multiline + public static String broJson; + + @Before + public void setUp() throws Exception { + objectMapper = mock(ObjectMapper.class); + curatorFramework = mock(CuratorFramework.class); + sensorEnrichmentConfigService = new SensorEnrichmentConfigServiceImpl(objectMapper, curatorFramework); + } + + + @Test + public void deleteShouldProperlyCatchNoNodeExceptionAndReturnFalse() throws Exception { + DeleteBuilder builder = mock(DeleteBuilder.class); + + when(curatorFramework.delete()).thenReturn(builder); + when(builder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro")).thenThrow(KeeperException.NoNodeException.class); + + assertFalse(sensorEnrichmentConfigService.delete("bro")); + } + + @Test + public void deleteShouldProperlyCatchNonNoNodeExceptionAndThrowRestException() throws Exception { + exception.expect(RestException.class); + + DeleteBuilder builder = mock(DeleteBuilder.class); + + when(curatorFramework.delete()).thenReturn(builder); + when(builder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro")).thenThrow(Exception.class); + + assertFalse(sensorEnrichmentConfigService.delete("bro")); + } + + @Test + public void deleteShouldReturnTrueWhenClientSuccessfullyCallsDelete() throws Exception { + DeleteBuilder builder = mock(DeleteBuilder.class); + + when(curatorFramework.delete()).thenReturn(builder); + when(builder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro")).thenReturn(null); + + assertTrue(sensorEnrichmentConfigService.delete("bro")); + + verify(curatorFramework).delete(); + } + + @Test + public void findOneShouldProperlyReturnSensorEnrichmentConfig() throws Exception { + final SensorEnrichmentConfig sensorEnrichmentConfig = getTestSensorEnrichmentConfig(); + + GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); + when(getDataBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro")).thenReturn(broJson.getBytes()); + when(curatorFramework.getData()).thenReturn(getDataBuilder); + + assertEquals(getTestSensorEnrichmentConfig(), sensorEnrichmentConfigService.findOne("bro")); + } + + @Test + public void findOneShouldReturnNullWhenNoNodeExceptionIsThrown() throws Exception { + GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); + when(getDataBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro")).thenThrow(KeeperException.NoNodeException.class); + + when(curatorFramework.getData()).thenReturn(getDataBuilder); + + assertNull(sensorEnrichmentConfigService.findOne("bro")); + } + + @Test + public void findOneShouldWrapNonNoNodeExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); + when(getDataBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro")).thenThrow(Exception.class); + + when(curatorFramework.getData()).thenReturn(getDataBuilder); + + sensorEnrichmentConfigService.findOne("bro"); + } + + @Test + public void getAllTypesShouldProperlyReturnTypes() throws Exception { + GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class); + when(getChildrenBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot())) + .thenReturn(new ArrayList() {{ + add("bro"); + add("squid"); + }}); + when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder); + + assertEquals(new ArrayList() {{ + add("bro"); + add("squid"); + }}, sensorEnrichmentConfigService.getAllTypes()); + } + + @Test + public void getAllTypesShouldReturnNullWhenNoNodeExceptionIsThrown() throws Exception { + GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class); + when(getChildrenBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot())).thenThrow(KeeperException.NoNodeException.class); + when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder); + + assertEquals(new ArrayList<>(), sensorEnrichmentConfigService.getAllTypes()); + } + + @Test + public void getAllTypesShouldWrapNonNoNodeExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class); + when(getChildrenBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot())).thenThrow(Exception.class); + when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder); + + sensorEnrichmentConfigService.getAllTypes(); + } + + @Test + public void getAllShouldProperlyReturnSensorEnrichmentConfigs() throws Exception { + GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class); + when(getChildrenBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot())) + .thenReturn(new ArrayList() {{ + add("bro"); + }}); + when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder); + + final SensorEnrichmentConfig sensorEnrichmentConfig = getTestSensorEnrichmentConfig(); + GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); + when(getDataBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro")).thenReturn(broJson.getBytes()); + when(curatorFramework.getData()).thenReturn(getDataBuilder); + + assertEquals(new HashMap() {{ put("bro", sensorEnrichmentConfig);}}, sensorEnrichmentConfigService.getAll()); + } + + @Test + public void saveShouldWrapExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + SetDataBuilder setDataBuilder = mock(SetDataBuilder.class); + when(setDataBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro", broJson.getBytes())).thenThrow(Exception.class); + + when(curatorFramework.setData()).thenReturn(setDataBuilder); + + sensorEnrichmentConfigService.save("bro", new SensorEnrichmentConfig()); + } + + @Test + public void saveShouldReturnSameConfigThatIsPassedOnSuccessfulSave() throws Exception { + final SensorEnrichmentConfig sensorEnrichmentConfig = getTestSensorEnrichmentConfig(); + + when(objectMapper.writeValueAsString(sensorEnrichmentConfig)).thenReturn(broJson); + + SetDataBuilder setDataBuilder = mock(SetDataBuilder.class); + when(setDataBuilder.forPath(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro", broJson.getBytes())).thenReturn(new Stat()); + when(curatorFramework.setData()).thenReturn(setDataBuilder); + + assertEquals(sensorEnrichmentConfig, sensorEnrichmentConfigService.save("bro", sensorEnrichmentConfig)); + verify(setDataBuilder).forPath(eq(ConfigurationType.ENRICHMENT.getZookeeperRoot() + "/bro"), eq(broJson.getBytes())); + } + + @Test + public void getAvailableEnrichmentsShouldReturnEnrichments() throws Exception { + assertEquals(new ArrayList() {{ + add("geo"); + add("host"); + add("whois"); + }}, sensorEnrichmentConfigService.getAvailableEnrichments()); + } + + private SensorEnrichmentConfig getTestSensorEnrichmentConfig() { + SensorEnrichmentConfig sensorEnrichmentConfig = new SensorEnrichmentConfig(); + EnrichmentConfig enrichmentConfig = new EnrichmentConfig(); + enrichmentConfig.setFieldMap(new HashMap() {{ put("geo", Arrays.asList("ip_dst_addr")); }}); + sensorEnrichmentConfig.setEnrichment(enrichmentConfig); + ThreatIntelConfig threatIntelConfig = new ThreatIntelConfig(); + threatIntelConfig.setFieldMap(new HashMap() {{ put("hbaseThreatIntel", Arrays.asList("ip_src_addr")); }}); + threatIntelConfig.setFieldToTypeMap(new HashMap() {{ put("ip_src_addr", Arrays.asList("malicious_ip")); }}); + sensorEnrichmentConfig.setThreatIntel(threatIntelConfig); + return sensorEnrichmentConfig; + } + } diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java new file mode 100644 index 0000000000..43ca0f75ee --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorIndexingConfigServiceImplTest.java @@ -0,0 +1,234 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.api.DeleteBuilder; +import org.apache.curator.framework.api.GetChildrenBuilder; +import org.apache.curator.framework.api.GetDataBuilder; +import org.apache.curator.framework.api.SetDataBuilder; +import org.apache.metron.common.configuration.ConfigurationType; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.service.SensorIndexingConfigService; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.data.Stat; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@SuppressWarnings("ALL") +public class SensorIndexingConfigServiceImplTest { + @Rule + public final ExpectedException exception = ExpectedException.none(); + + ObjectMapper objectMapper; + CuratorFramework curatorFramework; + SensorIndexingConfigService sensorIndexingConfigService; + + /** + { + "hdfs" : { + "index": "bro", + "batchSize": 5, + "enabled" : true + } + } + */ + @Multiline + public static String broJson; + + @Before + public void setUp() throws Exception { + objectMapper = mock(ObjectMapper.class); + curatorFramework = mock(CuratorFramework.class); + sensorIndexingConfigService = new SensorIndexingConfigServiceImpl(objectMapper, curatorFramework); + } + + + @Test + public void deleteShouldProperlyCatchNoNodeExceptionAndReturnFalse() throws Exception { + DeleteBuilder builder = mock(DeleteBuilder.class); + + when(curatorFramework.delete()).thenReturn(builder); + when(builder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro")).thenThrow(KeeperException.NoNodeException.class); + + assertFalse(sensorIndexingConfigService.delete("bro")); + } + + @Test + public void deleteShouldProperlyCatchNonNoNodeExceptionAndThrowRestException() throws Exception { + exception.expect(RestException.class); + + DeleteBuilder builder = mock(DeleteBuilder.class); + + when(curatorFramework.delete()).thenReturn(builder); + when(builder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro")).thenThrow(Exception.class); + + assertFalse(sensorIndexingConfigService.delete("bro")); + } + + @Test + public void deleteShouldReturnTrueWhenClientSuccessfullyCallsDelete() throws Exception { + DeleteBuilder builder = mock(DeleteBuilder.class); + + when(curatorFramework.delete()).thenReturn(builder); + when(builder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro")).thenReturn(null); + + assertTrue(sensorIndexingConfigService.delete("bro")); + + verify(curatorFramework).delete(); + } + + @Test + public void findOneShouldProperlyReturnSensorEnrichmentConfig() throws Exception { + final Map sensorIndexingConfig = getTestSensorIndexingConfig(); + + GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); + when(getDataBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro")).thenReturn(broJson.getBytes()); + when(curatorFramework.getData()).thenReturn(getDataBuilder); + + assertEquals(getTestSensorIndexingConfig(), sensorIndexingConfigService.findOne("bro")); + } + + @Test + public void findOneShouldReturnNullWhenNoNodeExceptionIsThrown() throws Exception { + GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); + when(getDataBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro")).thenThrow(KeeperException.NoNodeException.class); + + when(curatorFramework.getData()).thenReturn(getDataBuilder); + + assertNull(sensorIndexingConfigService.findOne("bro")); + } + + @Test + public void findOneShouldWrapNonNoNodeExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); + when(getDataBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro")).thenThrow(Exception.class); + + when(curatorFramework.getData()).thenReturn(getDataBuilder); + + sensorIndexingConfigService.findOne("bro"); + } + + @Test + public void getAllTypesShouldProperlyReturnTypes() throws Exception { + GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class); + when(getChildrenBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot())) + .thenReturn(new ArrayList() {{ + add("bro"); + add("squid"); + }}); + when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder); + + assertEquals(new ArrayList() {{ + add("bro"); + add("squid"); + }}, sensorIndexingConfigService.getAllTypes()); + } + + @Test + public void getAllTypesShouldReturnNullWhenNoNodeExceptionIsThrown() throws Exception { + GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class); + when(getChildrenBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot())).thenThrow(KeeperException.NoNodeException.class); + when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder); + + assertEquals(new ArrayList<>(), sensorIndexingConfigService.getAllTypes()); + } + + @Test + public void getAllTypesShouldWrapNonNoNodeExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class); + when(getChildrenBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot())).thenThrow(Exception.class); + when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder); + + sensorIndexingConfigService.getAllTypes(); + } + + @Test + public void getAllShouldProperlyReturnSensorEnrichmentConfigs() throws Exception { + final Map sensorIndexingConfig = getTestSensorIndexingConfig(); + + GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class); + when(getChildrenBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot())) + .thenReturn(new ArrayList() {{ + add("bro"); + }}); + when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder); + GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); + when(getDataBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro")).thenReturn(broJson.getBytes()); + when(curatorFramework.getData()).thenReturn(getDataBuilder); + + assertEquals(new HashMap() {{ put("bro", sensorIndexingConfig);}}, sensorIndexingConfigService.getAll()); + } + + @Test + public void saveShouldWrapExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + SetDataBuilder setDataBuilder = mock(SetDataBuilder.class); + when(setDataBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro", broJson.getBytes())).thenThrow(Exception.class); + + when(curatorFramework.setData()).thenReturn(setDataBuilder); + + sensorIndexingConfigService.save("bro", new HashMap<>()); + } + + @Test + public void saveShouldReturnSameConfigThatIsPassedOnSuccessfulSave() throws Exception { + final Map sensorIndexingConfig = getTestSensorIndexingConfig(); + + when(objectMapper.writeValueAsString(sensorIndexingConfig)).thenReturn(broJson); + + SetDataBuilder setDataBuilder = mock(SetDataBuilder.class); + when(setDataBuilder.forPath(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro", broJson.getBytes())).thenReturn(new Stat()); + when(curatorFramework.setData()).thenReturn(setDataBuilder); + + assertEquals(sensorIndexingConfig, sensorIndexingConfigService.save("bro", sensorIndexingConfig)); + verify(setDataBuilder).forPath(eq(ConfigurationType.INDEXING.getZookeeperRoot() + "/bro"), eq(broJson.getBytes())); + } + + private Map getTestSensorIndexingConfig() { + Map sensorIndexingConfig = new HashMap<>(); + sensorIndexingConfig.put("hdfs", new HashMap(){{ + put("index", "bro"); + put("batchSize", 5); + put("enabled", true); + }}); + return sensorIndexingConfig; + } + } diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java new file mode 100644 index 0000000000..d35a48c37c --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImplTest.java @@ -0,0 +1,344 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.api.DeleteBuilder; +import org.apache.curator.framework.api.GetChildrenBuilder; +import org.apache.curator.framework.api.GetDataBuilder; +import org.apache.curator.framework.api.SetDataBuilder; +import org.apache.metron.common.configuration.ConfigurationType; +import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.rest.RestException; +import org.apache.metron.rest.model.ParseMessageRequest; +import org.apache.metron.rest.service.GrokService; +import org.apache.metron.rest.service.SensorParserConfigService; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.data.Stat; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.springframework.core.env.Environment; + +import java.io.File; +import java.io.FileWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@SuppressWarnings("ALL") +public class SensorParserConfigServiceImplTest { + @Rule + public final ExpectedException exception = ExpectedException.none(); + + Environment environment; + ObjectMapper objectMapper; + CuratorFramework curatorFramework; + GrokService grokService; + SensorParserConfigService sensorParserConfigService; + + /** + { + "parserClassName": "org.apache.metron.parsers.GrokParser", + "sensorTopic": "squid", + "parserConfig": { + "grokPath": "/patterns/squid", + "patternLabel": "SQUID_DELIMITED", + "timestampField": "timestamp" + } + } + */ + @Multiline + public static String squidJson; + + /** + { + "parserClassName":"org.apache.metron.parsers.bro.BasicBroParser", + "sensorTopic":"bro", + "parserConfig": {} + } + */ + @Multiline + public static String broJson; + + @Before + public void setUp() throws Exception { + objectMapper = mock(ObjectMapper.class); + curatorFramework = mock(CuratorFramework.class); + grokService = mock(GrokService.class); + sensorParserConfigService = new SensorParserConfigServiceImpl(objectMapper, curatorFramework, grokService); + } + + + @Test + public void deleteShouldProperlyCatchNoNodeExceptionAndReturnFalse() throws Exception { + DeleteBuilder builder = mock(DeleteBuilder.class); + + when(curatorFramework.delete()).thenReturn(builder); + when(builder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenThrow(KeeperException.NoNodeException.class); + + assertFalse(sensorParserConfigService.delete("bro")); + } + + @Test + public void deleteShouldProperlyCatchNonNoNodeExceptionAndThrowRestException() throws Exception { + exception.expect(RestException.class); + + DeleteBuilder builder = mock(DeleteBuilder.class); + + when(curatorFramework.delete()).thenReturn(builder); + when(builder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenThrow(Exception.class); + + assertFalse(sensorParserConfigService.delete("bro")); + } + + @Test + public void deleteShouldReturnTrueWhenClientSuccessfullyCallsDelete() throws Exception { + DeleteBuilder builder = mock(DeleteBuilder.class); + + when(curatorFramework.delete()).thenReturn(builder); + when(builder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenReturn(null); + + assertTrue(sensorParserConfigService.delete("bro")); + + verify(curatorFramework).delete(); + } + + @Test + public void findOneShouldProperlyReturnSensorEnrichmentConfig() throws Exception { + final SensorParserConfig sensorParserConfig = getTestBroSensorParserConfig(); + + GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); + when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenReturn(broJson.getBytes()); + when(curatorFramework.getData()).thenReturn(getDataBuilder); + + assertEquals(getTestBroSensorParserConfig(), sensorParserConfigService.findOne("bro")); + } + + @Test + public void findOneShouldReturnNullWhenNoNodeExceptionIsThrown() throws Exception { + GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); + when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenThrow(KeeperException.NoNodeException.class); + + when(curatorFramework.getData()).thenReturn(getDataBuilder); + + assertNull(sensorParserConfigService.findOne("bro")); + } + + @Test + public void findOneShouldWrapNonNoNodeExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); + when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenThrow(Exception.class); + + when(curatorFramework.getData()).thenReturn(getDataBuilder); + + sensorParserConfigService.findOne("bro"); + } + + @Test + public void getAllTypesShouldProperlyReturnTypes() throws Exception { + GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class); + when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot())) + .thenReturn(new ArrayList() {{ + add("bro"); + add("squid"); + }}); + when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder); + + assertEquals(new ArrayList() {{ + add("bro"); + add("squid"); + }}, sensorParserConfigService.getAllTypes()); + } + + @Test + public void getAllTypesShouldReturnEmptyListWhenNoNodeExceptionIsThrown() throws Exception { + GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class); + when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot())).thenThrow(KeeperException.NoNodeException.class); + when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder); + + assertEquals(new ArrayList<>(), sensorParserConfigService.getAllTypes()); + } + + @Test + public void getAllTypesShouldWrapNonNoNodeExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class); + when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot())).thenThrow(Exception.class); + when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder); + + sensorParserConfigService.getAllTypes(); + } + + @Test + public void getAllShouldProperlyReturnSensorParserConfigs() throws Exception { + GetChildrenBuilder getChildrenBuilder = mock(GetChildrenBuilder.class); + when(getChildrenBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot())) + .thenReturn(new ArrayList() {{ + add("bro"); + add("squid"); + }}); + when(curatorFramework.getChildren()).thenReturn(getChildrenBuilder); + + final SensorParserConfig broSensorParserConfig = getTestBroSensorParserConfig(); + final SensorParserConfig squidSensorParserConfig = getTestSquidSensorParserConfig(); + GetDataBuilder getDataBuilder = mock(GetDataBuilder.class); + when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro")).thenReturn(broJson.getBytes()); + when(getDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/squid")).thenReturn(squidJson.getBytes()); + when(curatorFramework.getData()).thenReturn(getDataBuilder); + + assertEquals(new ArrayList() {{ + add(getTestBroSensorParserConfig()); + add(getTestSquidSensorParserConfig()); + }}, sensorParserConfigService.getAll()); + } + + @Test + public void saveShouldWrapExceptionInRestException() throws Exception { + exception.expect(RestException.class); + + SetDataBuilder setDataBuilder = mock(SetDataBuilder.class); + when(setDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro", broJson.getBytes())).thenThrow(Exception.class); + + when(curatorFramework.setData()).thenReturn(setDataBuilder); + + final SensorParserConfig sensorParserConfig = new SensorParserConfig(); + sensorParserConfig.setSensorTopic("bro"); + sensorParserConfigService.save(sensorParserConfig); + } + + @Test + public void saveShouldReturnSameConfigThatIsPassedOnSuccessfulSave() throws Exception { + final SensorParserConfig sensorParserConfig = getTestBroSensorParserConfig(); + + when(objectMapper.writeValueAsString(sensorParserConfig)).thenReturn(broJson); + + SetDataBuilder setDataBuilder = mock(SetDataBuilder.class); + when(setDataBuilder.forPath(ConfigurationType.PARSER.getZookeeperRoot() + "/bro", broJson.getBytes())).thenReturn(new Stat()); + when(curatorFramework.setData()).thenReturn(setDataBuilder); + + assertEquals(getTestBroSensorParserConfig(), sensorParserConfigService.save(sensorParserConfig)); + verify(setDataBuilder).forPath(eq(ConfigurationType.PARSER.getZookeeperRoot() + "/bro"), eq(broJson.getBytes())); + } + + @Test + public void reloadAvailableParsersShouldReturnParserClasses() throws Exception { + Map availableParsers = sensorParserConfigService.reloadAvailableParsers(); + assertTrue(availableParsers.size() > 0); + assertEquals("org.apache.metron.parsers.GrokParser", availableParsers.get("Grok")); + assertEquals("org.apache.metron.parsers.bro.BasicBroParser", availableParsers.get("Bro")); + } + + @Test + public void parseMessageShouldProperlyReturnParsedResults() throws Exception { + final SensorParserConfig sensorParserConfig = getTestSquidSensorParserConfig(); + String grokStatement = "SQUID_DELIMITED %{NUMBER:timestamp}[^0-9]*%{INT:elapsed} %{IP:ip_src_addr} %{WORD:action}/%{NUMBER:code} %{NUMBER:bytes} %{WORD:method} %{NOTSPACE:url}[^0-9]*(%{IP:ip_dst_addr})?"; + String sampleData = "1461576382.642 161 127.0.0.1 TCP_MISS/200 103701 GET http://www.cnn.com/ - DIRECT/199.27.79.73 text/html"; + ParseMessageRequest parseMessageRequest = new ParseMessageRequest(); + parseMessageRequest.setSensorParserConfig(sensorParserConfig); + parseMessageRequest.setGrokStatement(grokStatement); + parseMessageRequest.setSampleData(sampleData); + + File patternFile = new File("./target/squidTest"); + FileWriter writer = new FileWriter(patternFile); + writer.write(grokStatement); + writer.close(); + + when(grokService.saveTemporary(grokStatement, "squid")).thenReturn(patternFile); + + assertEquals(new HashMap() {{ + put("elapsed", 161); + put("code", 200); + put("ip_dst_addr", "199.27.79.73"); + put("ip_src_addr", "127.0.0.1"); + put("action", "TCP_MISS"); + put("bytes", 103701); + put("method", "GET"); + put("url", "http://www.cnn.com/"); + put("timestamp", 1461576382642L); + put("original_string", "1461576382.642 161 127.0.0.1 TCP_MISS/200 103701 GET http://www.cnn.com/ - DIRECT/199.27.79.73 text/html"); + }}, sensorParserConfigService.parseMessage(parseMessageRequest)); + + } + + @Test + public void missingSensorParserConfigShouldThrowRestException() throws Exception { + exception.expect(RestException.class); + + ParseMessageRequest parseMessageRequest = new ParseMessageRequest(); + sensorParserConfigService.parseMessage(parseMessageRequest); + } + + @Test + public void missingParserClassShouldThrowRestException() throws Exception { + exception.expect(RestException.class); + + final SensorParserConfig sensorParserConfig = new SensorParserConfig(); + sensorParserConfig.setSensorTopic("squid"); + ParseMessageRequest parseMessageRequest = new ParseMessageRequest(); + parseMessageRequest.setSensorParserConfig(sensorParserConfig); + sensorParserConfigService.parseMessage(parseMessageRequest); + } + + @Test + public void invalidParserClassShouldThrowRestException() throws Exception { + exception.expect(RestException.class); + + final SensorParserConfig sensorParserConfig = new SensorParserConfig(); + sensorParserConfig.setSensorTopic("squid"); + sensorParserConfig.setParserClassName("bad.class.package.BadClassName"); + ParseMessageRequest parseMessageRequest = new ParseMessageRequest(); + parseMessageRequest.setSensorParserConfig(sensorParserConfig); + sensorParserConfigService.parseMessage(parseMessageRequest); + } + + private SensorParserConfig getTestBroSensorParserConfig() { + SensorParserConfig sensorParserConfig = new SensorParserConfig(); + sensorParserConfig.setSensorTopic("bro"); + sensorParserConfig.setParserClassName("org.apache.metron.parsers.bro.BasicBroParser"); + return sensorParserConfig; + } + + private SensorParserConfig getTestSquidSensorParserConfig() { + SensorParserConfig sensorParserConfig = new SensorParserConfig(); + sensorParserConfig.setSensorTopic("squid"); + sensorParserConfig.setParserClassName("org.apache.metron.parsers.GrokParser"); + sensorParserConfig.setParserConfig(new HashMap() {{ + put("grokPath", "/patterns/squid"); + put("patternLabel", "SQUID_DELIMITED"); + put("timestampField", "timestamp"); + }}); + return sensorParserConfig; + } + + } diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StellarServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StellarServiceImplTest.java new file mode 100644 index 0000000000..3ddbf5644f --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StellarServiceImplTest.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.metron.common.configuration.FieldTransformer; +import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.rest.model.SensorParserContext; +import org.apache.metron.rest.service.StellarService; +import org.apache.storm.shade.com.google.common.collect.ImmutableList; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class StellarServiceImplTest { + + private StellarService stellarService; + + @Before + public void setUp() throws Exception { + stellarService = new StellarServiceImpl(); + } + + @Test + public void validateRulesShouldProperlyValidateRules() { + List rules = Arrays.asList("TO_LOWER(test)", "BAD_FUNCTION(test)"); + Map results = stellarService.validateRules(rules); + assertEquals(2, results.size()); + assertEquals(true, results.get("TO_LOWER(test)")); + assertEquals(false, results.get("BAD_FUNCTION(test)")); + } + + @Test + public void applyTransformationsShouldProperlyTransformData() { + SensorParserConfig sensorParserConfig = new SensorParserConfig(); + FieldTransformer fieldTransformater = new FieldTransformer(); + fieldTransformater.setOutput("url_host"); + fieldTransformater.setTransformation("STELLAR"); + fieldTransformater.setConfig(new LinkedHashMap() {{ + put("url_host", "TO_LOWER(URL_TO_HOST(url))"); + }}); + sensorParserConfig.setFieldTransformations(ImmutableList.of(fieldTransformater)); + SensorParserContext sensorParserContext = new SensorParserContext(); + sensorParserContext.setSensorParserConfig(sensorParserConfig); + sensorParserContext.setSampleData(new HashMap() {{ + put("url", "https://caseystella.com/blog"); + }}); + Map results = stellarService.applyTransformations(sensorParserContext); + assertEquals(2, results.size()); + assertEquals("https://caseystella.com/blog", results.get("url")); + assertEquals("caseystella.com", results.get("url_host")); + } + + @Test + public void getTransformationsShouldReturnTransformation() { + assertTrue(stellarService.getTransformations().length > 0); + } + + @Test + public void getStellarFunctionsShouldReturnFunctions() { + assertTrue(stellarService.getStellarFunctions().size() > 0); + } + + @Test + public void getSimpleStellarFunctionsShouldReturnFunctions() { + assertEquals(1, stellarService.getSimpleStellarFunctions().stream() + .filter(stellarFunctionDescription -> stellarFunctionDescription.getName().equals("TO_LOWER")).count()); + } + +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormAdminServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormAdminServiceImplTest.java new file mode 100644 index 0000000000..d83a74cac3 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormAdminServiceImplTest.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.rest.model.TopologyResponse; +import org.apache.metron.rest.model.TopologyStatusCode; +import org.apache.metron.rest.service.GlobalConfigService; +import org.apache.metron.rest.service.SensorParserConfigService; +import org.apache.metron.rest.service.StormAdminService; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@SuppressWarnings("ALL") +public class StormAdminServiceImplTest { + @Rule + public final ExpectedException exception = ExpectedException.none(); + + StormCLIWrapper stormCLIClientWrapper; + StormAdminService stormAdminService; + GlobalConfigService globalConfigService; + SensorParserConfigService sensorParserConfigService; + + @Before + public void setUp() throws Exception { + stormCLIClientWrapper = mock(StormCLIWrapper.class); + globalConfigService = mock(GlobalConfigService.class); + sensorParserConfigService = mock(SensorParserConfigService.class); + stormAdminService = new StormAdminServiceImpl(stormCLIClientWrapper, globalConfigService, sensorParserConfigService); + } + + @Test + public void startParserTopologyShouldProperlyReturnSuccessTopologyResponse() throws Exception { + when(stormCLIClientWrapper.startParserTopology("bro")).thenReturn(0); + when(globalConfigService.get()).thenReturn(new HashMap()); + when(sensorParserConfigService.findOne("bro")).thenReturn(new SensorParserConfig()); + + TopologyResponse expected = new TopologyResponse(); + expected.setSuccessMessage(TopologyStatusCode.STARTED.toString()); + TopologyResponse actual = stormAdminService.startParserTopology("bro"); + + assertEquals(expected, actual); + assertEquals(expected.hashCode(), actual.hashCode()); + } + + @Test + public void startParserTopologyShouldReturnGlobalConfigMissingError() throws Exception { + when(globalConfigService.get()).thenReturn(null); + + TopologyResponse expected = new TopologyResponse(); + expected.setErrorMessage(TopologyStatusCode.GLOBAL_CONFIG_MISSING.toString()); + + assertEquals(expected, stormAdminService.startParserTopology("bro")); + } + + @Test + public void startParserTopologyShouldReturnSensorParserConfigMissingError() throws Exception { + when(globalConfigService.get()).thenReturn(new HashMap()); + when(sensorParserConfigService.findOne("bro")).thenReturn(null); + + TopologyResponse expected = new TopologyResponse(); + expected.setErrorMessage(TopologyStatusCode.SENSOR_PARSER_CONFIG_MISSING.toString()); + + assertEquals(expected, stormAdminService.startParserTopology("bro")); + } + + @Test + public void stopParserTopologyShouldProperlyReturnErrorTopologyResponse() throws Exception { + when(stormCLIClientWrapper.stopParserTopology("bro", false)).thenReturn(1); + when(globalConfigService.get()).thenReturn(new HashMap()); + when(sensorParserConfigService.findOne("bro")).thenReturn(new SensorParserConfig()); + + TopologyResponse expected = new TopologyResponse(); + expected.setErrorMessage(TopologyStatusCode.STOP_ERROR.toString()); + + assertEquals(expected, stormAdminService.stopParserTopology("bro", false)); + } + + @Test + public void startEnrichmentTopologyShouldProperlyReturnSuccessTopologyResponse() throws Exception { + when(stormCLIClientWrapper.startEnrichmentTopology()).thenReturn(0); + + TopologyResponse expected = new TopologyResponse(); + expected.setSuccessMessage(TopologyStatusCode.STARTED.toString()); + + assertEquals(expected, stormAdminService.startEnrichmentTopology()); + } + + @Test + public void stopEnrichmentTopologyShouldProperlyReturnSuccessTopologyResponse() throws Exception { + when(stormCLIClientWrapper.stopEnrichmentTopology(false)).thenReturn(0); + + TopologyResponse expected = new TopologyResponse(); + expected.setSuccessMessage(TopologyStatusCode.STOPPED.toString()); + + assertEquals(expected, stormAdminService.stopEnrichmentTopology(false)); + } + + @Test + public void startIndexingTopologyShouldProperlyReturnSuccessTopologyResponse() throws Exception { + when(stormCLIClientWrapper.startIndexingTopology()).thenReturn(0); + + TopologyResponse expected = new TopologyResponse(); + expected.setSuccessMessage(TopologyStatusCode.STARTED.toString()); + + assertEquals(expected, stormAdminService.startIndexingTopology()); + } + + @Test + public void stopIndexingTopologyShouldProperlyReturnSuccessTopologyResponse() throws Exception { + when(stormCLIClientWrapper.stopIndexingTopology(false)).thenReturn(0); + + TopologyResponse expected = new TopologyResponse(); + expected.setSuccessMessage(TopologyStatusCode.STOPPED.toString()); + + assertEquals(expected, stormAdminService.stopIndexingTopology(false)); + } + + @Test + public void getStormClientStatusShouldProperlyReturnStatus() throws Exception { + final Map status = new HashMap() {{ + put("status", "statusValue"); + }}; + when(stormCLIClientWrapper.getStormClientStatus()).thenReturn(status); + + assertEquals(new HashMap() {{ + put("status", "statusValue"); + }}, stormAdminService.getStormClientStatus()); + } + + +} \ No newline at end of file diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormCLIWrapperTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormCLIWrapperTest.java new file mode 100644 index 0000000000..cb26783f16 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormCLIWrapperTest.java @@ -0,0 +1,216 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.metron.rest.MetronRestConstants; +import org.apache.metron.rest.RestException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.springframework.core.env.Environment; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.anyVararg; +import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.mock; +import static org.powermock.api.mockito.PowerMockito.verifyNew; +import static org.powermock.api.mockito.PowerMockito.when; +import static org.powermock.api.mockito.PowerMockito.whenNew; + +@SuppressWarnings("unchecked") +@RunWith(PowerMockRunner.class) +@PrepareForTest({DockerStormCLIWrapper.class, ProcessBuilder.class}) +public class StormCLIWrapperTest { + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + private ProcessBuilder processBuilder; + private Environment environment; + private Process process; + private StormCLIWrapper stormCLIWrapper; + + @Before + public void setUp() throws Exception { + processBuilder = mock(ProcessBuilder.class); + environment = mock(Environment.class); + process = mock(Process.class); + stormCLIWrapper = new StormCLIWrapper(); + stormCLIWrapper.setEnvironment(environment); + } + + @Test + public void startParserTopologyShouldRunCommandProperly() throws Exception { + whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder); + + when(processBuilder.start()).thenReturn(process); + when(environment.getProperty(MetronRestConstants.PARSER_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_parser"); + when(environment.getProperty(MetronRestConstants.KAFKA_BROKER_URL_SPRING_PROPERTY)).thenReturn("kafka_broker_url"); + when(environment.getProperty(MetronRestConstants.ZK_URL_SPRING_PROPERTY)).thenReturn("zookeeper_url"); + when(process.exitValue()).thenReturn(0); + + assertEquals(0, stormCLIWrapper.startParserTopology("bro")); + verify(process).waitFor(); + verifyNew(ProcessBuilder.class).withArguments("/start_parser", "-k", "kafka_broker_url", "-z", "zookeeper_url", "-s", "bro"); + } + + @Test + public void stopParserTopologyShouldRunCommandProperly() throws Exception { + whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder); + + when(processBuilder.start()).thenReturn(process); + when(process.exitValue()).thenReturn(0); + + assertEquals(0, stormCLIWrapper.stopParserTopology("bro", false)); + verify(process).waitFor(); + verifyNew(ProcessBuilder.class).withArguments("storm", "kill", "bro"); + } + + @Test + public void stopParserTopologyNowShouldRunCommandProperly() throws Exception { + whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder); + + when(processBuilder.start()).thenReturn(process); + when(process.exitValue()).thenReturn(0); + + assertEquals(0, stormCLIWrapper.stopParserTopology("bro", true)); + verify(process).waitFor(); + verifyNew(ProcessBuilder.class).withArguments("storm", "kill", "bro", "-w", "0"); + } + + @Test + public void startEnrichmentTopologyShouldRunCommandProperly() throws Exception { + whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder); + + when(processBuilder.start()).thenReturn(process); + when(environment.getProperty(MetronRestConstants.ENRICHMENT_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_enrichment"); + when(process.exitValue()).thenReturn(0); + + assertEquals(0, stormCLIWrapper.startEnrichmentTopology()); + verify(process).waitFor(); + verifyNew(ProcessBuilder.class).withArguments("/start_enrichment"); + + } + + @Test + public void stopEnrichmentTopologyShouldRunCommandProperly() throws Exception { + whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder); + + when(processBuilder.start()).thenReturn(process); + when(process.exitValue()).thenReturn(0); + + assertEquals(0, stormCLIWrapper.stopEnrichmentTopology(false)); + verify(process).waitFor(); + verifyNew(ProcessBuilder.class).withArguments("storm", "kill", MetronRestConstants.ENRICHMENT_TOPOLOGY_NAME); + } + + @Test + public void startIndexingTopologyShouldRunCommandProperly() throws Exception { + whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder); + + when(processBuilder.start()).thenReturn(process); + when(environment.getProperty(MetronRestConstants.INDEXING_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_indexing"); + when(process.exitValue()).thenReturn(0); + + assertEquals(0, stormCLIWrapper.startIndexingTopology()); + verify(process).waitFor(); + verifyNew(ProcessBuilder.class).withArguments("/start_indexing"); + + } + + @Test + public void stopIndexingTopologyShouldRunCommandProperly() throws Exception { + whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder); + + when(processBuilder.start()).thenReturn(process); + when(process.exitValue()).thenReturn(0); + + assertEquals(0, stormCLIWrapper.stopIndexingTopology(false)); + verify(process).waitFor(); + verifyNew(ProcessBuilder.class).withArguments("storm", "kill", MetronRestConstants.INDEXING_TOPOLOGY_NAME); + } + + @Test + public void getStormClientStatusShouldReturnCorrectStatus() throws Exception { + whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder); + + Process process = mock(Process.class); + InputStream inputStream = new ByteArrayInputStream("\nStorm 1.1".getBytes(UTF_8)); + + when(processBuilder.start()).thenReturn(process); + + when(process.getInputStream()).thenReturn(inputStream); + when(environment.getProperty(MetronRestConstants.PARSER_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_parser"); + when(environment.getProperty(MetronRestConstants.ENRICHMENT_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_enrichment"); + when(environment.getProperty(MetronRestConstants.INDEXING_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_indexing"); + + + Map actual = stormCLIWrapper.getStormClientStatus(); + assertEquals(new HashMap() {{ + put("parserScriptPath", "/start_parser"); + put("enrichmentScriptPath", "/start_enrichment"); + put("indexingScriptPath", "/start_indexing"); + put("stormClientVersionInstalled", "1.1"); + + }}, actual); + verifyNew(ProcessBuilder.class).withArguments("storm", "version"); + } + + @Test + public void stormClientVersionInstalledShouldReturnDefault() throws Exception { + whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder); + + Process process = mock(Process.class); + InputStream inputStream = new ByteArrayInputStream("".getBytes(UTF_8)); + + when(processBuilder.start()).thenReturn(process); + when(process.getInputStream()).thenReturn(inputStream); + assertEquals("Storm client is not installed", stormCLIWrapper.stormClientVersionInstalled()); + } + + @Test + public void runCommandShouldReturnRestExceptionOnError() throws Exception { + exception.expect(RestException.class); + + whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder); + when(processBuilder.start()).thenThrow(new IOException()); + + stormCLIWrapper.runCommand(new String[]{"storm", "kill"}); + } + + @Test + public void stormClientVersionInstalledShouldReturnRestExceptionOnError() throws Exception { + exception.expect(RestException.class); + + whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder); + when(processBuilder.start()).thenThrow(new IOException()); + + stormCLIWrapper.stormClientVersionInstalled(); + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormStatusServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormStatusServiceImplTest.java new file mode 100644 index 0000000000..db2cb7f911 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormStatusServiceImplTest.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.service.impl; + +import org.apache.metron.rest.model.TopologyResponse; +import org.apache.metron.rest.model.TopologyStatus; +import org.apache.metron.rest.model.TopologyStatusCode; +import org.apache.metron.rest.model.TopologySummary; +import org.apache.metron.rest.service.StormStatusService; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.springframework.core.env.Environment; +import org.springframework.web.client.RestTemplate; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.metron.rest.MetronRestConstants.STORM_UI_SPRING_PROPERTY; +import static org.apache.metron.rest.MetronRestConstants.TOPOLOGY_SUMMARY_URL; +import static org.apache.metron.rest.MetronRestConstants.TOPOLOGY_URL; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@SuppressWarnings("ALL") +public class StormStatusServiceImplTest { + @Rule + public final ExpectedException exception = ExpectedException.none(); + + Environment environment; + RestTemplate restTemplate; + StormStatusService stormStatusService; + + @Before + public void setUp() throws Exception { + environment = mock(Environment.class); + restTemplate = mock(RestTemplate.class); + stormStatusService = new StormStatusServiceImpl(environment, restTemplate); + } + + @Test + public void getTopologySummaryShouldReturnTopologySummary() throws Exception { + final TopologyStatus topologyStatus = new TopologyStatus(); + topologyStatus.setStatus(TopologyStatusCode.STARTED); + topologyStatus.setName("bro"); + topologyStatus.setId("bro_id"); + final TopologySummary topologySummary = new TopologySummary(); + topologySummary.setTopologies(new TopologyStatus[]{topologyStatus}); + + when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui"); + when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(topologySummary); + + TopologyStatus expectedStatus = new TopologyStatus(); + expectedStatus.setStatus(TopologyStatusCode.STARTED); + expectedStatus.setName("bro"); + expectedStatus.setId("bro_id"); + TopologySummary expected = new TopologySummary(); + expected.setTopologies(new TopologyStatus[]{expectedStatus}); + + TopologySummary actual = stormStatusService.getTopologySummary(); + assertEquals(expected, actual); + assertEquals(expected.hashCode(), actual.hashCode()); + } + + @Test + public void getTopologyStatusShouldReturnTopologyStatus() throws Exception { + final TopologyStatus topologyStatus = new TopologyStatus(); + topologyStatus.setStatus(TopologyStatusCode.STARTED); + topologyStatus.setName("bro"); + topologyStatus.setId("bro_id"); + final TopologySummary topologySummary = new TopologySummary(); + topologySummary.setTopologies(new TopologyStatus[]{topologyStatus}); + + when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui"); + when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(topologySummary); + when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_URL + "/bro_id", TopologyStatus.class)).thenReturn(topologyStatus); + + TopologyStatus expected = new TopologyStatus(); + expected.setStatus(TopologyStatusCode.STARTED); + expected.setName("bro"); + expected.setId("bro_id"); + + TopologyStatus actual = stormStatusService.getTopologyStatus("bro"); + assertEquals(expected, actual); + assertEquals(expected.hashCode(), actual.hashCode()); + } + + @Test + public void getAllTopologyStatusShouldReturnAllTopologyStatus() { + final TopologyStatus topologyStatus = new TopologyStatus(); + topologyStatus.setStatus(TopologyStatusCode.STARTED); + topologyStatus.setName("bro"); + topologyStatus.setId("bro_id"); + final TopologySummary topologySummary = new TopologySummary(); + topologySummary.setTopologies(new TopologyStatus[]{topologyStatus}); + + when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui"); + when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(topologySummary); + when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_URL + "/bro_id", TopologyStatus.class)).thenReturn(topologyStatus); + + TopologyStatus expected = new TopologyStatus(); + expected.setStatus(TopologyStatusCode.STARTED); + expected.setName("bro"); + expected.setId("bro_id"); + + assertEquals(new ArrayList() {{ add(expected); }}, stormStatusService.getAllTopologyStatus()); + } + + + @Test + public void activateTopologyShouldReturnActiveTopologyResponse() { + final TopologyStatus topologyStatus = new TopologyStatus(); + topologyStatus.setName("bro"); + topologyStatus.setId("bro_id"); + final TopologySummary topologySummary = new TopologySummary(); + topologySummary.setTopologies(new TopologyStatus[]{topologyStatus}); + + when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui"); + when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(topologySummary); + when(restTemplate.postForObject("http://storm_ui" + TOPOLOGY_URL + "/bro_id/activate", null, Map.class)) + .thenReturn(new HashMap() {{ put("status", "success"); }}); + + TopologyResponse expected = new TopologyResponse(); + expected.setSuccessMessage(TopologyStatusCode.ACTIVE.toString()); + assertEquals(expected, stormStatusService.activateTopology("bro")); + } + + @Test + public void activateTopologyShouldReturnErrorTopologyResponse() { + final TopologyStatus topologyStatus = new TopologyStatus(); + topologyStatus.setName("bro"); + topologyStatus.setId("bro_id"); + final TopologySummary topologySummary = new TopologySummary(); + topologySummary.setTopologies(new TopologyStatus[]{topologyStatus}); + + when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui"); + when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(topologySummary); + when(restTemplate.postForObject("http://storm_ui" + TOPOLOGY_URL + "/bro_id/activate", null, Map.class)) + .thenReturn(new HashMap() {{ put("status", "error message"); }}); + + TopologyResponse expected = new TopologyResponse(); + expected.setErrorMessage("error message"); + assertEquals(expected, stormStatusService.activateTopology("bro")); + } + + @Test + public void activateTopologyShouldReturnTopologyNotFoundTopologyResponse() { + when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui"); + when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(new TopologySummary()); + + TopologyResponse expected = new TopologyResponse(); + expected.setErrorMessage(TopologyStatusCode.TOPOLOGY_NOT_FOUND.toString()); + assertEquals(expected, stormStatusService.activateTopology("bro")); + } + + @Test + public void deactivateTopologyShouldReturnActiveTopologyResponse() { + final TopologyStatus topologyStatus = new TopologyStatus(); + topologyStatus.setName("bro"); + topologyStatus.setId("bro_id"); + final TopologySummary topologySummary = new TopologySummary(); + topologySummary.setTopologies(new TopologyStatus[]{topologyStatus}); + + when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui"); + when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(topologySummary); + when(restTemplate.postForObject("http://storm_ui" + TOPOLOGY_URL + "/bro_id/deactivate", null, Map.class)) + .thenReturn(new HashMap() {{ put("status", "success"); }}); + + TopologyResponse expected = new TopologyResponse(); + expected.setSuccessMessage(TopologyStatusCode.INACTIVE.toString()); + assertEquals(expected, stormStatusService.deactivateTopology("bro")); + } + + @Test + public void deactivateTopologyShouldReturnErrorTopologyResponse() { + final TopologyStatus topologyStatus = new TopologyStatus(); + topologyStatus.setName("bro"); + topologyStatus.setId("bro_id"); + final TopologySummary topologySummary = new TopologySummary(); + topologySummary.setTopologies(new TopologyStatus[]{topologyStatus}); + + when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui"); + when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(topologySummary); + when(restTemplate.postForObject("http://storm_ui" + TOPOLOGY_URL + "/bro_id/deactivate", null, Map.class)) + .thenReturn(new HashMap() {{ put("status", "error message"); }}); + + TopologyResponse expected = new TopologyResponse(); + expected.setErrorMessage("error message"); + assertEquals(expected, stormStatusService.deactivateTopology("bro")); + } + + @Test + public void deactivateTopologyShouldReturnTopologyNotFoundTopologyResponse() { + when(environment.getProperty(STORM_UI_SPRING_PROPERTY)).thenReturn("storm_ui"); + when(restTemplate.getForObject("http://storm_ui" + TOPOLOGY_SUMMARY_URL, TopologySummary.class)).thenReturn(new TopologySummary()); + + TopologyResponse expected = new TopologyResponse(); + expected.setErrorMessage(TopologyStatusCode.TOPOLOGY_NOT_FOUND.toString()); + assertEquals(expected, stormStatusService.deactivateTopology("bro")); + } + + +} \ No newline at end of file diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/ReadMeUtils.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/ReadMeUtils.java new file mode 100644 index 0000000000..d9522f2fa9 --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/ReadMeUtils.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.utils; + +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; +import org.apache.velocity.Template; +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.Velocity; +import org.apache.velocity.runtime.RuntimeConstants; +import org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader; +import org.reflections.Reflections; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import java.io.FileWriter; +import java.lang.reflect.Method; +import java.lang.reflect.Parameter; +import java.security.Principal; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +public class ReadMeUtils { + + public static void main(String[] args) throws Exception { + String path; + if (args.length == 0) { + System.out.println("README output path was not set. Defaulting to 'metron-interface/metron-rest/README.md'"); + path = "metron-interface/metron-rest/README.md"; + } else { + path = args[0]; + } + + Reflections reflections = new Reflections("org.apache.metron.rest.controller"); + List endpoints = new ArrayList<>(); + Set> controllers = reflections.getTypesAnnotatedWith(RestController.class); + for(Class controller: controllers) { + RequestMapping controllerRequestMapping = controller.getAnnotation(RequestMapping.class); + String pathPrefix; + if (controllerRequestMapping == null) { + pathPrefix = ""; + } else { + pathPrefix = controllerRequestMapping.value()[0]; + } + for(Method method: controller.getDeclaredMethods()) { + RestControllerInfo restControllerInfo = new RestControllerInfo(); + RequestMapping requestMapping = method.getAnnotation(RequestMapping.class); + if (requestMapping == null) { + throw new Exception(String.format("@RequestMapping annotation missing from method %s.%s. Every controller method must have a @RequestMapping annotation.", controller.getSimpleName(), method.getName())); + } + ApiOperation apiOperation = method.getAnnotation(ApiOperation.class); + if (apiOperation == null) { + throw new Exception(String.format("@ApiOperation annotation missing from method %s.%s. Every controller method must have an @ApiOperation annotation.", controller.getSimpleName(), method.getName())); + } + ApiResponse[] apiResponseArray; + ApiResponses apiResponses = method.getAnnotation(ApiResponses.class); + if (apiResponses == null) { + ApiResponse apiResponse = method.getAnnotation(ApiResponse.class); + if (apiResponse == null) { + throw new Exception(String.format("@ApiResponses or @ApiResponse annotation missing from method %s.%s. Every controller method must have an @ApiResponses or @ApiResponse annotation.", controller.getSimpleName(), method.getName())); + } else { + apiResponseArray = new ApiResponse[]{ apiResponse }; + } + } else { + apiResponseArray = apiResponses.value(); + } + String[] requestMappingValue = requestMapping.value(); + if (requestMappingValue.length == 0) { + restControllerInfo.setPath(pathPrefix); + } else { + restControllerInfo.setPath(pathPrefix + requestMappingValue[0]); + } + restControllerInfo.setDescription(apiOperation.value()); + RequestMethod requestMethod; + if (requestMapping.method().length == 0) { + requestMethod = RequestMethod.GET; + } else { + requestMethod = requestMapping.method()[0]; + } + restControllerInfo.setMethod(requestMethod); + for (ApiResponse apiResponse: apiResponseArray) { + restControllerInfo.addResponse(apiResponse.message(), apiResponse.code()); + } + for(Parameter parameter: method.getParameters()) { + if (!parameter.getType().equals(Principal.class)) { + ApiParam apiParam = parameter.getAnnotation(ApiParam.class); + if (apiParam == null) { + throw new Exception(String.format("@ApiParam annotation missing from parameter %s.%s.%s. Every controller method parameter must have an @ApiParam annotation.", controller.getSimpleName(), method.getName(), parameter.getName())); + } + restControllerInfo.addParameterDescription(apiParam.name(), apiParam.value()); + } + } + endpoints.add(restControllerInfo); + } + } + Collections.sort(endpoints, (o1, o2) -> o1.getPath().compareTo(o2.getPath())); + + Velocity.setProperty(RuntimeConstants.RESOURCE_LOADER, "classpath"); + Velocity.setProperty("classpath.resource.loader.class", ClasspathResourceLoader.class.getName()); + Velocity.init(); + + VelocityContext context = new VelocityContext(); + context.put( "endpoints", endpoints ); + + Template template = Velocity.getTemplate("README.vm"); + FileWriter fileWriter = new FileWriter(path); + template.merge( context, fileWriter ); + fileWriter.close(); + } +} diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/RestControllerInfo.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/RestControllerInfo.java new file mode 100644 index 0000000000..5b7ac0e3ba --- /dev/null +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/utils/RestControllerInfo.java @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.rest.utils; + +import org.springframework.web.bind.annotation.RequestMethod; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class RestControllerInfo { + + public class Response { + + private String message; + private int code; + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + } + + private String path; + private String description; + private RequestMethod method; + private List responses = new ArrayList<>(); + private Map parameterDescriptions = new HashMap<>(); + + public RestControllerInfo() {} + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public RequestMethod getMethod() { + return method; + } + + public void setMethod(RequestMethod method) { + this.method = method; + } + + public List getResponses() { + return responses; + } + + public void addResponse(String message, int code) { + Response response = new Response(); + response.setMessage(message); + response.setCode(code); + this.responses.add(response); + } + + public Map getParameterDescriptions() { + return parameterDescriptions; + } + + public void addParameterDescription(String name, String description) { + parameterDescriptions.put(name, description); + } +} + + diff --git a/metron-interface/metron-rest/src/test/resources/README.vm b/metron-interface/metron-rest/src/test/resources/README.vm new file mode 100644 index 0000000000..99f4286ff0 --- /dev/null +++ b/metron-interface/metron-rest/src/test/resources/README.vm @@ -0,0 +1,141 @@ +#[[#]]# Metron REST + +This module provides a RESTful API for interacting with Metron. + +#[[##]]# Prerequisites + +* A running Metron cluster +* Java 8 installed +* Storm CLI and Metron topology scripts (start_parser_topology.sh, start_enrichment_topology.sh, start_elasticsearch_topology.sh) installed + +#[[##]]# Installation +1. Package the application with Maven: +``` +mvn clean package +``` + +1. Untar the archive in the target directory. The directory structure will look like: +``` +bin + start_metron_rest.sh +lib + metron-rest-$METRON_VERSION.jar +``` + +1. Create an `application.yml` file with the contents of [application-docker.yml](src/main/resources/application-docker.yml). Substitute the appropriate Metron service urls (Kafka, Zookeeper, Storm, etc) in properties containing `${docker.host.address}` and update the `spring.datasource.*` properties as needed (see the [Security](#security) section for more details). + +1. Start the application with this command: +``` +./bin/start_metron_rest.sh /path/to/application.yml +``` + +#[[##]]# Usage + +The exposed REST endpoints can be accessed with the Swagger UI at http://host:port/swagger-ui.html#/. The default port is 8080 but can be changed in application.yml by setting "server.port" to the desired port. + +#[[##]]# Security + +The metron-rest module uses [Spring Security](http://projects.spring.io/spring-security/) for authentication and stores user credentials in a relational database. The H2 database is configured by default and is intended only for development purposes. The "dev" profile can be used to automatically load test users: +``` +./bin/start_metron_rest.sh /path/to/application.yml --spring.profiles.active=dev +``` + +For [production use](http://docs.spring.io/spring-boot/docs/1.4.1.RELEASE/reference/htmlsingle/#boot-features-connect-to-production-database), a relational database should be configured. For example, configuring MySQL would be done as follows: + +1. Create a MySQL user for the Metron REST application (http://dev.mysql.com/doc/refman/5.7/en/adding-users.html). + +1. Connect to MySQL and create a Metron REST database: +``` +CREATE DATABASE IF NOT EXISTS metronrest +``` + +1. Add users: +``` +use metronrest; +insert into users (username, password, enabled) values ('your_username','your_password',1); +insert into authorities (username, authority) values ('your_username', 'ROLE_USER'); +``` + +1. Replace the H2 connection information in the application.yml file with MySQL connection information: +``` +spring: + datasource: + driverClassName: com.mysql.jdbc.Driver + url: jdbc:mysql://mysql_host:3306/metronrest + username: metron_rest_user + password: metron_rest_password + platform: mysql +``` + +1. Add a dependency for the MySQL JDBC connector in the metron-rest pom.xml: +``` + + mysql + mysql-connector-java + ${mysql.client.version} + +``` + +1. Follow the steps in the [Installation](#installation) section + +#[[##]]# API + +Request and Response objects are JSON formatted. The JSON schemas are available in the Swagger UI. + +| | +| ---------- | +#foreach( $restControllerInfo in $endpoints ) +| [ `$restControllerInfo.getMethod().toString() $restControllerInfo.getPath()`](#$restControllerInfo.getMethod().toString().toLowerCase()-$restControllerInfo.getPath().toLowerCase().replaceAll("/", ""))| +#end + +#foreach( $restControllerInfo in $endpoints ) +#[[###]]# `$restControllerInfo.getMethod().toString() $restControllerInfo.getPath()` + * Description: $restControllerInfo.getDescription() +#if($restControllerInfo.getParameterDescriptions().size() > 0) + * Input: +#end +#foreach( $parameterDescription in $restControllerInfo.getParameterDescriptions().entrySet()) + * $parameterDescription.getKey() - $parameterDescription.getValue() +#end + * Returns: +#foreach( $response in $restControllerInfo.getResponses()) + * $response.getCode() - $response.getMessage() +#end + +#end +#[[##]]# Testing + +Profiles are includes for both the metron-docker and Quick Dev environments. + +#[[###]]# metron-docker + +Start the [metron-docker](../../metron-docker) environment. Build the metron-rest module and start it with the Spring Boot Maven plugin: +``` +mvn clean package +mvn spring-boot:run -Drun.profiles=docker,dev +``` +The metron-rest application will be available at http://localhost:8080/swagger-ui.html#/. + +#[[###]]# Quick Dev + +Start the [Quick Dev](../../metron-deployment/vagrant/quick-dev-platform) environment. Build the metron-rest module and start it with the Spring Boot Maven plugin: +``` +mvn clean package +mvn spring-boot:run -Drun.profiles=vagrant,dev +``` +The metron-rest application will be available at http://localhost:8080/swagger-ui.html#/. + +To run the application locally on the Quick Dev host, package the application and scp the archive to node1: +``` +mvn clean package +scp ./target/metron-rest-$METRON_VERSION-archive.tar.gz root@node1:~/ +``` +Login to node1 and unarchive the metron-rest application. Start the application on a different port to avoid conflicting with Ambari: +``` +java -jar ./lib/metron-rest-$METRON_VERSION.jar --spring.profiles.active=vagrant,dev --server.port=8082 +``` +The metron-rest application will be available at http://node1:8082/swagger-ui.html#/. + +#[[##]]# License + +This project depends on the Java Transaction API. See https://java.net/projects/jta-spec/ for more details. diff --git a/metron-interface/metron-rest/src/test/resources/log4j.properties b/metron-interface/metron-rest/src/test/resources/log4j.properties new file mode 100644 index 0000000000..492cecf9d7 --- /dev/null +++ b/metron-interface/metron-rest/src/test/resources/log4j.properties @@ -0,0 +1,16 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=ERROR, stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd'T'HH:mm:ss.SSS} %-5p [%c] - %m%n \ No newline at end of file diff --git a/metron-interface/pom.xml b/metron-interface/pom.xml new file mode 100644 index 0000000000..805c024649 --- /dev/null +++ b/metron-interface/pom.xml @@ -0,0 +1,90 @@ + + + + + 4.0.0 + metron-interface + pom + metron-interface + + org.apache.metron + Metron + 0.3.1 + + Interfaces for Metron + https://metron.incubator.apache.org/ + + scm:git:https://git-wip-us.apache.org/repos/asf/incubator-metron.git + scm:git:https://git-wip-us.apache.org/repos/asf/incubator-metron.git + HEAD + https://git-wip-us.apache.org/repos/asf/incubator-metron + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + metron-rest + metron-rest-client + + + + junit + junit + 4.12 + test + + + org.adrianwalker + multiline-string + 0.1.2 + test + + + + + + + + + + org.apache.maven.plugins + maven-pmd-plugin + 3.3 + + ${global_java_version} + + + + org.codehaus.mojo + emma-maven-plugin + 1.0-alpha-3 + true + + + + + + multiline-release-repo + https://raw.github.com/benelog/multiline/master/maven-repository + + false + + + + diff --git a/pom.xml b/pom.xml index db28adb21a..3a8eb9b22d 100644 --- a/pom.xml +++ b/pom.xml @@ -29,6 +29,7 @@ metron-platform metron-deployment metron-docker + metron-interface