diff --git a/build.gradle b/build.gradle index 0586dbe2df8..0d8e3f067a7 100644 --- a/build.gradle +++ b/build.gradle @@ -84,7 +84,9 @@ repositories { spotless { java { target fileTree('.') { - include 'core/**/*.java' + include 'common/**/*.java', + 'datasources/**/*.java', + 'core/**/*.java' exclude '**/build/**', '**/build-*/**' } importOrder() diff --git a/common/build.gradle b/common/build.gradle index 0367035ce2f..25cdcd6566e 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -31,6 +31,11 @@ repositories { mavenCentral() } +// Being ignored as a temporary measure before being removed in favour of +// spotless https://github.com/opensearch-project/sql/issues/1101 +checkstyleTest.ignoreFailures = true +checkstyleMain.ignoreFailures = true + dependencies { api "org.antlr:antlr4-runtime:4.7.1" api group: 'com.google.guava', name: 'guava', version: '32.0.1-jre' @@ -62,4 +67,4 @@ configurations.all { resolutionStrategy.force "org.apache.httpcomponents:httpcore:4.4.13" resolutionStrategy.force "joda-time:joda-time:2.10.12" resolutionStrategy.force "org.slf4j:slf4j-api:1.7.36" -} \ No newline at end of file +} diff --git a/common/src/main/java/org/opensearch/sql/common/antlr/CaseInsensitiveCharStream.java b/common/src/main/java/org/opensearch/sql/common/antlr/CaseInsensitiveCharStream.java index 0036da32a1a..89381872ce8 100644 --- a/common/src/main/java/org/opensearch/sql/common/antlr/CaseInsensitiveCharStream.java +++ b/common/src/main/java/org/opensearch/sql/common/antlr/CaseInsensitiveCharStream.java @@ -3,7 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.sql.common.antlr; import org.antlr.v4.runtime.CharStream; diff --git a/common/src/main/java/org/opensearch/sql/common/antlr/SyntaxAnalysisErrorListener.java b/common/src/main/java/org/opensearch/sql/common/antlr/SyntaxAnalysisErrorListener.java index b499a529674..76cbf52d583 100644 --- a/common/src/main/java/org/opensearch/sql/common/antlr/SyntaxAnalysisErrorListener.java +++ b/common/src/main/java/org/opensearch/sql/common/antlr/SyntaxAnalysisErrorListener.java @@ -3,7 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.sql.common.antlr; import java.util.Locale; diff --git a/common/src/main/java/org/opensearch/sql/common/antlr/SyntaxCheckException.java b/common/src/main/java/org/opensearch/sql/common/antlr/SyntaxCheckException.java index 806cb7208bb..d3c9c111efa 100644 --- a/common/src/main/java/org/opensearch/sql/common/antlr/SyntaxCheckException.java +++ b/common/src/main/java/org/opensearch/sql/common/antlr/SyntaxCheckException.java @@ -3,7 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.sql.common.antlr; public class SyntaxCheckException extends RuntimeException { diff --git a/common/src/main/java/org/opensearch/sql/common/authinterceptors/AwsSigningInterceptor.java b/common/src/main/java/org/opensearch/sql/common/authinterceptors/AwsSigningInterceptor.java index 6c65c69c317..e2d33dca8b1 100644 --- a/common/src/main/java/org/opensearch/sql/common/authinterceptors/AwsSigningInterceptor.java +++ b/common/src/main/java/org/opensearch/sql/common/authinterceptors/AwsSigningInterceptor.java @@ -31,15 +31,17 @@ public class AwsSigningInterceptor implements Interceptor { private static final Logger LOG = LogManager.getLogger(); /** - * AwsSigningInterceptor which intercepts http requests - * and adds required headers for sigv4 authentication. + * AwsSigningInterceptor which intercepts http requests and adds required headers for sigv4 + * authentication. * * @param awsCredentialsProvider awsCredentialsProvider. * @param region region. * @param serviceName serviceName. */ - public AwsSigningInterceptor(@NonNull AWSCredentialsProvider awsCredentialsProvider, - @NonNull String region, @NonNull String serviceName) { + public AwsSigningInterceptor( + @NonNull AWSCredentialsProvider awsCredentialsProvider, + @NonNull String region, + @NonNull String serviceName) { this.okHttpAwsV4Signer = new OkHttpAwsV4Signer(region, serviceName); this.awsCredentialsProvider = awsCredentialsProvider; } @@ -48,25 +50,27 @@ public AwsSigningInterceptor(@NonNull AWSCredentialsProvider awsCredentialsProvi public Response intercept(Interceptor.Chain chain) throws IOException { Request request = chain.request(); - DateTimeFormatter timestampFormat = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'") - .withZone(ZoneId.of("GMT")); + DateTimeFormatter timestampFormat = + DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'").withZone(ZoneId.of("GMT")); - - Request.Builder newRequestBuilder = request.newBuilder() - .addHeader("x-amz-date", timestampFormat.format(ZonedDateTime.now())) - .addHeader("host", request.url().host()); + Request.Builder newRequestBuilder = + request + .newBuilder() + .addHeader("x-amz-date", timestampFormat.format(ZonedDateTime.now())) + .addHeader("host", request.url().host()); AWSCredentials awsCredentials = awsCredentialsProvider.getCredentials(); if (awsCredentialsProvider instanceof STSAssumeRoleSessionCredentialsProvider) { - newRequestBuilder.addHeader("x-amz-security-token", + newRequestBuilder.addHeader( + "x-amz-security-token", ((STSAssumeRoleSessionCredentialsProvider) awsCredentialsProvider) .getCredentials() .getSessionToken()); } Request newRequest = newRequestBuilder.build(); - Request signed = okHttpAwsV4Signer.sign(newRequest, - awsCredentials.getAWSAccessKeyId(), awsCredentials.getAWSSecretKey()); + Request signed = + okHttpAwsV4Signer.sign( + newRequest, awsCredentials.getAWSAccessKeyId(), awsCredentials.getAWSSecretKey()); return chain.proceed(signed); } - } diff --git a/common/src/main/java/org/opensearch/sql/common/authinterceptors/BasicAuthenticationInterceptor.java b/common/src/main/java/org/opensearch/sql/common/authinterceptors/BasicAuthenticationInterceptor.java index 34634d15801..2275482e306 100644 --- a/common/src/main/java/org/opensearch/sql/common/authinterceptors/BasicAuthenticationInterceptor.java +++ b/common/src/main/java/org/opensearch/sql/common/authinterceptors/BasicAuthenticationInterceptor.java @@ -22,13 +22,11 @@ public BasicAuthenticationInterceptor(@NonNull String username, @NonNull String this.credentials = Credentials.basic(username, password); } - @Override public Response intercept(Interceptor.Chain chain) throws IOException { Request request = chain.request(); - Request authenticatedRequest = request.newBuilder() - .header("Authorization", credentials).build(); + Request authenticatedRequest = + request.newBuilder().header("Authorization", credentials).build(); return chain.proceed(authenticatedRequest); } - } diff --git a/common/src/main/java/org/opensearch/sql/common/grok/Converter.java b/common/src/main/java/org/opensearch/sql/common/grok/Converter.java index ebbe13f7611..ddd3a2bbb45 100644 --- a/common/src/main/java/org/opensearch/sql/common/grok/Converter.java +++ b/common/src/main/java/org/opensearch/sql/common/grok/Converter.java @@ -23,9 +23,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -/** - * Convert String argument to the right type. - */ +/** Convert String argument to the right type. */ public class Converter { public enum Type { @@ -51,13 +49,13 @@ public enum Type { private static final Pattern SPLITTER = Pattern.compile("[:;]"); private static final Map TYPES = - Arrays.stream(Type.values()) - .collect(Collectors.toMap(t -> t.name().toLowerCase(), t -> t)); + Arrays.stream(Type.values()).collect(Collectors.toMap(t -> t.name().toLowerCase(), t -> t)); private static final Map TYPE_ALIASES = Arrays.stream(Type.values()) - .flatMap(type -> type.aliases.stream() - .map(alias -> new AbstractMap.SimpleEntry<>(alias, type))) + .flatMap( + type -> + type.aliases.stream().map(alias -> new AbstractMap.SimpleEntry<>(alias, type))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); private static Type getType(String key) { @@ -69,34 +67,30 @@ private static Type getType(String key) { return type; } - /** - * getConverters. - */ - public static Map> - getConverters(Collection groupNames, Object... params) { + /** getConverters. */ + public static Map> getConverters( + Collection groupNames, Object... params) { return groupNames.stream() .filter(Converter::containsDelimiter) - .collect(Collectors.toMap(Function.identity(), key -> { - String[] list = splitGrokPattern(key); - IConverter converter = getType(list[1]).converter; - if (list.length == 3) { - converter = converter.newConverter(list[2], params); - } - return converter; - })); + .collect( + Collectors.toMap( + Function.identity(), + key -> { + String[] list = splitGrokPattern(key); + IConverter converter = getType(list[1]).converter; + if (list.length == 3) { + converter = converter.newConverter(list[2], params); + } + return converter; + })); } - /** - * getGroupTypes. - */ + /** getGroupTypes. */ public static Map getGroupTypes(Collection groupNames) { return groupNames.stream() .filter(Converter::containsDelimiter) .map(Converter::splitGrokPattern) - .collect(Collectors.toMap( - l -> l[0], - l -> getType(l[1]) - )); + .collect(Collectors.toMap(l -> l[0], l -> getType(l[1]))); } public static String extractKey(String key) { @@ -120,7 +114,6 @@ default IConverter newConverter(String param, Object... params) { } } - static class DateConverter implements IConverter { private final DateTimeFormatter formatter; @@ -138,8 +131,12 @@ private DateConverter(DateTimeFormatter formatter, ZoneId timeZone) { @Override public Instant convert(String value) { - TemporalAccessor dt = formatter - .parseBest(value.trim(), ZonedDateTime::from, LocalDateTime::from, OffsetDateTime::from, + TemporalAccessor dt = + formatter.parseBest( + value.trim(), + ZonedDateTime::from, + LocalDateTime::from, + OffsetDateTime::from, Instant::from, LocalDate::from); if (dt instanceof ZonedDateTime) { diff --git a/common/src/main/java/org/opensearch/sql/common/grok/Grok.java b/common/src/main/java/org/opensearch/sql/common/grok/Grok.java index f20f99cbc3d..6dfab3f7910 100644 --- a/common/src/main/java/org/opensearch/sql/common/grok/Grok.java +++ b/common/src/main/java/org/opensearch/sql/common/grok/Grok.java @@ -16,36 +16,29 @@ import org.opensearch.sql.common.grok.Converter.IConverter; /** - * {@code Grok} parse arbitrary text and structure it. - *
- * {@code Grok} is simple API that allows you to easily parse logs - * and other files (single line). With {@code Grok}, - * you can turn unstructured log and event data into structured data. + * {@code Grok} parse arbitrary text and structure it.
+ * {@code Grok} is simple API that allows you to easily parse logs and other files (single line). + * With {@code Grok}, you can turn unstructured log and event data into structured data. * * @since 0.0.1 */ public class Grok implements Serializable { - /** - * Named regex of the originalGrokPattern. - */ + /** Named regex of the originalGrokPattern. */ private final String namedRegex; + /** - * Map of the named regex of the originalGrokPattern - * with id = namedregexid and value = namedregex. + * Map of the named regex of the originalGrokPattern with id = namedregexid and value = + * namedregex. */ private final Map namedRegexCollection; - /** - * Original {@code Grok} pattern (expl: %{IP}). - */ + + /** Original {@code Grok} pattern (expl: %{IP}). */ private final String originalGrokPattern; - /** - * Pattern of the namedRegex. - */ + + /** Pattern of the namedRegex. */ private final Pattern compiledNamedRegex; - /** - * {@code Grok} patterns definition. - */ + /** {@code Grok} patterns definition. */ private final Map grokPatternDefinition; public final Set namedGroups; @@ -54,19 +47,16 @@ public class Grok implements Serializable { public final Map> converters; - /** - * only use in grok discovery. - */ + /** only use in grok discovery. */ private String savedPattern = ""; - /** - * Grok. - */ - public Grok(String pattern, - String namedRegex, - Map namedRegexCollection, - Map patternDefinitions, - ZoneId defaultTimeZone) { + /** Grok. */ + public Grok( + String pattern, + String namedRegex, + Map namedRegexCollection, + Map patternDefinitions, + ZoneId defaultTimeZone) { this.originalGrokPattern = pattern; this.namedRegex = namedRegex; this.compiledNamedRegex = Pattern.compile(namedRegex); @@ -132,8 +122,8 @@ public Map getNamedRegexCollection() { } /** - * Match the given log with the named regex. - * And return the json representation of the matched element + * Match the given log with the named regex. And return the json representation of the + * matched element * * @param log : log to match * @return map containing matches @@ -144,8 +134,8 @@ public Map capture(String log) { } /** - * Match the given list of log with the named regex - * and return the list of json representation of the matched elements. + * Match the given list of log with the named regex and return the list of json + * representation of the matched elements. * * @param logs : list of log * @return list of maps containing matches @@ -159,8 +149,8 @@ public ArrayList> capture(List logs) { } /** - * Match the given text with the named regex - * {@code Grok} will extract data from the string and get an extence of {@link Match}. + * Match the given text with the named regex {@code Grok} will extract data from the + * string and get an extence of {@link Match}. * * @param text : Single line of log * @return Grok Match @@ -172,9 +162,7 @@ public Match match(CharSequence text) { Matcher matcher = compiledNamedRegex.matcher(text); if (matcher.find()) { - return new Match( - text, this, matcher, matcher.start(0), matcher.end(0) - ); + return new Match(text, this, matcher, matcher.start(0), matcher.end(0)); } return Match.EMPTY; diff --git a/common/src/main/java/org/opensearch/sql/common/grok/GrokCompiler.java b/common/src/main/java/org/opensearch/sql/common/grok/GrokCompiler.java index 18894fc7a1a..aba96ad4cb1 100644 --- a/common/src/main/java/org/opensearch/sql/common/grok/GrokCompiler.java +++ b/common/src/main/java/org/opensearch/sql/common/grok/GrokCompiler.java @@ -31,13 +31,10 @@ public class GrokCompiler implements Serializable { // We don't want \n and commented line private static final Pattern patternLinePattern = Pattern.compile("^([A-z0-9_]+)\\s+(.*)$"); - /** - * {@code Grok} patterns definitions. - */ + /** {@code Grok} patterns definitions. */ private final Map grokPatternDefinitions = new HashMap<>(); - private GrokCompiler() { - } + private GrokCompiler() {} public static GrokCompiler newInstance() { return new GrokCompiler(); @@ -50,10 +47,10 @@ public Map getPatternDefinitions() { /** * Registers a new pattern definition. * - * @param name : Pattern Name + * @param name : Pattern Name * @param pattern : Regular expression Or {@code Grok} pattern * @throws GrokException runtime expt - **/ + */ public void register(String name, String pattern) { name = Objects.requireNonNull(name).trim(); pattern = Objects.requireNonNull(pattern).trim(); @@ -63,9 +60,7 @@ public void register(String name, String pattern) { } } - /** - * Registers multiple pattern definitions. - */ + /** Registers multiple pattern definitions. */ public void register(Map patternDefinitions) { Objects.requireNonNull(patternDefinitions); patternDefinitions.forEach(this::register); @@ -78,12 +73,9 @@ public void register(InputStream input) throws IOException { register(input, StandardCharsets.UTF_8); } - /** - * Registers multiple pattern definitions from a given inputStream. - */ + /** Registers multiple pattern definitions from a given inputStream. */ public void register(InputStream input, Charset charset) throws IOException { - try ( - BufferedReader in = new BufferedReader(new InputStreamReader(input, charset))) { + try (BufferedReader in = new BufferedReader(new InputStreamReader(input, charset))) { in.lines() .map(patternLinePattern::matcher) .filter(Matcher::matches) @@ -91,11 +83,10 @@ public void register(InputStream input, Charset charset) throws IOException { } } - /** - * Registers multiple pattern definitions from a given Reader. - */ + /** Registers multiple pattern definitions from a given Reader. */ public void register(Reader input) throws IOException { - new BufferedReader(input).lines() + new BufferedReader(input) + .lines() .map(patternLinePattern::matcher) .filter(Matcher::matches) .forEach(m -> register(m.group(1), m.group(2))); @@ -109,9 +100,7 @@ public void registerPatternFromClasspath(String path) throws GrokException { registerPatternFromClasspath(path, StandardCharsets.UTF_8); } - /** - * registerPatternFromClasspath. - */ + /** registerPatternFromClasspath. */ public void registerPatternFromClasspath(String path, Charset charset) throws GrokException { final InputStream inputStream = this.getClass().getResourceAsStream(path); try (Reader reader = new InputStreamReader(inputStream, charset)) { @@ -121,9 +110,7 @@ public void registerPatternFromClasspath(String path, Charset charset) throws Gr } } - /** - * Compiles a given Grok pattern and returns a Grok object which can parse the pattern. - */ + /** Compiles a given Grok pattern and returns a Grok object which can parse the pattern. */ public Grok compile(String pattern) throws IllegalArgumentException { return compile(pattern, false); } @@ -135,11 +122,11 @@ public Grok compile(final String pattern, boolean namedOnly) throws IllegalArgum /** * Compiles a given Grok pattern and returns a Grok object which can parse the pattern. * - * @param pattern : Grok pattern (ex: %{IP}) - * @param defaultTimeZone : time zone used to parse a timestamp when it doesn't contain - * the time zone - * @param namedOnly : Whether to capture named expressions only or not (i.e. %{IP:ip} - * but not ${IP}) + * @param pattern : Grok pattern (ex: %{IP}) + * @param defaultTimeZone : time zone used to parse a timestamp when it doesn't contain the time + * zone + * @param namedOnly : Whether to capture named expressions only or not (i.e. %{IP:ip} but not + * ${IP}) * @return a compiled pattern * @throws IllegalArgumentException when pattern definition is invalid */ @@ -184,14 +171,15 @@ public Grok compile(final String pattern, ZoneId defaultTimeZone, boolean namedO for (int i = 0; i < count; i++) { String definitionOfPattern = patternDefinitions.get(group.get("pattern")); if (definitionOfPattern == null) { - throw new IllegalArgumentException(format("No definition for key '%s' found, aborting", - group.get("pattern"))); + throw new IllegalArgumentException( + format("No definition for key '%s' found, aborting", group.get("pattern"))); } String replacement = String.format("(?%s)", index, definitionOfPattern); if (namedOnly && group.get("subname") == null) { replacement = String.format("(?:%s)", definitionOfPattern); } - namedRegexCollection.put("name" + index, + namedRegexCollection.put( + "name" + index, (group.get("subname") != null ? group.get("subname") : group.get("name"))); namedRegex = StringUtils.replace(namedRegex, "%{" + group.get("name") + "}", replacement, 1); @@ -205,12 +193,6 @@ public Grok compile(final String pattern, ZoneId defaultTimeZone, boolean namedO throw new IllegalArgumentException("Pattern not found"); } - return new Grok( - pattern, - namedRegex, - namedRegexCollection, - patternDefinitions, - defaultTimeZone - ); + return new Grok(pattern, namedRegex, namedRegexCollection, patternDefinitions, defaultTimeZone); } } diff --git a/common/src/main/java/org/opensearch/sql/common/grok/GrokUtils.java b/common/src/main/java/org/opensearch/sql/common/grok/GrokUtils.java index 9ff65acde25..4b145bbbe8a 100644 --- a/common/src/main/java/org/opensearch/sql/common/grok/GrokUtils.java +++ b/common/src/main/java/org/opensearch/sql/common/grok/GrokUtils.java @@ -12,7 +12,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; - /** * {@code GrokUtils} contain set of useful tools or methods. * @@ -20,29 +19,25 @@ */ public class GrokUtils { - /** - * Extract Grok patter like %{FOO} to FOO, Also Grok pattern with semantic. - */ - public static final Pattern GROK_PATTERN = Pattern.compile( - "%\\{" - + "(?" - + "(?[A-z0-9]+)" - + "(?::(?[A-z0-9_:;,\\-\\/\\s\\.']+))?" - + ")" - + "(?:=(?" - + "(?:" - + "(?:[^{}]+|\\.+)+" - + ")+" - + ")" - + ")?" - + "\\}"); - - public static final Pattern NAMED_REGEX = Pattern - .compile("\\(\\?<([a-zA-Z][a-zA-Z0-9]*)>"); - - /** - * getNameGroups. - */ + /** Extract Grok patter like %{FOO} to FOO, Also Grok pattern with semantic. */ + public static final Pattern GROK_PATTERN = + Pattern.compile( + "%\\{" + + "(?" + + "(?[A-z0-9]+)" + + "(?::(?[A-z0-9_:;,\\-\\/\\s\\.']+))?" + + ")" + + "(?:=(?" + + "(?:" + + "(?:[^{}]+|\\.+)+" + + ")+" + + ")" + + ")?" + + "\\}"); + + public static final Pattern NAMED_REGEX = Pattern.compile("\\(\\?<([a-zA-Z][a-zA-Z0-9]*)>"); + + /** getNameGroups. */ public static Set getNameGroups(String regex) { Set namedGroups = new LinkedHashSet<>(); Matcher matcher = NAMED_REGEX.matcher(regex); @@ -52,9 +47,7 @@ public static Set getNameGroups(String regex) { return namedGroups; } - /** - * namedGroups. - */ + /** namedGroups. */ public static Map namedGroups(Matcher matcher, Set groupNames) { Map namedGroups = new LinkedHashMap<>(); for (String groupName : groupNames) { diff --git a/common/src/main/java/org/opensearch/sql/common/grok/Match.java b/common/src/main/java/org/opensearch/sql/common/grok/Match.java index 6831f35ceec..3771817bba7 100644 --- a/common/src/main/java/org/opensearch/sql/common/grok/Match.java +++ b/common/src/main/java/org/opensearch/sql/common/grok/Match.java @@ -5,7 +5,6 @@ package org.opensearch.sql.common.grok; - import static java.lang.String.format; import java.util.ArrayList; @@ -31,9 +30,7 @@ public class Match { private boolean keepEmptyCaptures = true; private Map capture = Collections.emptyMap(); - /** - * Create a new {@code Match} object. - */ + /** Create a new {@code Match} object. */ public Match(CharSequence subject, Grok grok, Matcher match, int start, int end) { this.subject = subject; this.grok = grok; @@ -42,9 +39,7 @@ public Match(CharSequence subject, Grok grok, Matcher match, int start, int end) this.end = end; } - /** - * Create Empty grok matcher. - */ + /** Create Empty grok matcher. */ public static final Match EMPTY = new Match("", null, null, 0, 0); public Matcher getMatch() { @@ -59,9 +54,7 @@ public int getEnd() { return end; } - /** - * Ignore empty captures. - */ + /** Ignore empty captures. */ public void setKeepEmptyCaptures(boolean ignore) { // clear any cached captures if (capture.size() > 0) { @@ -97,8 +90,8 @@ public Map capture() { * * @param flattened will it flatten values. * @return the matched elements. - * @throws GrokException if a keys has multiple non-null values, but only if flattened is set - * to true. + * @throws GrokException if a keys has multiple non-null values, but only if flattened is set to + * true. */ private Map capture(boolean flattened) throws GrokException { if (match == null) { @@ -116,70 +109,69 @@ private Map capture(boolean flattened) throws GrokException { Map mappedw = GrokUtils.namedGroups(this.match, this.grok.namedGroups); - mappedw.forEach((key, valueString) -> { - String id = this.grok.getNamedRegexCollectionById(key); - if (id != null && !id.isEmpty()) { - key = id; - } - - if ("UNWANTED".equals(key)) { - return; - } - - Object value = valueString; - if (valueString != null) { - IConverter converter = grok.converters.get(key); - - if (converter != null) { - key = Converter.extractKey(key); - try { - value = converter.convert(valueString); - } catch (Exception e) { - capture.put(key + "_grokfailure", e.toString()); + mappedw.forEach( + (key, valueString) -> { + String id = this.grok.getNamedRegexCollectionById(key); + if (id != null && !id.isEmpty()) { + key = id; } - if (value instanceof String) { - value = cleanString((String) value); + if ("UNWANTED".equals(key)) { + return; } - } else { - value = cleanString(valueString); - } - } else if (!isKeepEmptyCaptures()) { - return; - } - - if (capture.containsKey(key)) { - Object currentValue = capture.get(key); - if (flattened) { - if (currentValue == null && value != null) { - capture.put(key, value); - } - if (currentValue != null && value != null) { - throw new GrokException( - format( - "key '%s' has multiple non-null values, this is not allowed in flattened mode," - + " values:'%s', '%s'", - key, - currentValue, - value)); + Object value = valueString; + if (valueString != null) { + IConverter converter = grok.converters.get(key); + + if (converter != null) { + key = Converter.extractKey(key); + try { + value = converter.convert(valueString); + } catch (Exception e) { + capture.put(key + "_grokfailure", e.toString()); + } + + if (value instanceof String) { + value = cleanString((String) value); + } + } else { + value = cleanString(valueString); + } + } else if (!isKeepEmptyCaptures()) { + return; } - } else { - if (currentValue instanceof List) { - @SuppressWarnings("unchecked") - List cvl = (List) currentValue; - cvl.add(value); + + if (capture.containsKey(key)) { + Object currentValue = capture.get(key); + + if (flattened) { + if (currentValue == null && value != null) { + capture.put(key, value); + } + if (currentValue != null && value != null) { + throw new GrokException( + format( + "key '%s' has multiple non-null values, this is not allowed in flattened" + + " mode, values:'%s', '%s'", + key, currentValue, value)); + } + } else { + if (currentValue instanceof List) { + @SuppressWarnings("unchecked") + List cvl = (List) currentValue; + cvl.add(value); + } else { + List list = new ArrayList(); + list.add(currentValue); + list.add(value); + capture.put(key, list); + } + } } else { - List list = new ArrayList(); - list.add(currentValue); - list.add(value); - capture.put(key, list); + capture.put(key, value); } - } - } else { - capture.put(key, value); - } - }); + }); capture = Collections.unmodifiableMap(capture); @@ -189,13 +181,11 @@ private Map capture(boolean flattened) throws GrokException { /** * Match to the subject the regex and save the matched element into a map * - *

Multiple values to the same key are flattened to one value: the sole non-null value will - * be captured. - * Should there be multiple non-null values a RuntimeException is being thrown. + *

Multiple values to the same key are flattened to one value: the sole non-null value will be + * captured. Should there be multiple non-null values a RuntimeException is being thrown. * *

This can be used in cases like: (foo (.*:message) bar|bar (.*:message) foo) where the regexp - * guarantees that only - * one value will be captured. + * guarantees that only one value will be captured. * *

See also {@link #capture} which returns multiple values of the same key as list. * @@ -220,9 +210,7 @@ private String cleanString(String value) { char firstChar = value.charAt(0); char lastChar = value.charAt(value.length() - 1); - if (firstChar == lastChar - && (firstChar == '"' || firstChar == '\'') - ) { + if (firstChar == lastChar && (firstChar == '"' || firstChar == '\'')) { if (value.length() <= 2) { return ""; } else { @@ -249,5 +237,4 @@ private String cleanString(String value) { public Boolean isNull() { return this.match == null; } - } diff --git a/common/src/main/java/org/opensearch/sql/common/grok/exception/GrokException.java b/common/src/main/java/org/opensearch/sql/common/grok/exception/GrokException.java index 54ca8aada3a..0e9d6d2ddf4 100644 --- a/common/src/main/java/org/opensearch/sql/common/grok/exception/GrokException.java +++ b/common/src/main/java/org/opensearch/sql/common/grok/exception/GrokException.java @@ -6,9 +6,8 @@ package org.opensearch.sql.common.grok.exception; /** - * Signals that an {@code Grok} exception of some sort has occurred. - * This class is the general class of - * exceptions produced by failed or interrupted Grok operations. + * Signals that an {@code Grok} exception of some sort has occurred. This class is the general class + * of exceptions produced by failed or interrupted Grok operations. * * @since 0.0.4 */ @@ -16,9 +15,7 @@ public class GrokException extends RuntimeException { private static final long serialVersionUID = 1L; - /** - * Creates a new GrokException. - */ + /** Creates a new GrokException. */ public GrokException() { super(); } @@ -27,7 +24,7 @@ public GrokException() { * Constructs a new GrokException. * * @param message the reason for the exception - * @param cause the underlying Throwable that caused this exception to be thrown. + * @param cause the underlying Throwable that caused this exception to be thrown. */ public GrokException(String message, Throwable cause) { super(message, cause); @@ -50,5 +47,4 @@ public GrokException(String message) { public GrokException(Throwable cause) { super(cause); } - } diff --git a/common/src/main/java/org/opensearch/sql/common/response/ResponseListener.java b/common/src/main/java/org/opensearch/sql/common/response/ResponseListener.java index 3d5eadc692a..bac79ddbbd4 100644 --- a/common/src/main/java/org/opensearch/sql/common/response/ResponseListener.java +++ b/common/src/main/java/org/opensearch/sql/common/response/ResponseListener.java @@ -3,7 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.sql.common.response; /** diff --git a/common/src/main/java/org/opensearch/sql/common/setting/LegacySettings.java b/common/src/main/java/org/opensearch/sql/common/setting/LegacySettings.java index 172a0d8023a..e8dc76645ae 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/LegacySettings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/LegacySettings.java @@ -3,42 +3,31 @@ * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.sql.common.setting; import lombok.Getter; import lombok.RequiredArgsConstructor; -/** - * Legacy Open Distro Settings. - */ +/** Legacy Open Distro Settings. */ public abstract class LegacySettings { @RequiredArgsConstructor public enum Key { - /** - * Legacy SQL Settings. - */ + /** Legacy SQL Settings. */ SQL_ENABLED("opendistro.sql.enabled"), SQL_QUERY_SLOWLOG("opendistro.sql.query.slowlog"), SQL_CURSOR_KEEPALIVE("opendistro.sql.cursor.keep_alive"), METRICS_ROLLING_WINDOW("opendistro.sql.metrics.rollingwindow"), METRICS_ROLLING_INTERVAL("opendistro.sql.metrics.rollinginterval"), - /** - * Legacy PPL Settings. - */ + /** Legacy PPL Settings. */ PPL_ENABLED("opendistro.ppl.enabled"), PPL_QUERY_MEMORY_LIMIT("opendistro.ppl.query.memory_limit"), - /** - * Legacy Common Settings. - */ + /** Legacy Common Settings. */ QUERY_SIZE_LIMIT("opendistro.query.size_limit"), - /** - * Deprecated Settings. - */ + /** Deprecated Settings. */ SQL_NEW_ENGINE_ENABLED("opendistro.sql.engine.new.enabled"), QUERY_ANALYSIS_ENABLED("opendistro.sql.query.analysis.enabled"), QUERY_ANALYSIS_SEMANTIC_SUGGESTION("opendistro.sql.query.analysis.semantic.suggestion"), @@ -47,8 +36,7 @@ public enum Key { SQL_CURSOR_ENABLED("opendistro.sql.cursor.enabled"), SQL_CURSOR_FETCH_SIZE("opendistro.sql.cursor.fetch_size"); - @Getter - private final String keyValue; + @Getter private final String keyValue; } public abstract T getSettingValue(Key key); diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 3b0eba157d4..1e5243f91f9 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -3,7 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.sql.common.setting; import com.google.common.base.Strings; @@ -14,29 +13,21 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; -/** - * Setting. - */ +/** Setting. */ public abstract class Settings { @RequiredArgsConstructor public enum Key { - /** - * SQL Settings. - */ + /** SQL Settings. */ SQL_ENABLED("plugins.sql.enabled"), SQL_SLOWLOG("plugins.sql.slowlog"), SQL_CURSOR_KEEP_ALIVE("plugins.sql.cursor.keep_alive"), SQL_DELETE_ENABLED("plugins.sql.delete.enabled"), - /** - * PPL Settings. - */ + /** PPL Settings. */ PPL_ENABLED("plugins.ppl.enabled"), - /** - * Common Settings for SQL and PPL. - */ + /** Common Settings for SQL and PPL. */ QUERY_MEMORY_LIMIT("plugins.query.memory_limit"), QUERY_SIZE_LIMIT("plugins.query.size_limit"), ENCYRPTION_MASTER_KEY("plugins.query.datasources.encryption.masterkey"), @@ -47,8 +38,7 @@ public enum Key { CLUSTER_NAME("cluster.name"); - @Getter - private final String keyValue; + @Getter private final String keyValue; private static final Map ALL_KEYS; @@ -66,9 +56,7 @@ public static Optional of(String keyValue) { } } - /** - * Get Setting Value. - */ + /** Get Setting Value. */ public abstract T getSettingValue(Key key); public abstract List getSettings(); diff --git a/common/src/main/java/org/opensearch/sql/common/utils/QueryContext.java b/common/src/main/java/org/opensearch/sql/common/utils/QueryContext.java index ab11029d73e..686263238aa 100644 --- a/common/src/main/java/org/opensearch/sql/common/utils/QueryContext.java +++ b/common/src/main/java/org/opensearch/sql/common/utils/QueryContext.java @@ -3,35 +3,30 @@ * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.sql.common.utils; -import java.time.LocalDateTime; import java.util.Map; -import java.util.Optional; import java.util.UUID; import org.apache.logging.log4j.ThreadContext; /** - * Utility class for recording and accessing context for the query being executed. - * Implementation Details: context variables is being persisted statically in the thread context + * Utility class for recording and accessing context for the query being executed. Implementation + * Details: context variables is being persisted statically in the thread context + * * @see: @ThreadContext */ public class QueryContext { - /** - * The key of the request id in the context map. - */ + /** The key of the request id in the context map. */ private static final String REQUEST_ID_KEY = "request_id"; /** * Generates a random UUID and adds to the {@link ThreadContext} as the request id. - *

- * Note: If a request id already present, this method will overwrite it with a new - * one. This is to pre-vent re-using the same request id for different requests in - * case the same thread handles both of them. But this also means one should not - * call this method twice on the same thread within the lifetime of the request. - *

+ * + *

Note: If a request id already present, this method will overwrite it with a new one. This is + * to pre-vent re-using the same request id for different requests in case the same thread handles + * both of them. But this also means one should not call this method twice on the same thread + * within the lifetime of the request. */ public static String addRequestId() { var id = UUID.randomUUID().toString(); @@ -41,6 +36,7 @@ public static String addRequestId() { /** * Get RequestID. + * * @return the current request id from {@link ThreadContext}. */ public static String getRequestId() { @@ -52,8 +48,8 @@ public static String getRequestId() { } /** - * Wraps a given instance of {@link Runnable} into a new one which gets all the - * entries from current ThreadContext map. + * Wraps a given instance of {@link Runnable} into a new one which gets all the entries from + * current ThreadContext map. * * @param task the instance of Runnable to wrap * @return the new task diff --git a/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java b/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java index 06992453383..c81f56ef634 100644 --- a/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java +++ b/common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java @@ -3,7 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.sql.common.utils; import com.google.common.base.Strings; @@ -13,6 +12,7 @@ public class StringUtils { /** * Unquote any string with mark specified. + * * @param text string * @param mark quotation mark * @return An unquoted string whose outer pair of (single/double/back-tick) quotes have been @@ -26,11 +26,10 @@ public static String unquote(String text, String mark) { } /** - * Unquote Identifier which has " or ' or ` as mark. - * Strings quoted by ' or " with two of these quotes appearing next to each other in the quote - * acts as an escape - * Example: 'Test''s' will result in 'Test's', similar with those single quotes being replaced - * with double. + * Unquote Identifier which has " or ' or ` as mark. Strings quoted by ' or " with two of these + * quotes appearing next to each other in the quote acts as an escape Example: 'Test''s' will + * result in 'Test's', similar with those single quotes being replaced with double. + * * @param text string * @return An unquoted string whose outer pair of (single/double/back-tick) quotes have been * removed @@ -45,10 +44,7 @@ public static String unquoteText(String text) { char firstChar = text.charAt(0); char lastChar = text.charAt(text.length() - 1); - if (firstChar == lastChar - && (firstChar == '\'' - || firstChar == '"' - || firstChar == '`')) { + if (firstChar == lastChar && (firstChar == '\'' || firstChar == '"' || firstChar == '`')) { enclosingQuote = firstChar; } else { return text; @@ -67,8 +63,7 @@ public static String unquoteText(String text) { for (int chIndex = 1; chIndex < text.length() - 1; chIndex++) { currentChar = text.charAt(chIndex); nextChar = text.charAt(chIndex + 1); - if (currentChar == enclosingQuote - && nextChar == currentChar) { + if (currentChar == enclosingQuote && nextChar == currentChar) { chIndex++; } textSB.append(currentChar); @@ -79,9 +74,9 @@ public static String unquoteText(String text) { /** * Unquote Identifier which has ` as mark. + * * @param identifier identifier that possibly enclosed by double quotes or back ticks - * @return An unquoted string whose outer pair of (double/back-tick) quotes have been - * removed + * @return An unquoted string whose outer pair of (double/back-tick) quotes have been removed */ public static String unquoteIdentifier(String identifier) { if (isQuoted(identifier, "`")) { @@ -92,16 +87,15 @@ public static String unquoteIdentifier(String identifier) { } /** - * Returns a formatted string using the specified format string and - * arguments, as well as the {@link Locale#ROOT} locale. + * Returns a formatted string using the specified format string and arguments, as well as the + * {@link Locale#ROOT} locale. * * @param format format string - * @param args arguments referenced by the format specifiers in the format string + * @param args arguments referenced by the format specifiers in the format string * @return A formatted string * @throws IllegalFormatException If a format string contains an illegal syntax, a format - * specifier that is incompatible with the given arguments, - * insufficient arguments given the format string, or other - * illegal conditions. + * specifier that is incompatible with the given arguments, insufficient arguments given the + * format string, or other illegal conditions. * @see java.lang.String#format(Locale, String, Object...) */ public static String format(final String format, Object... args) { diff --git a/common/src/test/java/org/opensearch/sql/common/authinterceptors/AwsSigningInterceptorTest.java b/common/src/test/java/org/opensearch/sql/common/authinterceptors/AwsSigningInterceptorTest.java index 894f3974ce5..435ac9dc937 100644 --- a/common/src/test/java/org/opensearch/sql/common/authinterceptors/AwsSigningInterceptorTest.java +++ b/common/src/test/java/org/opensearch/sql/common/authinterceptors/AwsSigningInterceptorTest.java @@ -7,9 +7,6 @@ package org.opensearch.sql.common.authinterceptors; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSSessionCredentials; import com.amazonaws.auth.AWSStaticCredentialsProvider; @@ -26,42 +23,40 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.common.authinterceptors.AwsSigningInterceptor; @ExtendWith(MockitoExtension.class) public class AwsSigningInterceptorTest { - @Mock - private Interceptor.Chain chain; + @Mock private Interceptor.Chain chain; - @Captor - ArgumentCaptor requestArgumentCaptor; + @Captor ArgumentCaptor requestArgumentCaptor; - @Mock - private STSAssumeRoleSessionCredentialsProvider stsAssumeRoleSessionCredentialsProvider; + @Mock private STSAssumeRoleSessionCredentialsProvider stsAssumeRoleSessionCredentialsProvider; @Test void testConstructors() { - Assertions.assertThrows(NullPointerException.class, () -> - new AwsSigningInterceptor(null, "us-east-1", "aps")); - Assertions.assertThrows(NullPointerException.class, () -> - new AwsSigningInterceptor(getStaticAWSCredentialsProvider("accessKey", "secretKey"), null, - "aps")); - Assertions.assertThrows(NullPointerException.class, () -> - new AwsSigningInterceptor(getStaticAWSCredentialsProvider("accessKey", "secretKey"), - "us-east-1", null)); + Assertions.assertThrows( + NullPointerException.class, () -> new AwsSigningInterceptor(null, "us-east-1", "aps")); + Assertions.assertThrows( + NullPointerException.class, + () -> + new AwsSigningInterceptor( + getStaticAWSCredentialsProvider("accessKey", "secretKey"), null, "aps")); + Assertions.assertThrows( + NullPointerException.class, + () -> + new AwsSigningInterceptor( + getStaticAWSCredentialsProvider("accessKey", "secretKey"), "us-east-1", null)); } @Test @SneakyThrows void testIntercept() { - Mockito.when(chain.request()).thenReturn(new Request.Builder() - .url("http://localhost:9090") - .build()); - AwsSigningInterceptor awsSigningInterceptor - = new AwsSigningInterceptor( - getStaticAWSCredentialsProvider("testAccessKey", "testSecretKey"), - "us-east-1", "aps"); + Mockito.when(chain.request()) + .thenReturn(new Request.Builder().url("http://localhost:9090").build()); + AwsSigningInterceptor awsSigningInterceptor = + new AwsSigningInterceptor( + getStaticAWSCredentialsProvider("testAccessKey", "testSecretKey"), "us-east-1", "aps"); awsSigningInterceptor.intercept(chain); Mockito.verify(chain).proceed(requestArgumentCaptor.capture()); Request request = requestArgumentCaptor.getValue(); @@ -70,31 +65,26 @@ void testIntercept() { Assertions.assertNotNull(request.headers("host")); } - @Test @SneakyThrows void testSTSCredentialsProviderInterceptor() { - Mockito.when(chain.request()).thenReturn(new Request.Builder() - .url("http://localhost:9090") - .build()); + Mockito.when(chain.request()) + .thenReturn(new Request.Builder().url("http://localhost:9090").build()); Mockito.when(stsAssumeRoleSessionCredentialsProvider.getCredentials()) .thenReturn(getAWSSessionCredentials()); - AwsSigningInterceptor awsSigningInterceptor - = new AwsSigningInterceptor(stsAssumeRoleSessionCredentialsProvider, - "us-east-1", "aps"); + AwsSigningInterceptor awsSigningInterceptor = + new AwsSigningInterceptor(stsAssumeRoleSessionCredentialsProvider, "us-east-1", "aps"); awsSigningInterceptor.intercept(chain); Mockito.verify(chain).proceed(requestArgumentCaptor.capture()); Request request = requestArgumentCaptor.getValue(); Assertions.assertNotNull(request.headers("Authorization")); Assertions.assertNotNull(request.headers("x-amz-date")); Assertions.assertNotNull(request.headers("host")); - Assertions.assertEquals("session_token", - request.headers("x-amz-security-token").get(0)); + Assertions.assertEquals("session_token", request.headers("x-amz-security-token").get(0)); } - - private AWSCredentialsProvider getStaticAWSCredentialsProvider(String accessKey, - String secretKey) { + private AWSCredentialsProvider getStaticAWSCredentialsProvider( + String accessKey, String secretKey) { return new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey)); } @@ -116,5 +106,4 @@ public String getAWSSecretKey() { } }; } - } diff --git a/common/src/test/java/org/opensearch/sql/common/authinterceptors/BasicAuthenticationInterceptorTest.java b/common/src/test/java/org/opensearch/sql/common/authinterceptors/BasicAuthenticationInterceptorTest.java index 596894da6db..d59928d2efa 100644 --- a/common/src/test/java/org/opensearch/sql/common/authinterceptors/BasicAuthenticationInterceptorTest.java +++ b/common/src/test/java/org/opensearch/sql/common/authinterceptors/BasicAuthenticationInterceptorTest.java @@ -24,29 +24,25 @@ @ExtendWith(MockitoExtension.class) public class BasicAuthenticationInterceptorTest { - @Mock - private Interceptor.Chain chain; + @Mock private Interceptor.Chain chain; - @Captor - ArgumentCaptor requestArgumentCaptor; + @Captor ArgumentCaptor requestArgumentCaptor; @Test void testConstructors() { - Assertions.assertThrows(NullPointerException.class, () -> - new BasicAuthenticationInterceptor(null, "test")); - Assertions.assertThrows(NullPointerException.class, () -> - new BasicAuthenticationInterceptor("testAdmin", null)); + Assertions.assertThrows( + NullPointerException.class, () -> new BasicAuthenticationInterceptor(null, "test")); + Assertions.assertThrows( + NullPointerException.class, () -> new BasicAuthenticationInterceptor("testAdmin", null)); } - @Test @SneakyThrows void testIntercept() { - Mockito.when(chain.request()).thenReturn(new Request.Builder() - .url("http://localhost:9090") - .build()); - BasicAuthenticationInterceptor basicAuthenticationInterceptor - = new BasicAuthenticationInterceptor("testAdmin", "testPassword"); + Mockito.when(chain.request()) + .thenReturn(new Request.Builder().url("http://localhost:9090").build()); + BasicAuthenticationInterceptor basicAuthenticationInterceptor = + new BasicAuthenticationInterceptor("testAdmin", "testPassword"); basicAuthenticationInterceptor.intercept(chain); Mockito.verify(chain).proceed(requestArgumentCaptor.capture()); Request request = requestArgumentCaptor.getValue(); @@ -54,5 +50,4 @@ void testIntercept() { Collections.singletonList(Credentials.basic("testAdmin", "testPassword")), request.headers("Authorization")); } - } diff --git a/common/src/test/java/org/opensearch/sql/common/grok/ApacheDataTypeTest.java b/common/src/test/java/org/opensearch/sql/common/grok/ApacheDataTypeTest.java index 09695c8220a..7eb0e964dee 100644 --- a/common/src/test/java/org/opensearch/sql/common/grok/ApacheDataTypeTest.java +++ b/common/src/test/java/org/opensearch/sql/common/grok/ApacheDataTypeTest.java @@ -5,7 +5,6 @@ package org.opensearch.sql.common.grok; - import static org.junit.Assert.assertEquals; import com.google.common.io.Resources; @@ -42,12 +41,13 @@ public void setup() throws Exception { @Test public void test002_httpd_access_semi() throws GrokException { - Grok grok = compiler.compile( - "%{IPORHOST:clientip} %{USER:ident;boolean} %{USER:auth} " - + "\\[%{HTTPDATE:timestamp;date;dd/MMM/yyyy:HH:mm:ss Z}\\] \"(?:%{WORD:verb;string} " - + "%{NOTSPACE:request}" - + "(?: HTTP/%{NUMBER:httpversion;float})?|%{DATA:rawrequest})\" %{NUMBER:response;int} " - + "(?:%{NUMBER:bytes;long}|-)"); + Grok grok = + compiler.compile( + "%{IPORHOST:clientip} %{USER:ident;boolean} %{USER:auth}" + + " \\[%{HTTPDATE:timestamp;date;dd/MMM/yyyy:HH:mm:ss Z}\\]" + + " \"(?:%{WORD:verb;string} %{NOTSPACE:request}(?:" + + " HTTP/%{NUMBER:httpversion;float})?|%{DATA:rawrequest})\" %{NUMBER:response;int}" + + " (?:%{NUMBER:bytes;long}|-)"); System.out.println(line); Match gm = grok.match(line); @@ -61,17 +61,17 @@ public void test002_httpd_access_semi() throws GrokException { assertEquals(map.get("httpversion"), 1.1f); assertEquals(map.get("bytes"), 12846L); assertEquals("GET", map.get("verb")); - } @Test public void test002_httpd_access_colon() throws GrokException { - Grok grok = compiler.compile( - "%{IPORHOST:clientip} %{USER:ident:boolean} %{USER:auth} " - + "\\[%{HTTPDATE:timestamp:date:dd/MMM/yyyy:HH:mm:ss Z}\\] \"(?:%{WORD:verb:string} " - + "%{NOTSPACE:request}" - + "(?: HTTP/%{NUMBER:httpversion:float})?|%{DATA:rawrequest})\" %{NUMBER:response:int} " - + "(?:%{NUMBER:bytes:long}|-)"); + Grok grok = + compiler.compile( + "%{IPORHOST:clientip} %{USER:ident:boolean} %{USER:auth}" + + " \\[%{HTTPDATE:timestamp:date:dd/MMM/yyyy:HH:mm:ss Z}\\]" + + " \"(?:%{WORD:verb:string} %{NOTSPACE:request}(?:" + + " HTTP/%{NUMBER:httpversion:float})?|%{DATA:rawrequest})\" %{NUMBER:response:int}" + + " (?:%{NUMBER:bytes:long}|-)"); Match gm = grok.match(line); Map map = gm.capture(); @@ -85,6 +85,5 @@ public void test002_httpd_access_colon() throws GrokException { assertEquals(map.get("httpversion"), 1.1f); assertEquals(map.get("bytes"), 12846L); assertEquals("GET", map.get("verb")); - } } diff --git a/common/src/test/java/org/opensearch/sql/common/grok/ApacheTest.java b/common/src/test/java/org/opensearch/sql/common/grok/ApacheTest.java index 33113d1996e..db420b16d35 100644 --- a/common/src/test/java/org/opensearch/sql/common/grok/ApacheTest.java +++ b/common/src/test/java/org/opensearch/sql/common/grok/ApacheTest.java @@ -5,7 +5,6 @@ package org.opensearch.sql.common.grok; - import com.google.common.io.Resources; import java.io.BufferedReader; import java.io.File; @@ -65,5 +64,4 @@ public void test002_nasa_httpd_access() throws GrokException, IOException { br.close(); } } - } diff --git a/common/src/test/java/org/opensearch/sql/common/grok/BasicTest.java b/common/src/test/java/org/opensearch/sql/common/grok/BasicTest.java index 26df7ba57ed..c724b58f3ee 100644 --- a/common/src/test/java/org/opensearch/sql/common/grok/BasicTest.java +++ b/common/src/test/java/org/opensearch/sql/common/grok/BasicTest.java @@ -33,8 +33,7 @@ @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class BasicTest { - @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(); + @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); private GrokCompiler compiler; @@ -111,8 +110,8 @@ public void test005_testLoadPatternFromFile() throws IOException, GrokException public void test006_testLoadPatternFromFileIso_8859_1() throws IOException, GrokException { File temp = tempFolder.newFile("grok-tmp-pattern"); try (FileOutputStream fis = new FileOutputStream(temp); - BufferedWriter bw = new BufferedWriter( - new OutputStreamWriter(fis, StandardCharsets.ISO_8859_1))) { + BufferedWriter bw = + new BufferedWriter(new OutputStreamWriter(fis, StandardCharsets.ISO_8859_1))) { bw.write("TEST \\u2022"); } @@ -130,5 +129,4 @@ public void test007_testLoadPatternFromReader() throws IOException, GrokExceptio Grok grok = compiler.compile("%{TEST}"); assertEquals("(?\\u20AC)", grok.getNamedRegex()); } - } diff --git a/common/src/test/java/org/opensearch/sql/common/grok/CaptureTest.java b/common/src/test/java/org/opensearch/sql/common/grok/CaptureTest.java index 1173541e161..60e2761c83d 100644 --- a/common/src/test/java/org/opensearch/sql/common/grok/CaptureTest.java +++ b/common/src/test/java/org/opensearch/sql/common/grok/CaptureTest.java @@ -98,7 +98,8 @@ public void test005_captureSubName() throws GrokException { Map map = match.capture(); assertEquals(1, map.size()); assertEquals("Hello", map.get(subname).toString()); - assertEquals("{abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_abcdef=Hello}", + assertEquals( + "{abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_abcdef=Hello}", map.toString()); } @@ -145,7 +146,8 @@ public void test008_flattenDuplicateKeys() throws GrokException { m3.captureFlattened(); fail("should report error due tu ambiguity"); } catch (RuntimeException e) { - assertThat(e.getMessage(), + assertThat( + e.getMessage(), containsString("has multiple non-null values, this is not allowed in flattened mode")); } } diff --git a/common/src/test/java/org/opensearch/sql/common/grok/GrokDocumentationTest.java b/common/src/test/java/org/opensearch/sql/common/grok/GrokDocumentationTest.java index 22115a825f9..15d450e812f 100644 --- a/common/src/test/java/org/opensearch/sql/common/grok/GrokDocumentationTest.java +++ b/common/src/test/java/org/opensearch/sql/common/grok/GrokDocumentationTest.java @@ -40,23 +40,43 @@ public void assureCodeInReadmeWorks() { Assertions.assertThat(capture).hasSize(22); final boolean debug = false; - final Object[] keywordArray = new Object[] {"COMBINEDAPACHELOG", - "COMMONAPACHELOG", "clientip", "ident", "auth", "timestamp", "MONTHDAY", - "MONTH", "YEAR", "TIME", "HOUR", "MINUTE", "SECOND", "INT", "verb", - "httpversion", "rawrequest", "request", "response", "bytes", "referrer", - "agent"}; + final Object[] keywordArray = + new Object[] { + "COMBINEDAPACHELOG", + "COMMONAPACHELOG", + "clientip", + "ident", + "auth", + "timestamp", + "MONTHDAY", + "MONTH", + "YEAR", + "TIME", + "HOUR", + "MINUTE", + "SECOND", + "INT", + "verb", + "httpversion", + "rawrequest", + "request", + "response", + "bytes", + "referrer", + "agent" + }; if (debug) { capture.keySet().stream().forEach(System.err::println); } - assertTrue(new HashSet(Arrays.asList(keywordArray)) - .containsAll(new HashSet(capture.keySet()))); + assertTrue( + new HashSet(Arrays.asList(keywordArray)) + .containsAll(new HashSet(capture.keySet()))); Arrays.asList(keywordArray).stream() .forEach(o -> assertThat(capture.keySet(), hasItem((String) o))); - assertThat(new HashSet(capture.keySet()), - containsInAnyOrder(keywordArray)); - assertTrue(new HashSet(capture.keySet()) - .containsAll(new HashSet(Arrays.asList(keywordArray)))); - + assertThat(new HashSet(capture.keySet()), containsInAnyOrder(keywordArray)); + assertTrue( + new HashSet(capture.keySet()) + .containsAll(new HashSet(Arrays.asList(keywordArray)))); } } diff --git a/common/src/test/java/org/opensearch/sql/common/grok/GrokTest.java b/common/src/test/java/org/opensearch/sql/common/grok/GrokTest.java index b5e83668076..862f9b8195e 100644 --- a/common/src/test/java/org/opensearch/sql/common/grok/GrokTest.java +++ b/common/src/test/java/org/opensearch/sql/common/grok/GrokTest.java @@ -37,7 +37,6 @@ import org.junit.runners.MethodSorters; import org.opensearch.sql.common.grok.exception.GrokException; - @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class GrokTest { @@ -138,7 +137,6 @@ public void test002_numbers() { Match gm = grok.match("-42"); Map map = gm.capture(); assertEquals("{NUMBER=-42}", map.toString()); - } @Test @@ -152,7 +150,6 @@ public void test003_word() { gm = grok.match("abc"); map = gm.capture(); assertEquals("{WORD=abc}", map.toString()); - } @Test @@ -162,7 +159,6 @@ public void test004_space() { Match gm = grok.match("abc dc"); Map map = gm.capture(); assertEquals("{SPACE=}", map.toString()); - } @Test @@ -172,7 +168,6 @@ public void test004_number() { Match gm = grok.match("Something costs $55.4!"); Map map = gm.capture(); assertEquals("{NUMBER=55.4}", map.toString()); - } @Test @@ -182,7 +177,6 @@ public void test005_notSpace() { Match gm = grok.match("abc dc"); Map map = gm.capture(); assertEquals("{NOTSPACE=abc}", map.toString()); - } @Test @@ -209,7 +203,6 @@ public void test007_uuid() { gm = grok.match("03A8413C-F604-4D21-8F4D-24B19D98B5A7"); map = gm.capture(); assertEquals("{UUID=03A8413C-F604-4D21-8F4D-24B19D98B5A7}", map.toString()); - } @Test @@ -219,7 +212,6 @@ public void test008_mac() { Match gm = grok.match("5E:FF:56:A2:AF:15"); Map map = gm.capture(); assertEquals("{MAC=5E:FF:56:A2:AF:15}", map.toString()); - } @Test @@ -241,10 +233,12 @@ public void test010_hostPort() { Match gm = grok.match("www.google.fr:80"); Map map = gm.capture(); - assertEquals(ImmutableMap.of( - "HOSTPORT", "www.google.fr:80", - "IPORHOST", "www.google.fr", - "PORT", "80"), map); + assertEquals( + ImmutableMap.of( + "HOSTPORT", "www.google.fr:80", + "IPORHOST", "www.google.fr", + "PORT", "80"), + map); } @Test @@ -267,10 +261,11 @@ public void test011_combineApache() { assertEquals(map.get("TIME").toString(), "01:36:30"); gm = - grok.match("112.169.19.192 - - [06/Mar/2013:01:36:30 +0900] \"GET " - + "/wp-content/plugins/easy-table/themes/default/style.css?ver=1.0 HTTP/1.1\" " - + "304 - \"http://www.nflabs.com/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) " - + "AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.152 Safari/537.22\""); + grok.match( + "112.169.19.192 - - [06/Mar/2013:01:36:30 +0900] \"GET" + + " /wp-content/plugins/easy-table/themes/default/style.css?ver=1.0 HTTP/1.1\" 304" + + " - \"http://www.nflabs.com/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2)" + + " AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.152 Safari/537.22\""); map = gm.capture(); assertEquals( map.get("agent").toString(), @@ -278,7 +273,8 @@ public void test011_combineApache() { + "Chrome/25.0.1364.152 Safari/537.22"); assertEquals(map.get("clientip").toString(), "112.169.19.192"); assertEquals(map.get("httpversion").toString(), "1.1"); - assertEquals(map.get("request").toString(), + assertEquals( + map.get("request").toString(), "/wp-content/plugins/easy-table/themes/default/style.css?ver=1.0"); assertEquals(map.get("TIME").toString(), "01:36:30"); } @@ -319,7 +315,7 @@ public void test013_IpSet() throws Throwable { Grok grok = compiler.compile("%{IP}"); try (FileReader fr = new FileReader(Resources.getResource(ResourceManager.IP).getFile()); - BufferedReader br = new BufferedReader(fr)) { + BufferedReader br = new BufferedReader(fr)) { String line; System.out.println("Starting test with ip"); while ((line = br.readLine()) != null) { @@ -336,10 +332,31 @@ public void test014_month() { Grok grok = compiler.compile("%{MONTH}"); - String[] months = - {"Jan", "January", "Feb", "February", "Mar", "March", "Apr", "April", "May", "Jun", "June", - "Jul", "July", "Aug", "August", "Sep", "September", "Oct", "October", "Nov", - "November", "Dec", "December"}; + String[] months = { + "Jan", + "January", + "Feb", + "February", + "Mar", + "March", + "Apr", + "April", + "May", + "Jun", + "June", + "Jul", + "July", + "Aug", + "August", + "Sep", + "September", + "Oct", + "October", + "Nov", + "November", + "Dec", + "December" + }; int counter = 0; for (String month : months) { Match match = grok.match(month); @@ -355,20 +372,21 @@ public void test015_iso8601() throws GrokException { Grok grok = compiler.compile("%{TIMESTAMP_ISO8601}"); String[] times = { - "2001-01-01T00:00:00", - "1974-03-02T04:09:09", - "2010-05-03T08:18:18+00:00", - "2004-07-04T12:27:27-00:00", - "2001-09-05T16:36:36+0000", - "2001-11-06T20:45:45-0000", - "2001-12-07T23:54:54Z", - "2001-01-01T00:00:00.123456", - "1974-03-02T04:09:09.123456", - "2010-05-03T08:18:18.123456+00:00", - "2004-07-04T12:27:27.123456-00:00", - "2001-09-05T16:36:36.123456+0000", - "2001-11-06T20:45:45.123456-0000", - "2001-12-07T23:54:54.123456Z"}; + "2001-01-01T00:00:00", + "1974-03-02T04:09:09", + "2010-05-03T08:18:18+00:00", + "2004-07-04T12:27:27-00:00", + "2001-09-05T16:36:36+0000", + "2001-11-06T20:45:45-0000", + "2001-12-07T23:54:54Z", + "2001-01-01T00:00:00.123456", + "1974-03-02T04:09:09.123456", + "2010-05-03T08:18:18.123456+00:00", + "2004-07-04T12:27:27.123456-00:00", + "2001-09-05T16:36:36.123456+0000", + "2001-11-06T20:45:45.123456-0000", + "2001-12-07T23:54:54.123456Z" + }; int counter = 0; for (String time : times) { @@ -385,33 +403,34 @@ public void test016_uri() throws GrokException { Grok grok = compiler.compile("%{URI}"); String[] uris = { - "http://www.google.com", - "telnet://helloworld", - "http://www.example.com/", - "http://www.example.com/test.html", - "http://www.example.com/test.html?foo=bar", - "http://www.example.com/test.html?foo=bar&fizzle=baz", - "http://www.example.com:80/test.html?foo=bar&fizzle=baz", - "https://www.example.com:443/test.html?foo=bar&fizzle=baz", - "https://user@www.example.com:443/test.html?foo=bar&fizzle=baz", - "https://user:pass@somehost/fetch.pl", - "puppet:///", - "http://www.foo.com", - "http://www.foo.com/", - "http://www.foo.com/?testing", - "http://www.foo.com/?one=two", - "http://www.foo.com/?one=two&foo=bar", - "foo://somehost.com:12345", - "foo://user@somehost.com:12345", - "foo://user@somehost.com:12345/", - "foo://user@somehost.com:12345/foo.bar/baz/fizz", - "foo://user@somehost.com:12345/foo.bar/baz/fizz?test", - "foo://user@somehost.com:12345/foo.bar/baz/fizz?test=1&sink&foo=4", - "http://www.google.com/search?hl=en&source=hp&q=hello+world+%5E%40%23%24&btnG=Google+Search", - "http://www.freebsd.org/cgi/url.cgi?ports/sysutils/grok/pkg-descr", - "http://www.google.com/search?q=CAPTCHA+ssh&start=0&ie=utf-8&oe=utf-8&client=firefox-a" - + "&rls=org.mozilla:en-US:official", - "svn+ssh://somehost:12345/testing"}; + "http://www.google.com", + "telnet://helloworld", + "http://www.example.com/", + "http://www.example.com/test.html", + "http://www.example.com/test.html?foo=bar", + "http://www.example.com/test.html?foo=bar&fizzle=baz", + "http://www.example.com:80/test.html?foo=bar&fizzle=baz", + "https://www.example.com:443/test.html?foo=bar&fizzle=baz", + "https://user@www.example.com:443/test.html?foo=bar&fizzle=baz", + "https://user:pass@somehost/fetch.pl", + "puppet:///", + "http://www.foo.com", + "http://www.foo.com/", + "http://www.foo.com/?testing", + "http://www.foo.com/?one=two", + "http://www.foo.com/?one=two&foo=bar", + "foo://somehost.com:12345", + "foo://user@somehost.com:12345", + "foo://user@somehost.com:12345/", + "foo://user@somehost.com:12345/foo.bar/baz/fizz", + "foo://user@somehost.com:12345/foo.bar/baz/fizz?test", + "foo://user@somehost.com:12345/foo.bar/baz/fizz?test=1&sink&foo=4", + "http://www.google.com/search?hl=en&source=hp&q=hello+world+%5E%40%23%24&btnG=Google+Search", + "http://www.freebsd.org/cgi/url.cgi?ports/sysutils/grok/pkg-descr", + "http://www.google.com/search?q=CAPTCHA+ssh&start=0&ie=utf-8&oe=utf-8&client=firefox-a" + + "&rls=org.mozilla:en-US:official", + "svn+ssh://somehost:12345/testing" + }; int counter = 0; for (String uri : uris) { @@ -429,10 +448,7 @@ public void test017_nonMachingList() throws GrokException { Grok grok = compiler.compile("%{URI}"); String[] uris = { - "http://www.google.com", - "telnet://helloworld", - "", - "svn+ssh://somehost:12345/testing" + "http://www.google.com", "telnet://helloworld", "", "svn+ssh://somehost:12345/testing" }; int counter = 0; @@ -458,9 +474,7 @@ public void test018_namedOnlySimpleCase() throws GrokException { String text = "<< barfoobarfoo >>"; Match match = grok.match(text); Map map = match.capture(); - assertEquals("unable to parse: " + text, - text, - map.get("text")); + assertEquals("unable to parse: " + text, text, map.get("text")); } @Test @@ -488,9 +502,7 @@ private void testPatternRepetitions(boolean namedOnly, String pattern) throws Gr private void assertMatches(String description, Grok grok, String text) { Match match = grok.match(text); Map map = match.capture(); - assertEquals(format("%s: unable to parse '%s'", description, text), - text, - map.get("text")); + assertEquals(format("%s: unable to parse '%s'", description, text), text, map.get("text")); } @Test @@ -630,8 +642,8 @@ public void createGrokWithDefaultPatterns() throws GrokException { compiler.compile("%{USERNAME}", false); } - private void ensureAbortsWithDefinitionMissing(String pattern, String compilePattern, - boolean namedOnly) { + private void ensureAbortsWithDefinitionMissing( + String pattern, String compilePattern, boolean namedOnly) { try { compiler.compile(pattern); compiler.compile(compilePattern, namedOnly); @@ -643,10 +655,11 @@ private void ensureAbortsWithDefinitionMissing(String pattern, String compilePat @Test public void testGroupTypes() { - Grok grok = compiler.compile( - "%{HTTPDATE:timestamp;date;dd/MMM/yyyy:HH:mm:ss Z} %{USERNAME:username:text} " - + "%{IPORHOST:host}:%{POSINT:port:integer}", - true); + Grok grok = + compiler.compile( + "%{HTTPDATE:timestamp;date;dd/MMM/yyyy:HH:mm:ss Z} %{USERNAME:username:text} " + + "%{IPORHOST:host}:%{POSINT:port:integer}", + true); assertEquals(Converter.Type.DATETIME, grok.groupTypes.get("timestamp")); assertEquals(Converter.Type.STRING, grok.groupTypes.get("username")); assertEquals(Converter.Type.INT, grok.groupTypes.get("port")); @@ -667,8 +680,8 @@ public void testTimeZone() { DateTimeFormatter dtf = DateTimeFormatter.ofPattern("MM/dd/yyyy HH:mm:ss"); Grok grok = compiler.compile("%{DATESTAMP:timestamp;date;MM/dd/yyyy HH:mm:ss}", true); Instant instant = (Instant) grok.match(date).capture().get("timestamp"); - assertEquals(ZonedDateTime.parse(date, dtf.withZone(ZoneOffset.systemDefault())).toInstant(), - instant); + assertEquals( + ZonedDateTime.parse(date, dtf.withZone(ZoneOffset.systemDefault())).toInstant(), instant); // set default timezone to PST ZoneId pst = ZoneId.of("PST", ZoneId.SHORT_IDS); diff --git a/common/src/test/java/org/opensearch/sql/common/grok/MessagesTest.java b/common/src/test/java/org/opensearch/sql/common/grok/MessagesTest.java index 98cbb3aaebf..930da8caa8b 100644 --- a/common/src/test/java/org/opensearch/sql/common/grok/MessagesTest.java +++ b/common/src/test/java/org/opensearch/sql/common/grok/MessagesTest.java @@ -16,7 +16,6 @@ import org.junit.Test; import org.opensearch.sql.common.grok.exception.GrokException; - public class MessagesTest { @Test @@ -26,8 +25,9 @@ public void test001_linux_messages() throws GrokException, IOException { Grok grok = compiler.compile("%{MESSAGESLOG}"); - BufferedReader br = new BufferedReader( - new FileReader(Resources.getResource(ResourceManager.MESSAGES).getFile())); + BufferedReader br = + new BufferedReader( + new FileReader(Resources.getResource(ResourceManager.MESSAGES).getFile())); String line; System.out.println("Starting test with linux messages log -- may take a while"); while ((line = br.readLine()) != null) { @@ -38,5 +38,4 @@ public void test001_linux_messages() throws GrokException, IOException { } br.close(); } - } diff --git a/common/src/test/java/org/opensearch/sql/common/grok/ResourceManager.java b/common/src/test/java/org/opensearch/sql/common/grok/ResourceManager.java index a13a72cd00a..fba64b59d33 100644 --- a/common/src/test/java/org/opensearch/sql/common/grok/ResourceManager.java +++ b/common/src/test/java/org/opensearch/sql/common/grok/ResourceManager.java @@ -5,9 +5,7 @@ package org.opensearch.sql.common.grok; -/** - * {@code ResourceManager} . - */ +/** {@code ResourceManager} . */ public final class ResourceManager { public static final String PATTERNS = "patterns/patterns"; diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/TableFunction.java b/core/src/main/java/org/opensearch/sql/ast/tree/TableFunction.java index a67a943dcb1..711a8a0ec5f 100644 --- a/core/src/main/java/org/opensearch/sql/ast/tree/TableFunction.java +++ b/core/src/main/java/org/opensearch/sql/ast/tree/TableFunction.java @@ -15,7 +15,7 @@ import org.opensearch.sql.ast.expression.QualifiedName; import org.opensearch.sql.ast.expression.UnresolvedExpression; -/** AST Node for Table Function. */ +/** ASTNode for Table Function. */ @ToString @EqualsAndHashCode(callSuper = false) @RequiredArgsConstructor diff --git a/core/src/main/java/org/opensearch/sql/expression/ReferenceExpression.java b/core/src/main/java/org/opensearch/sql/expression/ReferenceExpression.java index eb510e3b8ae..cfb87940b4a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/ReferenceExpression.java +++ b/core/src/main/java/org/opensearch/sql/expression/ReferenceExpression.java @@ -88,8 +88,7 @@ public String toString() { * e.g. "address.local.state" been resolved to Missing. but "address.project.year" could been * resolved as 1990. * - *

Resolve Rule - * 1. Resolve the full name by combine the paths("x"."y"."z") as whole ("x.y.z"). + *

Resolve Rule 1. Resolve the full name by combine the paths("x"."y"."z") as whole ("x.y.z"). * 2. Resolve the path recursively through ExprValue. * * @param value {@link ExprTupleValue}. diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFormatterUtil.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFormatterUtil.java index 13f9a077e47..339ebe4b2d9 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFormatterUtil.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFormatterUtil.java @@ -138,8 +138,10 @@ interface DateTimeFormatHandler { .put("%Y", (date) -> "0000") .put("%y", (date) -> "00") .put("%D", (date) -> null) - // %f - Microseconds - .put("%f", (date) -> String.format(NANO_SEC_FORMAT, (date.getNano() / 1000))) + .put( + "%f", + (date) -> // %f - Microseconds + String.format(NANO_SEC_FORMAT, (date.getNano() / 1000))) .put("%w", (date) -> null) .put("%U", (date) -> null) .put("%u", (date) -> null) diff --git a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java index 74aab31a309..06b6404f52b 100644 --- a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java @@ -43,6 +43,7 @@ import org.opensearch.sql.storage.write.TableWriteBuilder; import org.opensearch.sql.storage.write.TableWriteOperator; + /** Added for UT coverage */ @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) class LogicalPlanNodeVisitorTest { diff --git a/datasources/build.gradle b/datasources/build.gradle index ef52db23055..830fadbc354 100644 --- a/datasources/build.gradle +++ b/datasources/build.gradle @@ -31,6 +31,9 @@ dependencies { testImplementation 'org.junit.jupiter:junit-jupiter:5.6.2' } +checkstyleTest.ignoreFailures = true +checkstyleMain.ignoreFailures = true + test { useJUnitPlatform() testLogging { diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/auth/AuthenticationType.java b/datasources/src/main/java/org/opensearch/sql/datasources/auth/AuthenticationType.java index 715e72c0c3b..b6581608bf8 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/auth/AuthenticationType.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/auth/AuthenticationType.java @@ -12,8 +12,8 @@ import java.util.Map; public enum AuthenticationType { - - BASICAUTH("basicauth"), AWSSIGV4AUTH("awssigv4"); + BASICAUTH("basicauth"), + AWSSIGV4AUTH("awssigv4"); private String name; diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelper.java b/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelper.java index adcfb0bdfde..75d0ec85396 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelper.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelper.java @@ -8,9 +8,8 @@ import org.opensearch.sql.datasource.model.DataSourceMetadata; /** - * Interface for datasource authorization helper. - * The implementation of this class helps in determining - * if authorization is required and the roles associated with the user. + * Interface for datasource authorization helper. The implementation of this class helps in + * determining if authorization is required and the roles associated with the user. */ public interface DataSourceUserAuthorizationHelper { diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImpl.java b/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImpl.java index cd55991d006..67d747f0bff 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImpl.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImpl.java @@ -19,36 +19,39 @@ public class DataSourceUserAuthorizationHelperImpl implements DataSourceUserAuth private final Client client; private Boolean isAuthorizationRequired() { - String userString = client.threadPool() - .getThreadContext().getTransient( - ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT); + String userString = + client + .threadPool() + .getThreadContext() + .getTransient(ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT); return userString != null; } private List getUserRoles() { - String userString = client.threadPool() - .getThreadContext().getTransient( - ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT); + String userString = + client + .threadPool() + .getThreadContext() + .getTransient(ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT); User user = User.parse(userString); return user.getRoles(); } - @Override public void authorizeDataSource(DataSourceMetadata dataSourceMetadata) { if (isAuthorizationRequired() && !dataSourceMetadata.getName().equals(DEFAULT_DATASOURCE_NAME)) { boolean isAuthorized = false; for (String role : getUserRoles()) { - if (dataSourceMetadata.getAllowedRoles().contains(role) - || role.equals("all_access")) { + if (dataSourceMetadata.getAllowedRoles().contains(role) || role.equals("all_access")) { isAuthorized = true; break; } } if (!isAuthorized) { throw new SecurityException( - String.format("User is not authorized to access datasource %s. " + String.format( + "User is not authorized to access datasource %s. " + "User should be mapped to any of the roles in %s for access.", dataSourceMetadata.getName(), dataSourceMetadata.getAllowedRoles().toString())); } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/Encryptor.java b/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/Encryptor.java index 578b66d0ba9..4572b45f539 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/Encryptor.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/Encryptor.java @@ -24,5 +24,4 @@ public interface Encryptor { * @return String plainText. */ String decrypt(String encryptedText); - } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java b/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java index 98f693eca19..c6abe78394d 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/encryptor/EncryptorImpl.java @@ -25,32 +25,40 @@ public class EncryptorImpl implements Encryptor { @Override public String encrypt(String plainText) { validate(masterKey); - final AwsCrypto crypto = AwsCrypto.builder() - .withCommitmentPolicy(CommitmentPolicy.RequireEncryptRequireDecrypt) - .build(); + final AwsCrypto crypto = + AwsCrypto.builder() + .withCommitmentPolicy(CommitmentPolicy.RequireEncryptRequireDecrypt) + .build(); - JceMasterKey jceMasterKey - = JceMasterKey.getInstance(new SecretKeySpec(masterKey.getBytes(), "AES"), "Custom", - "opensearch.config.master.key", "AES/GCM/NoPadding"); + JceMasterKey jceMasterKey = + JceMasterKey.getInstance( + new SecretKeySpec(masterKey.getBytes(), "AES"), + "Custom", + "opensearch.config.master.key", + "AES/GCM/NoPadding"); - final CryptoResult encryptResult = crypto.encryptData(jceMasterKey, - plainText.getBytes(StandardCharsets.UTF_8)); + final CryptoResult encryptResult = + crypto.encryptData(jceMasterKey, plainText.getBytes(StandardCharsets.UTF_8)); return Base64.getEncoder().encodeToString(encryptResult.getResult()); } @Override public String decrypt(String encryptedText) { validate(masterKey); - final AwsCrypto crypto = AwsCrypto.builder() - .withCommitmentPolicy(CommitmentPolicy.RequireEncryptRequireDecrypt) - .build(); + final AwsCrypto crypto = + AwsCrypto.builder() + .withCommitmentPolicy(CommitmentPolicy.RequireEncryptRequireDecrypt) + .build(); - JceMasterKey jceMasterKey - = JceMasterKey.getInstance(new SecretKeySpec(masterKey.getBytes(), "AES"), "Custom", - "opensearch.config.master.key", "AES/GCM/NoPadding"); + JceMasterKey jceMasterKey = + JceMasterKey.getInstance( + new SecretKeySpec(masterKey.getBytes(), "AES"), + "Custom", + "opensearch.config.master.key", + "AES/GCM/NoPadding"); - final CryptoResult decryptedResult - = crypto.decryptData(jceMasterKey, Base64.getDecoder().decode(encryptedText)); + final CryptoResult decryptedResult = + crypto.decryptData(jceMasterKey, Base64.getDecoder().decode(encryptedText)); return new String(decryptedResult.getResult()); } @@ -65,6 +73,4 @@ private void validate(String masterKey) { + "admin/datasources.rst#master-key-config-for-encrypting-credential-information"); } } - - } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/DataSourceNotFoundException.java b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/DataSourceNotFoundException.java index 484b0b92b29..40b601000c5 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/DataSourceNotFoundException.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/DataSourceNotFoundException.java @@ -7,12 +7,9 @@ package org.opensearch.sql.datasources.exceptions; -/** - * DataSourceNotFoundException. - */ +/** DataSourceNotFoundException. */ public class DataSourceNotFoundException extends RuntimeException { public DataSourceNotFoundException(String message) { super(message); } - } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java index 265b3ddf310..386eb780cd7 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/exceptions/ErrorMessage.java @@ -3,7 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ - package org.opensearch.sql.datasources.exceptions; import com.google.gson.Gson; @@ -12,27 +11,20 @@ import lombok.Getter; import org.opensearch.core.rest.RestStatus; -/** - * Error Message. - */ +/** Error Message. */ public class ErrorMessage { protected Throwable exception; private final int status; - @Getter - private final String type; + @Getter private final String type; - @Getter - private final String reason; + @Getter private final String reason; - @Getter - private final String details; + @Getter private final String details; - /** - * Error Message Constructor. - */ + /** Error Message Constructor. */ public ErrorMessage(Throwable exception, int status) { this.exception = exception; this.status = status; diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionRequest.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionRequest.java index 0cbb2355cae..b01d5b40ddc 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionRequest.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionRequest.java @@ -7,7 +7,6 @@ package org.opensearch.sql.datasources.model.transport; - import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import java.io.IOException; @@ -17,15 +16,11 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.sql.datasource.model.DataSourceMetadata; -public class CreateDataSourceActionRequest - extends ActionRequest { +public class CreateDataSourceActionRequest extends ActionRequest { - @Getter - private DataSourceMetadata dataSourceMetadata; + @Getter private DataSourceMetadata dataSourceMetadata; - /** - * Constructor of CreateDataSourceActionRequest from StreamInput. - */ + /** Constructor of CreateDataSourceActionRequest from StreamInput. */ public CreateDataSourceActionRequest(StreamInput in) throws IOException { super(in); } @@ -38,9 +33,8 @@ public CreateDataSourceActionRequest(DataSourceMetadata dataSourceMetadata) { public ActionRequestValidationException validate() { if (this.dataSourceMetadata.getName().equals(DEFAULT_DATASOURCE_NAME)) { ActionRequestValidationException exception = new ActionRequestValidationException(); - exception - .addValidationError( - "Not allowed to create datasource with name : " + DEFAULT_DATASOURCE_NAME); + exception.addValidationError( + "Not allowed to create datasource with name : " + DEFAULT_DATASOURCE_NAME); return exception; } else { return null; diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionResponse.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionResponse.java index 4ed0464a25e..aeb1e2d3d9a 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionResponse.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/CreateDataSourceActionResponse.java @@ -15,11 +15,9 @@ import org.opensearch.core.common.io.stream.StreamOutput; @RequiredArgsConstructor -public class CreateDataSourceActionResponse - extends ActionResponse { +public class CreateDataSourceActionResponse extends ActionResponse { - @Getter - private final String result; + @Getter private final String result; public CreateDataSourceActionResponse(StreamInput in) throws IOException { super(in); diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionRequest.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionRequest.java index 1eb2d17bff3..d6e3bcb3f9c 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionRequest.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionRequest.java @@ -18,8 +18,7 @@ public class DeleteDataSourceActionRequest extends ActionRequest { - @Getter - private String dataSourceName; + @Getter private String dataSourceName; /** Constructor of DeleteDataSourceActionRequest from StreamInput. */ public DeleteDataSourceActionRequest(StreamInput in) throws IOException { @@ -34,18 +33,15 @@ public DeleteDataSourceActionRequest(String dataSourceName) { public ActionRequestValidationException validate() { if (StringUtils.isEmpty(this.dataSourceName)) { ActionRequestValidationException exception = new ActionRequestValidationException(); - exception - .addValidationError("Datasource Name cannot be empty or null"); + exception.addValidationError("Datasource Name cannot be empty or null"); return exception; } else if (this.dataSourceName.equals(DEFAULT_DATASOURCE_NAME)) { ActionRequestValidationException exception = new ActionRequestValidationException(); - exception - .addValidationError( - "Not allowed to delete datasource with name : " + DEFAULT_DATASOURCE_NAME); + exception.addValidationError( + "Not allowed to delete datasource with name : " + DEFAULT_DATASOURCE_NAME); return exception; } else { return null; } } - } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionResponse.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionResponse.java index ec57c4aee7b..d8c29c2a670 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionResponse.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/DeleteDataSourceActionResponse.java @@ -17,8 +17,7 @@ @RequiredArgsConstructor public class DeleteDataSourceActionResponse extends ActionResponse { - @Getter - private final String result; + @Getter private final String result; public DeleteDataSourceActionResponse(StreamInput in) throws IOException { super(in); @@ -29,5 +28,4 @@ public DeleteDataSourceActionResponse(StreamInput in) throws IOException { public void writeTo(StreamOutput streamOutput) throws IOException { streamOutput.writeString(result); } - } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionRequest.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionRequest.java index 23f48985439..2d9a4de35a6 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionRequest.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionRequest.java @@ -19,12 +19,9 @@ @NoArgsConstructor public class GetDataSourceActionRequest extends ActionRequest { - @Getter - private String dataSourceName; + @Getter private String dataSourceName; - /** - * Constructor of GetDataSourceActionRequest from StreamInput. - */ + /** Constructor of GetDataSourceActionRequest from StreamInput. */ public GetDataSourceActionRequest(StreamInput in) throws IOException { super(in); } @@ -37,13 +34,11 @@ public GetDataSourceActionRequest(String dataSourceName) { public ActionRequestValidationException validate() { if (this.dataSourceName != null && this.dataSourceName.equals(DEFAULT_DATASOURCE_NAME)) { ActionRequestValidationException exception = new ActionRequestValidationException(); - exception - .addValidationError( - "Not allowed to fetch datasource with name : " + DEFAULT_DATASOURCE_NAME); + exception.addValidationError( + "Not allowed to fetch datasource with name : " + DEFAULT_DATASOURCE_NAME); return exception; } else { return null; } } - } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionResponse.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionResponse.java index dccb3e9b524..ac8d5d4c626 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionResponse.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/GetDataSourceActionResponse.java @@ -17,8 +17,7 @@ @RequiredArgsConstructor public class GetDataSourceActionResponse extends ActionResponse { - @Getter - private final String result; + @Getter private final String result; public GetDataSourceActionResponse(StreamInput in) throws IOException { super(in); @@ -29,5 +28,4 @@ public GetDataSourceActionResponse(StreamInput in) throws IOException { public void writeTo(StreamOutput streamOutput) throws IOException { streamOutput.writeString(result); } - } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionRequest.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionRequest.java index 11bc2d1e20a..b502f348e23 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionRequest.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionRequest.java @@ -7,7 +7,6 @@ package org.opensearch.sql.datasources.model.transport; - import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import java.io.IOException; @@ -17,11 +16,9 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.sql.datasource.model.DataSourceMetadata; -public class UpdateDataSourceActionRequest - extends ActionRequest { +public class UpdateDataSourceActionRequest extends ActionRequest { - @Getter - private DataSourceMetadata dataSourceMetadata; + @Getter private DataSourceMetadata dataSourceMetadata; /** Constructor of UpdateDataSourceActionRequest from StreamInput. */ public UpdateDataSourceActionRequest(StreamInput in) throws IOException { @@ -36,9 +33,8 @@ public UpdateDataSourceActionRequest(DataSourceMetadata dataSourceMetadata) { public ActionRequestValidationException validate() { if (this.dataSourceMetadata.getName().equals(DEFAULT_DATASOURCE_NAME)) { ActionRequestValidationException exception = new ActionRequestValidationException(); - exception - .addValidationError( - "Not allowed to update datasource with name : " + DEFAULT_DATASOURCE_NAME); + exception.addValidationError( + "Not allowed to update datasource with name : " + DEFAULT_DATASOURCE_NAME); return exception; } else { return null; diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionResponse.java b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionResponse.java index 88e8c41ea98..0be992d067f 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionResponse.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/model/transport/UpdateDataSourceActionResponse.java @@ -15,11 +15,9 @@ import org.opensearch.core.common.io.stream.StreamOutput; @RequiredArgsConstructor -public class UpdateDataSourceActionResponse - extends ActionResponse { +public class UpdateDataSourceActionResponse extends ActionResponse { - @Getter - private final String result; + @Getter private final String result; public UpdateDataSourceActionResponse(StreamInput in) throws IOException { super(in); diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java index 15735b945a8..b5929d0f202 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/rest/RestDataSourceQueryAction.java @@ -47,7 +47,6 @@ import org.opensearch.sql.datasources.utils.Scheduler; import org.opensearch.sql.datasources.utils.XContentParserUtils; - public class RestDataSourceQueryAction extends BaseRestHandler { public static final String DATASOURCE_ACTIONS = "datasource_actions"; @@ -83,8 +82,9 @@ public List routes() { * Response body: * Ref [org.opensearch.sql.plugin.transport.datasource.model.GetDataSourceActionResponse] */ - new Route(GET, String.format(Locale.ROOT, "%s/{%s}", - BASE_DATASOURCE_ACTION_URL, "dataSourceName")), + new Route( + GET, + String.format(Locale.ROOT, "%s/{%s}", BASE_DATASOURCE_ACTION_URL, "dataSourceName")), new Route(GET, BASE_DATASOURCE_ACTION_URL), /* @@ -107,9 +107,9 @@ public List routes() { * Response body: Ref * [org.opensearch.sql.plugin.transport.datasource.model.DeleteDataSourceActionResponse] */ - new Route(DELETE, String.format(Locale.ROOT, "%s/{%s}", - BASE_DATASOURCE_ACTION_URL, "dataSourceName")) - ); + new Route( + DELETE, + String.format(Locale.ROOT, "%s/{%s}", BASE_DATASOURCE_ACTION_URL, "dataSourceName"))); } @Override @@ -125,101 +125,125 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient case DELETE: return executeDeleteRequest(restRequest, nodeClient); default: - return restChannel - -> restChannel.sendResponse(new BytesRestResponse(RestStatus.METHOD_NOT_ALLOWED, - String.valueOf(restRequest.method()))); + return restChannel -> + restChannel.sendResponse( + new BytesRestResponse( + RestStatus.METHOD_NOT_ALLOWED, String.valueOf(restRequest.method()))); } } - private RestChannelConsumer executePostRequest(RestRequest restRequest, - NodeClient nodeClient) throws IOException { - - DataSourceMetadata dataSourceMetadata - = XContentParserUtils.toDataSourceMetadata(restRequest.contentParser()); - return restChannel -> Scheduler.schedule(nodeClient, - () -> nodeClient.execute(TransportCreateDataSourceAction.ACTION_TYPE, - new CreateDataSourceActionRequest(dataSourceMetadata), - new ActionListener<>() { - @Override - public void onResponse( - CreateDataSourceActionResponse createDataSourceActionResponse) { - restChannel.sendResponse( - new BytesRestResponse(RestStatus.CREATED, "application/json; charset=UTF-8", - createDataSourceActionResponse.getResult())); - } - - @Override - public void onFailure(Exception e) { - handleException(e, restChannel); - } - })); + private RestChannelConsumer executePostRequest(RestRequest restRequest, NodeClient nodeClient) + throws IOException { + + DataSourceMetadata dataSourceMetadata = + XContentParserUtils.toDataSourceMetadata(restRequest.contentParser()); + return restChannel -> + Scheduler.schedule( + nodeClient, + () -> + nodeClient.execute( + TransportCreateDataSourceAction.ACTION_TYPE, + new CreateDataSourceActionRequest(dataSourceMetadata), + new ActionListener<>() { + @Override + public void onResponse( + CreateDataSourceActionResponse createDataSourceActionResponse) { + restChannel.sendResponse( + new BytesRestResponse( + RestStatus.CREATED, + "application/json; charset=UTF-8", + createDataSourceActionResponse.getResult())); + } + + @Override + public void onFailure(Exception e) { + handleException(e, restChannel); + } + })); } - private RestChannelConsumer executeGetRequest(RestRequest restRequest, - NodeClient nodeClient) { + private RestChannelConsumer executeGetRequest(RestRequest restRequest, NodeClient nodeClient) { String dataSourceName = restRequest.param("dataSourceName"); - return restChannel -> Scheduler.schedule(nodeClient, - () -> nodeClient.execute(TransportGetDataSourceAction.ACTION_TYPE, - new GetDataSourceActionRequest(dataSourceName), - new ActionListener<>() { - @Override - public void onResponse(GetDataSourceActionResponse getDataSourceActionResponse) { - restChannel.sendResponse( - new BytesRestResponse(RestStatus.OK, "application/json; charset=UTF-8", - getDataSourceActionResponse.getResult())); - } - - @Override - public void onFailure(Exception e) { - handleException(e, restChannel); - } - })); + return restChannel -> + Scheduler.schedule( + nodeClient, + () -> + nodeClient.execute( + TransportGetDataSourceAction.ACTION_TYPE, + new GetDataSourceActionRequest(dataSourceName), + new ActionListener<>() { + @Override + public void onResponse( + GetDataSourceActionResponse getDataSourceActionResponse) { + restChannel.sendResponse( + new BytesRestResponse( + RestStatus.OK, + "application/json; charset=UTF-8", + getDataSourceActionResponse.getResult())); + } + + @Override + public void onFailure(Exception e) { + handleException(e, restChannel); + } + })); } - private RestChannelConsumer executeUpdateRequest(RestRequest restRequest, - NodeClient nodeClient) throws IOException { - DataSourceMetadata dataSourceMetadata - = XContentParserUtils.toDataSourceMetadata(restRequest.contentParser()); - return restChannel -> Scheduler.schedule(nodeClient, - () -> nodeClient.execute(TransportUpdateDataSourceAction.ACTION_TYPE, - new UpdateDataSourceActionRequest(dataSourceMetadata), - new ActionListener<>() { - @Override - public void onResponse( - UpdateDataSourceActionResponse updateDataSourceActionResponse) { - restChannel.sendResponse( - new BytesRestResponse(RestStatus.OK, "application/json; charset=UTF-8", - updateDataSourceActionResponse.getResult())); - } - - @Override - public void onFailure(Exception e) { - handleException(e, restChannel); - } - })); + private RestChannelConsumer executeUpdateRequest(RestRequest restRequest, NodeClient nodeClient) + throws IOException { + DataSourceMetadata dataSourceMetadata = + XContentParserUtils.toDataSourceMetadata(restRequest.contentParser()); + return restChannel -> + Scheduler.schedule( + nodeClient, + () -> + nodeClient.execute( + TransportUpdateDataSourceAction.ACTION_TYPE, + new UpdateDataSourceActionRequest(dataSourceMetadata), + new ActionListener<>() { + @Override + public void onResponse( + UpdateDataSourceActionResponse updateDataSourceActionResponse) { + restChannel.sendResponse( + new BytesRestResponse( + RestStatus.OK, + "application/json; charset=UTF-8", + updateDataSourceActionResponse.getResult())); + } + + @Override + public void onFailure(Exception e) { + handleException(e, restChannel); + } + })); } - private RestChannelConsumer executeDeleteRequest(RestRequest restRequest, - NodeClient nodeClient) { + private RestChannelConsumer executeDeleteRequest(RestRequest restRequest, NodeClient nodeClient) { String dataSourceName = restRequest.param("dataSourceName"); - return restChannel -> Scheduler.schedule(nodeClient, - () -> nodeClient.execute(TransportDeleteDataSourceAction.ACTION_TYPE, - new DeleteDataSourceActionRequest(dataSourceName), - new ActionListener<>() { - @Override - public void onResponse( - DeleteDataSourceActionResponse deleteDataSourceActionResponse) { - restChannel.sendResponse( - new BytesRestResponse(RestStatus.NO_CONTENT, "application/json; charset=UTF-8", - deleteDataSourceActionResponse.getResult())); - } - - @Override - public void onFailure(Exception e) { - handleException(e, restChannel); - } - })); + return restChannel -> + Scheduler.schedule( + nodeClient, + () -> + nodeClient.execute( + TransportDeleteDataSourceAction.ACTION_TYPE, + new DeleteDataSourceActionRequest(dataSourceName), + new ActionListener<>() { + @Override + public void onResponse( + DeleteDataSourceActionResponse deleteDataSourceActionResponse) { + restChannel.sendResponse( + new BytesRestResponse( + RestStatus.NO_CONTENT, + "application/json; charset=UTF-8", + deleteDataSourceActionResponse.getResult())); + } + + @Override + public void onFailure(Exception e) { + handleException(e, restChannel); + } + })); } private void handleException(Exception e, RestChannel restChannel) { @@ -240,8 +264,7 @@ private void handleException(Exception e, RestChannel restChannel) { private void reportError(final RestChannel channel, final Exception e, final RestStatus status) { channel.sendResponse( - new BytesRestResponse( - status, new ErrorMessage(e, status.getStatus()).toString())); + new BytesRestResponse(status, new ErrorMessage(e, status.getStatus()).toString())); } private static boolean isClientError(Exception e) { @@ -250,5 +273,4 @@ private static boolean isClientError(Exception e) { || e instanceof IllegalArgumentException || e instanceof IllegalStateException; } - } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCache.java b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCache.java index 3fe2954c129..dbcc321b3f2 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCache.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCache.java @@ -4,8 +4,8 @@ import org.opensearch.sql.datasource.model.DataSourceMetadata; /** - * Interface for DataSourceLoaderCache which provides methods for - * fetch, loading and invalidating DataSource cache. + * Interface for DataSourceLoaderCache which provides methods for fetch, loading and invalidating + * DataSource cache. */ public interface DataSourceLoaderCache { @@ -16,5 +16,4 @@ public interface DataSourceLoaderCache { * @return {@link DataSource} */ DataSource getOrLoadDataSource(DataSourceMetadata dataSourceMetadata); - } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImpl.java b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImpl.java index ba9520fc0cd..44454dbd381 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImpl.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImpl.java @@ -12,10 +12,9 @@ import org.opensearch.sql.storage.DataSourceFactory; /** - * Default implementation of DataSourceLoaderCache. This implementation - * utilizes Google Guava Cache {@link Cache} for caching DataSource objects - * against {@link DataSourceMetadata}. Expires the cache objects every 24 hrs after - * the last access. + * Default implementation of DataSourceLoaderCache. This implementation utilizes Google Guava Cache + * {@link Cache} for caching DataSource objects against {@link DataSourceMetadata}. Expires the + * cache objects every 24 hrs after the last access. */ public class DataSourceLoaderCacheImpl implements DataSourceLoaderCache { private final Map dataSourceFactoryMap; @@ -27,24 +26,24 @@ public class DataSourceLoaderCacheImpl implements DataSourceLoaderCache { * @param dataSourceFactorySet set of {@link DataSourceFactory}. */ public DataSourceLoaderCacheImpl(Set dataSourceFactorySet) { - this.dataSourceFactoryMap = dataSourceFactorySet.stream() - .collect(Collectors.toMap(DataSourceFactory::getDataSourceType, f -> f)); - this.dataSourceCache = CacheBuilder.newBuilder() - .maximumSize(1000) - .expireAfterAccess(24, TimeUnit.HOURS) - .build(); + this.dataSourceFactoryMap = + dataSourceFactorySet.stream() + .collect(Collectors.toMap(DataSourceFactory::getDataSourceType, f -> f)); + this.dataSourceCache = + CacheBuilder.newBuilder().maximumSize(1000).expireAfterAccess(24, TimeUnit.HOURS).build(); } @Override public DataSource getOrLoadDataSource(DataSourceMetadata dataSourceMetadata) { DataSource dataSource = this.dataSourceCache.getIfPresent(dataSourceMetadata); if (dataSource == null) { - dataSource = this.dataSourceFactoryMap.get(dataSourceMetadata.getConnector()) - .createDataSource(dataSourceMetadata); + dataSource = + this.dataSourceFactoryMap + .get(dataSourceMetadata.getConnector()) + .createDataSource(dataSourceMetadata); this.dataSourceCache.put(dataSourceMetadata, dataSource); return dataSource; } return dataSource; } - } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceMetadataStorage.java b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceMetadataStorage.java index e6483900c64..4d59c68fa0a 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceMetadataStorage.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceMetadataStorage.java @@ -13,29 +13,26 @@ import org.opensearch.sql.datasource.model.DataSourceMetadata; /** - * Interface for DataSourceMetadata Storage - * which will be only used by DataSourceService for Storage. + * Interface for DataSourceMetadata Storage which will be only used by DataSourceService for + * Storage. */ public interface DataSourceMetadataStorage { /** - * Returns all dataSource Metadata objects. The returned objects won't contain - * any of the credential info. + * Returns all dataSource Metadata objects. The returned objects won't contain any of the + * credential info. * * @return list of {@link DataSourceMetadata}. */ List getDataSourceMetadata(); - /** - * Gets {@link DataSourceMetadata} corresponding to the - * datasourceName from underlying storage. + * Gets {@link DataSourceMetadata} corresponding to the datasourceName from underlying storage. * * @param datasourceName name of the {@link DataSource}. */ Optional getDataSourceMetadata(String datasourceName); - /** * Stores {@link DataSourceMetadata} in underlying storage. * @@ -43,7 +40,6 @@ public interface DataSourceMetadataStorage { */ void createDataSourceMetadata(DataSourceMetadata dataSourceMetadata); - /** * Updates {@link DataSourceMetadata} in underlying storage. * @@ -51,13 +47,10 @@ public interface DataSourceMetadataStorage { */ void updateDataSourceMetadata(DataSourceMetadata dataSourceMetadata); - /** - * Deletes {@link DataSourceMetadata} corresponding to the - * datasourceName from underlying storage. + * Deletes {@link DataSourceMetadata} corresponding to the datasourceName from underlying storage. * * @param datasourceName name of the {@link DataSource}. */ void deleteDataSourceMetadata(String datasourceName); - } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceServiceImpl.java b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceServiceImpl.java index 86afa90c2ba..2ac480bbf2e 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceServiceImpl.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/service/DataSourceServiceImpl.java @@ -41,13 +41,11 @@ public class DataSourceServiceImpl implements DataSourceService { private final DataSourceUserAuthorizationHelper dataSourceUserAuthorizationHelper; - /** - * Construct from the set of {@link DataSourceFactory} at bootstrap time. - */ - public DataSourceServiceImpl(Set dataSourceFactories, - DataSourceMetadataStorage dataSourceMetadataStorage, - DataSourceUserAuthorizationHelper - dataSourceUserAuthorizationHelper) { + /** Construct from the set of {@link DataSourceFactory} at bootstrap time. */ + public DataSourceServiceImpl( + Set dataSourceFactories, + DataSourceMetadataStorage dataSourceMetadataStorage, + DataSourceUserAuthorizationHelper dataSourceUserAuthorizationHelper) { this.dataSourceMetadataStorage = dataSourceMetadataStorage; this.dataSourceUserAuthorizationHelper = dataSourceUserAuthorizationHelper; this.dataSourceLoaderCache = new DataSourceLoaderCacheImpl(dataSourceFactories); @@ -55,8 +53,8 @@ public DataSourceServiceImpl(Set dataSourceFactories, @Override public Set getDataSourceMetadata(boolean isDefaultDataSourceRequired) { - List dataSourceMetadataList - = this.dataSourceMetadataStorage.getDataSourceMetadata(); + List dataSourceMetadataList = + this.dataSourceMetadataStorage.getDataSourceMetadata(); Set dataSourceMetadataSet = new HashSet<>(dataSourceMetadataList); if (isDefaultDataSourceRequired) { dataSourceMetadataSet.add(DataSourceMetadata.defaultOpenSearchDataSourceMetadata()); @@ -67,28 +65,26 @@ public Set getDataSourceMetadata(boolean isDefaultDataSource @Override public DataSourceMetadata getDataSourceMetadata(String datasourceName) { - Optional dataSourceMetadataOptional - = getDataSourceMetadataFromName(datasourceName); + Optional dataSourceMetadataOptional = + getDataSourceMetadataFromName(datasourceName); if (dataSourceMetadataOptional.isEmpty()) { - throw new IllegalArgumentException("DataSource with name: " + datasourceName - + " doesn't exist."); + throw new IllegalArgumentException( + "DataSource with name: " + datasourceName + " doesn't exist."); } removeAuthInfo(dataSourceMetadataOptional.get()); return dataSourceMetadataOptional.get(); } - @Override public DataSource getDataSource(String dataSourceName) { - Optional - dataSourceMetadataOptional = getDataSourceMetadataFromName(dataSourceName); + Optional dataSourceMetadataOptional = + getDataSourceMetadataFromName(dataSourceName); if (dataSourceMetadataOptional.isEmpty()) { throw new DataSourceNotFoundException( String.format("DataSource with name %s doesn't exist.", dataSourceName)); } else { DataSourceMetadata dataSourceMetadata = dataSourceMetadataOptional.get(); - this.dataSourceUserAuthorizationHelper - .authorizeDataSource(dataSourceMetadata); + this.dataSourceUserAuthorizationHelper.authorizeDataSource(dataSourceMetadata); return dataSourceLoaderCache.getOrLoadDataSource(dataSourceMetadata); } } @@ -130,7 +126,6 @@ public Boolean dataSourceExists(String dataSourceName) { || this.dataSourceMetadataStorage.getDataSourceMetadata(dataSourceName).isPresent(); } - /** * This can be moved to a different validator class when we introduce more connectors. * @@ -159,7 +154,6 @@ private Optional getDataSourceMetadataFromName(String dataSo } } - // It is advised to avoid sending any kind credential // info in api response from security point of view. private void removeAuthInfo(Set dataSourceMetadataSet) { @@ -167,11 +161,8 @@ private void removeAuthInfo(Set dataSourceMetadataSet) { } private void removeAuthInfo(DataSourceMetadata dataSourceMetadata) { - HashMap safeProperties - = new HashMap<>(dataSourceMetadata.getProperties()); - safeProperties - .entrySet() - .removeIf(entry -> entry.getKey().contains("auth")); + HashMap safeProperties = new HashMap<>(dataSourceMetadata.getProperties()); + safeProperties.entrySet().removeIf(entry -> entry.getKey().contains("auth")); dataSourceMetadata.setProperties(safeProperties); } } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorage.java b/datasources/src/main/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorage.java index e2927a4b0c7..4eb16924c44 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorage.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorage.java @@ -55,8 +55,8 @@ public class OpenSearchDataSourceMetadataStorage implements DataSourceMetadataSt private static final String DATASOURCE_INDEX_MAPPING_FILE_NAME = "datasources-index-mapping.yml"; private static final Integer DATASOURCE_QUERY_RESULT_SIZE = 10000; - private static final String DATASOURCE_INDEX_SETTINGS_FILE_NAME - = "datasources-index-settings.yml"; + private static final String DATASOURCE_INDEX_SETTINGS_FILE_NAME = + "datasources-index-settings.yml"; private static final Logger LOG = LogManager.getLogger(); private final Client client; private final ClusterService clusterService; @@ -64,15 +64,15 @@ public class OpenSearchDataSourceMetadataStorage implements DataSourceMetadataSt private final Encryptor encryptor; /** - * This class implements DataSourceMetadataStorage interface - * using OpenSearch as underlying storage. + * This class implements DataSourceMetadataStorage interface using OpenSearch as underlying + * storage. * - * @param client opensearch NodeClient. + * @param client opensearch NodeClient. * @param clusterService ClusterService. - * @param encryptor Encryptor. + * @param encryptor Encryptor. */ - public OpenSearchDataSourceMetadataStorage(Client client, ClusterService clusterService, - Encryptor encryptor) { + public OpenSearchDataSourceMetadataStorage( + Client client, ClusterService clusterService, Encryptor encryptor) { this.client = client; this.clusterService = clusterService; this.encryptor = encryptor; @@ -93,8 +93,7 @@ public Optional getDataSourceMetadata(String datasourceName) createDataSourcesIndex(); return Optional.empty(); } - return searchInDataSourcesIndex(QueryBuilders.termQuery("name", datasourceName)) - .stream() + return searchInDataSourcesIndex(QueryBuilders.termQuery("name", datasourceName)).stream() .findFirst() .map(x -> this.encryptDecryptAuthenticationData(x, false)); } @@ -111,14 +110,14 @@ public void createDataSourceMetadata(DataSourceMetadata dataSourceMetadata) { indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); ActionFuture indexResponseActionFuture; IndexResponse indexResponse; - try (ThreadContext.StoredContext storedContext = client.threadPool().getThreadContext() - .stashContext()) { + try (ThreadContext.StoredContext storedContext = + client.threadPool().getThreadContext().stashContext()) { indexRequest.source(XContentParserUtils.convertToXContent(dataSourceMetadata)); indexResponseActionFuture = client.index(indexRequest); indexResponse = indexResponseActionFuture.actionGet(); } catch (VersionConflictEngineException exception) { - throw new IllegalArgumentException("A datasource already exists with name: " - + dataSourceMetadata.getName()); + throw new IllegalArgumentException( + "A datasource already exists with name: " + dataSourceMetadata.getName()); } catch (Exception e) { throw new RuntimeException(e); } @@ -126,27 +125,27 @@ public void createDataSourceMetadata(DataSourceMetadata dataSourceMetadata) { if (indexResponse.getResult().equals(DocWriteResponse.Result.CREATED)) { LOG.debug("DatasourceMetadata : {} successfully created", dataSourceMetadata.getName()); } else { - throw new RuntimeException("Saving dataSource metadata information failed with result : " - + indexResponse.getResult().getLowercase()); + throw new RuntimeException( + "Saving dataSource metadata information failed with result : " + + indexResponse.getResult().getLowercase()); } } @Override public void updateDataSourceMetadata(DataSourceMetadata dataSourceMetadata) { encryptDecryptAuthenticationData(dataSourceMetadata, true); - UpdateRequest updateRequest - = new UpdateRequest(DATASOURCE_INDEX_NAME, dataSourceMetadata.getName()); + UpdateRequest updateRequest = + new UpdateRequest(DATASOURCE_INDEX_NAME, dataSourceMetadata.getName()); UpdateResponse updateResponse; - try (ThreadContext.StoredContext storedContext = client.threadPool().getThreadContext() - .stashContext()) { + try (ThreadContext.StoredContext storedContext = + client.threadPool().getThreadContext().stashContext()) { updateRequest.doc(XContentParserUtils.convertToXContent(dataSourceMetadata)); updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - ActionFuture updateResponseActionFuture - = client.update(updateRequest); + ActionFuture updateResponseActionFuture = client.update(updateRequest); updateResponse = updateResponseActionFuture.actionGet(); } catch (DocumentMissingException exception) { - throw new DataSourceNotFoundException("Datasource with name: " - + dataSourceMetadata.getName() + " doesn't exist"); + throw new DataSourceNotFoundException( + "Datasource with name: " + dataSourceMetadata.getName() + " doesn't exist"); } catch (Exception e) { throw new RuntimeException(e); } @@ -155,8 +154,9 @@ public void updateDataSourceMetadata(DataSourceMetadata dataSourceMetadata) { || updateResponse.getResult().equals(DocWriteResponse.Result.NOOP)) { LOG.debug("DatasourceMetadata : {} successfully updated", dataSourceMetadata.getName()); } else { - throw new RuntimeException("Saving dataSource metadata information failed with result : " - + updateResponse.getResult().getLowercase()); + throw new RuntimeException( + "Saving dataSource metadata information failed with result : " + + updateResponse.getResult().getLowercase()); } } @@ -165,48 +165,54 @@ public void deleteDataSourceMetadata(String datasourceName) { DeleteRequest deleteRequest = new DeleteRequest(DATASOURCE_INDEX_NAME); deleteRequest.id(datasourceName); ActionFuture deleteResponseActionFuture; - try (ThreadContext.StoredContext storedContext = client.threadPool().getThreadContext() - .stashContext()) { + try (ThreadContext.StoredContext storedContext = + client.threadPool().getThreadContext().stashContext()) { deleteResponseActionFuture = client.delete(deleteRequest); } DeleteResponse deleteResponse = deleteResponseActionFuture.actionGet(); if (deleteResponse.getResult().equals(DocWriteResponse.Result.DELETED)) { LOG.debug("DatasourceMetadata : {} successfully deleted", datasourceName); } else if (deleteResponse.getResult().equals(DocWriteResponse.Result.NOT_FOUND)) { - throw new DataSourceNotFoundException("Datasource with name: " - + datasourceName + " doesn't exist"); + throw new DataSourceNotFoundException( + "Datasource with name: " + datasourceName + " doesn't exist"); } else { - throw new RuntimeException("Deleting dataSource metadata information failed with result : " - + deleteResponse.getResult().getLowercase()); + throw new RuntimeException( + "Deleting dataSource metadata information failed with result : " + + deleteResponse.getResult().getLowercase()); } } private void createDataSourcesIndex() { try { - InputStream mappingFileStream = OpenSearchDataSourceMetadataStorage.class.getClassLoader() - .getResourceAsStream(DATASOURCE_INDEX_MAPPING_FILE_NAME); - InputStream settingsFileStream = OpenSearchDataSourceMetadataStorage.class.getClassLoader() - .getResourceAsStream(DATASOURCE_INDEX_SETTINGS_FILE_NAME); + InputStream mappingFileStream = + OpenSearchDataSourceMetadataStorage.class + .getClassLoader() + .getResourceAsStream(DATASOURCE_INDEX_MAPPING_FILE_NAME); + InputStream settingsFileStream = + OpenSearchDataSourceMetadataStorage.class + .getClassLoader() + .getResourceAsStream(DATASOURCE_INDEX_SETTINGS_FILE_NAME); CreateIndexRequest createIndexRequest = new CreateIndexRequest(DATASOURCE_INDEX_NAME); - createIndexRequest.mapping(IOUtils.toString(mappingFileStream, StandardCharsets.UTF_8), - XContentType.YAML) - .settings(IOUtils.toString(settingsFileStream, StandardCharsets.UTF_8), - XContentType.YAML); + createIndexRequest + .mapping(IOUtils.toString(mappingFileStream, StandardCharsets.UTF_8), XContentType.YAML) + .settings( + IOUtils.toString(settingsFileStream, StandardCharsets.UTF_8), XContentType.YAML); ActionFuture createIndexResponseActionFuture; - try (ThreadContext.StoredContext ignored = client.threadPool().getThreadContext() - .stashContext()) { + try (ThreadContext.StoredContext ignored = + client.threadPool().getThreadContext().stashContext()) { createIndexResponseActionFuture = client.admin().indices().create(createIndexRequest); } CreateIndexResponse createIndexResponse = createIndexResponseActionFuture.actionGet(); if (createIndexResponse.isAcknowledged()) { LOG.info("Index: {} creation Acknowledged", DATASOURCE_INDEX_NAME); } else { - throw new RuntimeException( - "Index creation is not acknowledged."); + throw new RuntimeException("Index creation is not acknowledged."); } } catch (Throwable e) { throw new RuntimeException( - "Internal server error while creating" + DATASOURCE_INDEX_NAME + " index:: " + "Internal server error while creating" + + DATASOURCE_INDEX_NAME + + " index:: " + e.getMessage()); } } @@ -218,17 +224,19 @@ private List searchInDataSourcesIndex(QueryBuilder query) { searchSourceBuilder.query(query); searchSourceBuilder.size(DATASOURCE_QUERY_RESULT_SIZE); searchRequest.source(searchSourceBuilder); - // strongly consistent reads is requred. more info https://github.com/opensearch-project/sql/issues/1801. + // strongly consistent reads is requred. more info + // https://github.com/opensearch-project/sql/issues/1801. searchRequest.preference("_primary"); ActionFuture searchResponseActionFuture; - try (ThreadContext.StoredContext ignored = client.threadPool().getThreadContext() - .stashContext()) { + try (ThreadContext.StoredContext ignored = + client.threadPool().getThreadContext().stashContext()) { searchResponseActionFuture = client.search(searchRequest); } SearchResponse searchResponse = searchResponseActionFuture.actionGet(); if (searchResponse.status().getStatus() != 200) { - throw new RuntimeException("Fetching dataSource metadata information failed with status : " - + searchResponse.status()); + throw new RuntimeException( + "Fetching dataSource metadata information failed with status : " + + searchResponse.status()); } else { List list = new ArrayList<>(); for (SearchHit searchHit : searchResponse.getHits().getHits()) { @@ -246,14 +254,15 @@ private List searchInDataSourcesIndex(QueryBuilder query) { } @SuppressWarnings("missingswitchdefault") - private DataSourceMetadata encryptDecryptAuthenticationData(DataSourceMetadata dataSourceMetadata, - Boolean isEncryption) { + private DataSourceMetadata encryptDecryptAuthenticationData( + DataSourceMetadata dataSourceMetadata, Boolean isEncryption) { Map propertiesMap = dataSourceMetadata.getProperties(); - Optional authTypeOptional - = propertiesMap.keySet().stream().filter(s -> s.endsWith("auth.type")) - .findFirst() - .map(propertiesMap::get) - .map(AuthenticationType::get); + Optional authTypeOptional = + propertiesMap.keySet().stream() + .filter(s -> s.endsWith("auth.type")) + .findFirst() + .map(propertiesMap::get) + .map(AuthenticationType::get); if (authTypeOptional.isPresent()) { switch (authTypeOptional.get()) { case BASICAUTH: @@ -267,8 +276,8 @@ private DataSourceMetadata encryptDecryptAuthenticationData(DataSourceMetadata d return dataSourceMetadata; } - private void handleBasicAuthPropertiesEncryptionDecryption(Map propertiesMap, - Boolean isEncryption) { + private void handleBasicAuthPropertiesEncryptionDecryption( + Map propertiesMap, Boolean isEncryption) { ArrayList list = new ArrayList<>(); propertiesMap.keySet().stream() .filter(s -> s.endsWith("auth.username")) @@ -281,21 +290,19 @@ private void handleBasicAuthPropertiesEncryptionDecryption(Map p encryptOrDecrypt(propertiesMap, isEncryption, list); } - private void encryptOrDecrypt(Map propertiesMap, Boolean isEncryption, - List keyIdentifiers) { + private void encryptOrDecrypt( + Map propertiesMap, Boolean isEncryption, List keyIdentifiers) { for (String key : keyIdentifiers) { if (isEncryption) { - propertiesMap.put(key, - this.encryptor.encrypt(propertiesMap.get(key))); + propertiesMap.put(key, this.encryptor.encrypt(propertiesMap.get(key))); } else { - propertiesMap.put(key, - this.encryptor.decrypt(propertiesMap.get(key))); + propertiesMap.put(key, this.encryptor.decrypt(propertiesMap.get(key))); } } } - private void handleSigV4PropertiesEncryptionDecryption(Map propertiesMap, - Boolean isEncryption) { + private void handleSigV4PropertiesEncryptionDecryption( + Map propertiesMap, Boolean isEncryption) { ArrayList list = new ArrayList<>(); propertiesMap.keySet().stream() .filter(s -> s.endsWith("auth.access_key")) @@ -307,5 +314,4 @@ private void handleSigV4PropertiesEncryptionDecryption(Map prope .ifPresent(list::add); encryptOrDecrypt(propertiesMap, isEncryption, list); } - } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceAction.java index fefd0250ce0..54ca92b6954 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceAction.java @@ -23,38 +23,44 @@ public class TransportCreateDataSourceAction extends HandledTransportAction { public static final String NAME = "cluster:admin/opensearch/ql/datasources/create"; - public static final ActionType - ACTION_TYPE = new ActionType<>(NAME, CreateDataSourceActionResponse::new); + public static final ActionType ACTION_TYPE = + new ActionType<>(NAME, CreateDataSourceActionResponse::new); private DataSourceService dataSourceService; /** * TransportCreateDataSourceAction action for creating datasource. * - * @param transportService transportService. - * @param actionFilters actionFilters. + * @param transportService transportService. + * @param actionFilters actionFilters. * @param dataSourceService dataSourceService. */ @Inject - public TransportCreateDataSourceAction(TransportService transportService, - ActionFilters actionFilters, - DataSourceServiceImpl dataSourceService) { - super(TransportCreateDataSourceAction.NAME, transportService, actionFilters, + public TransportCreateDataSourceAction( + TransportService transportService, + ActionFilters actionFilters, + DataSourceServiceImpl dataSourceService) { + super( + TransportCreateDataSourceAction.NAME, + transportService, + actionFilters, CreateDataSourceActionRequest::new); this.dataSourceService = dataSourceService; } @Override - protected void doExecute(Task task, CreateDataSourceActionRequest request, - ActionListener actionListener) { + protected void doExecute( + Task task, + CreateDataSourceActionRequest request, + ActionListener actionListener) { try { DataSourceMetadata dataSourceMetadata = request.getDataSourceMetadata(); dataSourceService.createDataSource(dataSourceMetadata); - actionListener.onResponse(new CreateDataSourceActionResponse("Created DataSource with name " - + dataSourceMetadata.getName())); + actionListener.onResponse( + new CreateDataSourceActionResponse( + "Created DataSource with name " + dataSourceMetadata.getName())); } catch (Exception e) { actionListener.onFailure(e); } } - } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceAction.java index 39e51aabefd..5578d406510 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceAction.java @@ -23,37 +23,43 @@ public class TransportDeleteDataSourceAction extends HandledTransportAction { public static final String NAME = "cluster:admin/opensearch/ql/datasources/delete"; - public static final ActionType - ACTION_TYPE = new ActionType<>(NAME, DeleteDataSourceActionResponse::new); + public static final ActionType ACTION_TYPE = + new ActionType<>(NAME, DeleteDataSourceActionResponse::new); private DataSourceService dataSourceService; /** * TransportDeleteDataSourceAction action for deleting datasource. * - * @param transportService transportService. - * @param actionFilters actionFilters. + * @param transportService transportService. + * @param actionFilters actionFilters. * @param dataSourceService dataSourceService. */ @Inject - public TransportDeleteDataSourceAction(TransportService transportService, - ActionFilters actionFilters, - DataSourceServiceImpl dataSourceService) { - super(TransportDeleteDataSourceAction.NAME, transportService, actionFilters, + public TransportDeleteDataSourceAction( + TransportService transportService, + ActionFilters actionFilters, + DataSourceServiceImpl dataSourceService) { + super( + TransportDeleteDataSourceAction.NAME, + transportService, + actionFilters, DeleteDataSourceActionRequest::new); this.dataSourceService = dataSourceService; } @Override - protected void doExecute(Task task, DeleteDataSourceActionRequest request, - ActionListener actionListener) { + protected void doExecute( + Task task, + DeleteDataSourceActionRequest request, + ActionListener actionListener) { try { dataSourceService.deleteDataSource(request.getDataSourceName()); - actionListener.onResponse(new DeleteDataSourceActionResponse("Deleted DataSource with name " - + request.getDataSourceName())); + actionListener.onResponse( + new DeleteDataSourceActionResponse( + "Deleted DataSource with name " + request.getDataSourceName())); } catch (Exception e) { actionListener.onFailure(e); } } - } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceAction.java index 477d10fa0b3..34ad59c80fe 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceAction.java @@ -26,30 +26,36 @@ public class TransportGetDataSourceAction extends HandledTransportAction { public static final String NAME = "cluster:admin/opensearch/ql/datasources/read"; - public static final ActionType - ACTION_TYPE = new ActionType<>(NAME, GetDataSourceActionResponse::new); + public static final ActionType ACTION_TYPE = + new ActionType<>(NAME, GetDataSourceActionResponse::new); private DataSourceService dataSourceService; /** * TransportGetDataSourceAction action for getting datasource. * - * @param transportService transportService. - * @param actionFilters actionFilters. + * @param transportService transportService. + * @param actionFilters actionFilters. * @param dataSourceService dataSourceService. */ @Inject - public TransportGetDataSourceAction(TransportService transportService, - ActionFilters actionFilters, - DataSourceServiceImpl dataSourceService) { - super(TransportGetDataSourceAction.NAME, transportService, actionFilters, + public TransportGetDataSourceAction( + TransportService transportService, + ActionFilters actionFilters, + DataSourceServiceImpl dataSourceService) { + super( + TransportGetDataSourceAction.NAME, + transportService, + actionFilters, GetDataSourceActionRequest::new); this.dataSourceService = dataSourceService; } @Override - protected void doExecute(Task task, GetDataSourceActionRequest request, - ActionListener actionListener) { + protected void doExecute( + Task task, + GetDataSourceActionRequest request, + ActionListener actionListener) { try { String responseContent; if (request.getDataSourceName() == null) { @@ -66,30 +72,27 @@ protected void doExecute(Task task, GetDataSourceActionRequest request, private String handleGetAllDataSourcesRequest() { String responseContent; - Set dataSourceMetadataSet = - dataSourceService.getDataSourceMetadata(false); - responseContent = new JsonResponseFormatter>( - JsonResponseFormatter.Style.PRETTY) { - @Override - protected Object buildJsonObject(Set response) { - return response; - } - }.format(dataSourceMetadataSet); + Set dataSourceMetadataSet = dataSourceService.getDataSourceMetadata(false); + responseContent = + new JsonResponseFormatter>(JsonResponseFormatter.Style.PRETTY) { + @Override + protected Object buildJsonObject(Set response) { + return response; + } + }.format(dataSourceMetadataSet); return responseContent; } private String handleSingleDataSourceRequest(String datasourceName) { String responseContent; - DataSourceMetadata dataSourceMetadata - = dataSourceService - .getDataSourceMetadata(datasourceName); - responseContent = new JsonResponseFormatter( - JsonResponseFormatter.Style.PRETTY) { - @Override - protected Object buildJsonObject(DataSourceMetadata response) { - return response; - } - }.format(dataSourceMetadata); + DataSourceMetadata dataSourceMetadata = dataSourceService.getDataSourceMetadata(datasourceName); + responseContent = + new JsonResponseFormatter(JsonResponseFormatter.Style.PRETTY) { + @Override + protected Object buildJsonObject(DataSourceMetadata response) { + return response; + } + }.format(dataSourceMetadata); return responseContent; } } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceAction.java b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceAction.java index e01a8ffea27..4325282f83e 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceAction.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceAction.java @@ -23,8 +23,8 @@ public class TransportUpdateDataSourceAction extends HandledTransportAction { public static final String NAME = "cluster:admin/opensearch/ql/datasources/update"; - public static final ActionType - ACTION_TYPE = new ActionType<>(NAME, UpdateDataSourceActionResponse::new); + public static final ActionType ACTION_TYPE = + new ActionType<>(NAME, UpdateDataSourceActionResponse::new); private DataSourceService dataSourceService; @@ -36,24 +36,30 @@ public class TransportUpdateDataSourceAction * @param dataSourceService dataSourceService. */ @Inject - public TransportUpdateDataSourceAction(TransportService transportService, - ActionFilters actionFilters, - DataSourceServiceImpl dataSourceService) { - super(TransportUpdateDataSourceAction.NAME, transportService, actionFilters, + public TransportUpdateDataSourceAction( + TransportService transportService, + ActionFilters actionFilters, + DataSourceServiceImpl dataSourceService) { + super( + TransportUpdateDataSourceAction.NAME, + transportService, + actionFilters, UpdateDataSourceActionRequest::new); this.dataSourceService = dataSourceService; } @Override - protected void doExecute(Task task, UpdateDataSourceActionRequest request, - ActionListener actionListener) { + protected void doExecute( + Task task, + UpdateDataSourceActionRequest request, + ActionListener actionListener) { try { dataSourceService.updateDataSource(request.getDataSourceMetadata()); - actionListener.onResponse(new UpdateDataSourceActionResponse("Updated DataSource with name " - + request.getDataSourceMetadata().getName())); + actionListener.onResponse( + new UpdateDataSourceActionResponse( + "Updated DataSource with name " + request.getDataSourceMetadata().getName())); } catch (Exception e) { actionListener.onFailure(e); } } - } diff --git a/datasources/src/main/java/org/opensearch/sql/datasources/utils/XContentParserUtils.java b/datasources/src/main/java/org/opensearch/sql/datasources/utils/XContentParserUtils.java index 38a500afaea..1ad79addace 100644 --- a/datasources/src/main/java/org/opensearch/sql/datasources/utils/XContentParserUtils.java +++ b/datasources/src/main/java/org/opensearch/sql/datasources/utils/XContentParserUtils.java @@ -22,9 +22,7 @@ import org.opensearch.sql.datasource.model.DataSourceMetadata; import org.opensearch.sql.datasource.model.DataSourceType; -/** - * Utitlity class to serialize and deserialize objects in XContent. - */ +/** Utitlity class to serialize and deserialize objects in XContent. */ @UtilityClass public class XContentParserUtils { public static final String NAME_FIELD = "name"; @@ -87,9 +85,13 @@ public static DataSourceMetadata toDataSourceMetadata(XContentParser parser) thr * @throws IOException IOException. */ public static DataSourceMetadata toDataSourceMetadata(String json) throws IOException { - try (XContentParser parser = XContentType.JSON.xContent() - .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - json)) { + try (XContentParser parser = + XContentType.JSON + .xContent() + .createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + json)) { return toDataSourceMetadata(parser); } } @@ -116,6 +118,4 @@ public static XContentBuilder convertToXContent(DataSourceMetadata metadata) thr builder.endObject(); return builder; } - - } diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/auth/AuthenticationTypeTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/auth/AuthenticationTypeTest.java index 23bb4688e12..4bc4800093b 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/auth/AuthenticationTypeTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/auth/AuthenticationTypeTest.java @@ -5,7 +5,6 @@ package org.opensearch.sql.datasources.auth; - import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImplTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImplTest.java index 552bd0edf97..6ee3c12edd9 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImplTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/auth/DataSourceUserAuthorizationHelperImplTest.java @@ -27,65 +27,76 @@ public class DataSourceUserAuthorizationHelperImplTest { @Mock(answer = Answers.RETURNS_DEEP_STUBS) private Client client; - @InjectMocks - private DataSourceUserAuthorizationHelperImpl dataSourceUserAuthorizationHelper; - + @InjectMocks private DataSourceUserAuthorizationHelperImpl dataSourceUserAuthorizationHelper; @Test public void testAuthorizeDataSourceWithAllowedRoles() { String userString = "myuser|bckrole1,bckrol2|prometheus_access|myTenant"; - Mockito.when(client.threadPool().getThreadContext() - .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) + Mockito.when( + client + .threadPool() + .getThreadContext() + .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) .thenReturn(userString); DataSourceMetadata dataSourceMetadata = dataSourceMetadata(); - this.dataSourceUserAuthorizationHelper - .authorizeDataSource(dataSourceMetadata); + this.dataSourceUserAuthorizationHelper.authorizeDataSource(dataSourceMetadata); } @Test public void testAuthorizeDataSourceWithAdminRole() { String userString = "myuser|bckrole1,bckrol2|all_access|myTenant"; - Mockito.when(client.threadPool().getThreadContext() - .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) + Mockito.when( + client + .threadPool() + .getThreadContext() + .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) .thenReturn(userString); DataSourceMetadata dataSourceMetadata = dataSourceMetadata(); - this.dataSourceUserAuthorizationHelper - .authorizeDataSource(dataSourceMetadata); + this.dataSourceUserAuthorizationHelper.authorizeDataSource(dataSourceMetadata); } @Test public void testAuthorizeDataSourceWithNullUserString() { - Mockito.when(client.threadPool().getThreadContext() - .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) + Mockito.when( + client + .threadPool() + .getThreadContext() + .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) .thenReturn(null); DataSourceMetadata dataSourceMetadata = dataSourceMetadata(); - this.dataSourceUserAuthorizationHelper - .authorizeDataSource(dataSourceMetadata); + this.dataSourceUserAuthorizationHelper.authorizeDataSource(dataSourceMetadata); } @Test public void testAuthorizeDataSourceWithDefaultDataSource() { String userString = "myuser|bckrole1,bckrol2|role1|myTenant"; - Mockito.when(client.threadPool().getThreadContext() - .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) + Mockito.when( + client + .threadPool() + .getThreadContext() + .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) .thenReturn(userString); DataSourceMetadata dataSourceMetadata = DataSourceMetadata.defaultOpenSearchDataSourceMetadata(); - this.dataSourceUserAuthorizationHelper - .authorizeDataSource(dataSourceMetadata); + this.dataSourceUserAuthorizationHelper.authorizeDataSource(dataSourceMetadata); } @Test public void testAuthorizeDataSourceWithException() { String userString = "myuser|bckrole1,bckrol2|role1|myTenant"; - Mockito.when(client.threadPool().getThreadContext() - .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) + Mockito.when( + client + .threadPool() + .getThreadContext() + .getTransient(OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT)) .thenReturn(userString); DataSourceMetadata dataSourceMetadata = dataSourceMetadata(); - SecurityException securityException - = Assert.assertThrows(SecurityException.class, + SecurityException securityException = + Assert.assertThrows( + SecurityException.class, () -> this.dataSourceUserAuthorizationHelper.authorizeDataSource(dataSourceMetadata)); - Assert.assertEquals("User is not authorized to access datasource test. " + Assert.assertEquals( + "User is not authorized to access datasource test. " + "User should be mapped to any of the roles in [prometheus_access] for access.", securityException.getMessage()); } @@ -98,5 +109,4 @@ private DataSourceMetadata dataSourceMetadata() { dataSourceMetadata.setProperties(new HashMap<>()); return dataSourceMetadata; } - } diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/encryptor/EncryptorImplTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/encryptor/EncryptorImplTest.java index d62a5a957ac..26432b139b2 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/encryptor/EncryptorImplTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/encryptor/EncryptorImplTest.java @@ -17,7 +17,6 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; - @ExtendWith(MockitoExtension.class) public class EncryptorImplTest { @@ -38,9 +37,11 @@ public void testMasterKeySize() { String input = "This is a test input"; String masterKey8 = "12345678"; Encryptor encryptor8 = new EncryptorImpl(masterKey8); - assertThrows(AwsCryptoException.class, () -> { - encryptor8.encrypt(input); - }); + assertThrows( + AwsCryptoException.class, + () -> { + encryptor8.encrypt(input); + }); String masterKey16 = "1234567812345678"; Encryptor encryptor16 = new EncryptorImpl(masterKey16); @@ -54,9 +55,11 @@ public void testMasterKeySize() { String masterKey17 = "12345678123456781"; Encryptor encryptor17 = new EncryptorImpl(masterKey17); - assertThrows(AwsCryptoException.class, () -> { - encryptor17.encrypt(input); - }); + assertThrows( + AwsCryptoException.class, + () -> { + encryptor17.encrypt(input); + }); } @Test @@ -64,9 +67,11 @@ public void testInvalidBase64String() { String encrypted = "invalidBase64String"; Encryptor encryptor = new EncryptorImpl("randomMasterKey"); - assertThrows(BadCiphertextException.class, () -> { - encryptor.decrypt(encrypted); - }); + assertThrows( + BadCiphertextException.class, + () -> { + encryptor.decrypt(encrypted); + }); } @Test @@ -80,19 +85,21 @@ public void testDecryptWithDifferentKey() { String encrypted = encryptor1.encrypt(input); - assertThrows(Exception.class, () -> { - encryptor2.decrypt(encrypted); - }); + assertThrows( + Exception.class, + () -> { + encryptor2.decrypt(encrypted); + }); } @Test public void testEncryptionAndDecryptionWithNullMasterKey() { String input = "This is a test input"; Encryptor encryptor = new EncryptorImpl(null); - IllegalStateException illegalStateException - = Assertions.assertThrows(IllegalStateException.class, - () -> encryptor.encrypt(input)); - Assertions.assertEquals("Master key is a required config for using create and" + IllegalStateException illegalStateException = + Assertions.assertThrows(IllegalStateException.class, () -> encryptor.encrypt(input)); + Assertions.assertEquals( + "Master key is a required config for using create and" + " update datasource APIs." + "Please set plugins.query.datasources.encryption.masterkey config " + "in opensearch.yml in all the cluster nodes. " @@ -100,10 +107,10 @@ public void testEncryptionAndDecryptionWithNullMasterKey() { + "https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/" + "admin/datasources.rst#master-key-config-for-encrypting-credential-information", illegalStateException.getMessage()); - illegalStateException - = Assertions.assertThrows(IllegalStateException.class, - () -> encryptor.decrypt(input)); - Assertions.assertEquals("Master key is a required config for using create and" + illegalStateException = + Assertions.assertThrows(IllegalStateException.class, () -> encryptor.decrypt(input)); + Assertions.assertEquals( + "Master key is a required config for using create and" + " update datasource APIs." + "Please set plugins.query.datasources.encryption.masterkey config " + "in opensearch.yml in all the cluster nodes. " @@ -118,10 +125,10 @@ public void testEncryptionAndDecryptionWithEmptyMasterKey() { String masterKey = ""; String input = "This is a test input"; Encryptor encryptor = new EncryptorImpl(masterKey); - IllegalStateException illegalStateException - = Assertions.assertThrows(IllegalStateException.class, - () -> encryptor.encrypt(input)); - Assertions.assertEquals("Master key is a required config for using create and" + IllegalStateException illegalStateException = + Assertions.assertThrows(IllegalStateException.class, () -> encryptor.encrypt(input)); + Assertions.assertEquals( + "Master key is a required config for using create and" + " update datasource APIs." + "Please set plugins.query.datasources.encryption.masterkey config " + "in opensearch.yml in all the cluster nodes. " @@ -129,10 +136,10 @@ public void testEncryptionAndDecryptionWithEmptyMasterKey() { + "https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/" + "admin/datasources.rst#master-key-config-for-encrypting-credential-information", illegalStateException.getMessage()); - illegalStateException - = Assertions.assertThrows(IllegalStateException.class, - () -> encryptor.decrypt(input)); - Assertions.assertEquals("Master key is a required config for using create and" + illegalStateException = + Assertions.assertThrows(IllegalStateException.class, () -> encryptor.decrypt(input)); + Assertions.assertEquals( + "Master key is a required config for using create and" + " update datasource APIs." + "Please set plugins.query.datasources.encryption.masterkey config " + "in opensearch.yml in all the cluster nodes. " @@ -141,5 +148,4 @@ public void testEncryptionAndDecryptionWithEmptyMasterKey() { + "admin/datasources.rst#master-key-config-for-encrypting-credential-information", illegalStateException.getMessage()); } - } diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImplTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImplTest.java index bf656857b02..b2ea221eb70 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImplTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceLoaderCacheImplTest.java @@ -25,11 +25,9 @@ @ExtendWith(MockitoExtension.class) class DataSourceLoaderCacheImplTest { - @Mock - private DataSourceFactory dataSourceFactory; + @Mock private DataSourceFactory dataSourceFactory; - @Mock - private StorageEngine storageEngine; + @Mock private StorageEngine storageEngine; @BeforeEach public void setup() { @@ -55,8 +53,8 @@ void testGetOrLoadDataSource() { dataSourceMetadata.setProperties(ImmutableMap.of()); DataSource dataSource = dataSourceLoaderCache.getOrLoadDataSource(dataSourceMetadata); verify(dataSourceFactory, times(1)).createDataSource(dataSourceMetadata); - Assertions.assertEquals(dataSource, - dataSourceLoaderCache.getOrLoadDataSource(dataSourceMetadata)); + Assertions.assertEquals( + dataSource, dataSourceLoaderCache.getOrLoadDataSource(dataSourceMetadata)); verifyNoMoreInteractions(dataSourceFactory); } @@ -81,5 +79,4 @@ private DataSourceMetadata getMetadata() { dataSourceMetadata.setProperties(ImmutableMap.of()); return dataSourceMetadata; } - } diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceServiceImplTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceServiceImplTest.java index e1312ec582c..56d3586c6ea 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceServiceImplTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/service/DataSourceServiceImplTest.java @@ -46,15 +46,11 @@ @ExtendWith(MockitoExtension.class) class DataSourceServiceImplTest { - @Mock - private DataSourceFactory dataSourceFactory; - @Mock - private StorageEngine storageEngine; - @Mock - private DataSourceMetadataStorage dataSourceMetadataStorage; + @Mock private DataSourceFactory dataSourceFactory; + @Mock private StorageEngine storageEngine; + @Mock private DataSourceMetadataStorage dataSourceMetadataStorage; - @Mock - private DataSourceUserAuthorizationHelper dataSourceUserAuthorizationHelper; + @Mock private DataSourceUserAuthorizationHelper dataSourceUserAuthorizationHelper; private DataSourceService dataSourceService; @@ -75,7 +71,8 @@ public void setup() { { add(dataSourceFactory); } - }, dataSourceMetadataStorage, + }, + dataSourceMetadataStorage, dataSourceUserAuthorizationHelper); } @@ -91,22 +88,18 @@ void testGetDataSourceForDefaultOpenSearchDataSource() { @Test void testGetDataSourceForNonExistingDataSource() { - when(dataSourceMetadataStorage.getDataSourceMetadata("test")) - .thenReturn(Optional.empty()); + when(dataSourceMetadataStorage.getDataSourceMetadata("test")).thenReturn(Optional.empty()); DataSourceNotFoundException exception = assertThrows( - DataSourceNotFoundException.class, - () -> - dataSourceService.getDataSource("test")); + DataSourceNotFoundException.class, () -> dataSourceService.getDataSource("test")); assertEquals("DataSource with name test doesn't exist.", exception.getMessage()); - verify(dataSourceMetadataStorage, times(1)) - .getDataSourceMetadata("test"); + verify(dataSourceMetadataStorage, times(1)).getDataSourceMetadata("test"); } @Test void testGetDataSourceSuccessCase() { - DataSourceMetadata dataSourceMetadata = metadata("test", DataSourceType.OPENSEARCH, - Collections.emptyList(), ImmutableMap.of()); + DataSourceMetadata dataSourceMetadata = + metadata("test", DataSourceType.OPENSEARCH, Collections.emptyList(), ImmutableMap.of()); doNothing().when(dataSourceUserAuthorizationHelper).authorizeDataSource(dataSourceMetadata); when(dataSourceMetadataStorage.getDataSourceMetadata("test")) .thenReturn(Optional.of(dataSourceMetadata)); @@ -114,26 +107,31 @@ void testGetDataSourceSuccessCase() { assertEquals("test", dataSource.getName()); assertEquals(DataSourceType.OPENSEARCH, dataSource.getConnectorType()); verify(dataSourceMetadataStorage, times(1)).getDataSourceMetadata("test"); - verify(dataSourceFactory, times(1)) - .createDataSource(dataSourceMetadata); + verify(dataSourceFactory, times(1)).createDataSource(dataSourceMetadata); } @Test void testGetDataSourceWithAuthorizationFailure() { - DataSourceMetadata dataSourceMetadata = metadata("test", DataSourceType.OPENSEARCH, - Collections.singletonList("prometheus_access"), ImmutableMap.of()); - doThrow(new SecurityException("User is not authorized to access datasource test. " - + "User should be mapped to any of the roles in [prometheus_access] for access.")) + DataSourceMetadata dataSourceMetadata = + metadata( + "test", + DataSourceType.OPENSEARCH, + Collections.singletonList("prometheus_access"), + ImmutableMap.of()); + doThrow( + new SecurityException( + "User is not authorized to access datasource test. User should be mapped to any of" + + " the roles in [prometheus_access] for access.")) .when(dataSourceUserAuthorizationHelper) .authorizeDataSource(dataSourceMetadata); when(dataSourceMetadataStorage.getDataSourceMetadata("test")) .thenReturn(Optional.of(dataSourceMetadata)); - - SecurityException securityException - = Assertions.assertThrows(SecurityException.class, - () -> dataSourceService.getDataSource("test")); - Assertions.assertEquals("User is not authorized to access datasource test. " + SecurityException securityException = + Assertions.assertThrows( + SecurityException.class, () -> dataSourceService.getDataSource("test")); + Assertions.assertEquals( + "User is not authorized to access datasource test. " + "User should be mapped to any of the roles in [prometheus_access] for access.", securityException.getMessage()); @@ -141,21 +139,23 @@ void testGetDataSourceWithAuthorizationFailure() { verify(dataSourceFactory, times(0)).createDataSource(dataSourceMetadata); } - @Test void testCreateDataSourceSuccessCase() { - DataSourceMetadata dataSourceMetadata = metadata("testDS", DataSourceType.OPENSEARCH, - Collections.emptyList(), ImmutableMap.of()); + DataSourceMetadata dataSourceMetadata = + metadata("testDS", DataSourceType.OPENSEARCH, Collections.emptyList(), ImmutableMap.of()); dataSourceService.createDataSource(dataSourceMetadata); - verify(dataSourceMetadataStorage, times(1)) - .createDataSourceMetadata(dataSourceMetadata); - verify(dataSourceFactory, times(1)) - .createDataSource(dataSourceMetadata); + verify(dataSourceMetadataStorage, times(1)).createDataSourceMetadata(dataSourceMetadata); + verify(dataSourceFactory, times(1)).createDataSource(dataSourceMetadata); when(dataSourceMetadataStorage.getDataSourceMetadata("testDS")) - .thenReturn(Optional.ofNullable(metadata("testDS", DataSourceType.OPENSEARCH, - Collections.emptyList(), ImmutableMap.of()))); + .thenReturn( + Optional.ofNullable( + metadata( + "testDS", + DataSourceType.OPENSEARCH, + Collections.emptyList(), + ImmutableMap.of()))); DataSource dataSource = dataSourceService.getDataSource("testDS"); assertEquals("testDS", dataSource.getName()); assertEquals(storageEngine, dataSource.getStorageEngine()); @@ -164,14 +164,15 @@ void testCreateDataSourceSuccessCase() { @Test void testCreateDataSourceWithDisallowedDatasourceName() { - DataSourceMetadata dataSourceMetadata = metadata("testDS$$$", DataSourceType.OPENSEARCH, - Collections.emptyList(), ImmutableMap.of()); + DataSourceMetadata dataSourceMetadata = + metadata( + "testDS$$$", DataSourceType.OPENSEARCH, Collections.emptyList(), ImmutableMap.of()); IllegalArgumentException exception = assertThrows( IllegalArgumentException.class, - () -> - dataSourceService.createDataSource(dataSourceMetadata)); - assertEquals("DataSource Name: testDS$$$ contains illegal characters." + () -> dataSourceService.createDataSource(dataSourceMetadata)); + assertEquals( + "DataSource Name: testDS$$$ contains illegal characters." + " Allowed characters: a-zA-Z0-9_-*@.", exception.getMessage()); verify(dataSourceFactory, times(1)).getDataSourceType(); @@ -181,14 +182,14 @@ void testCreateDataSourceWithDisallowedDatasourceName() { @Test void testCreateDataSourceWithEmptyDatasourceName() { - DataSourceMetadata dataSourceMetadata = metadata("", DataSourceType.OPENSEARCH, - Collections.emptyList(), ImmutableMap.of()); + DataSourceMetadata dataSourceMetadata = + metadata("", DataSourceType.OPENSEARCH, Collections.emptyList(), ImmutableMap.of()); IllegalArgumentException exception = assertThrows( IllegalArgumentException.class, - () -> - dataSourceService.createDataSource(dataSourceMetadata)); - assertEquals("Missing Name Field from a DataSource. Name is a required parameter.", + () -> dataSourceService.createDataSource(dataSourceMetadata)); + assertEquals( + "Missing Name Field from a DataSource. Name is a required parameter.", exception.getMessage()); verify(dataSourceFactory, times(1)).getDataSourceType(); verify(dataSourceFactory, times(0)).createDataSource(dataSourceMetadata); @@ -197,14 +198,14 @@ void testCreateDataSourceWithEmptyDatasourceName() { @Test void testCreateDataSourceWithNullParameters() { - DataSourceMetadata dataSourceMetadata = metadata("testDS", DataSourceType.OPENSEARCH, - Collections.emptyList(), null); + DataSourceMetadata dataSourceMetadata = + metadata("testDS", DataSourceType.OPENSEARCH, Collections.emptyList(), null); IllegalArgumentException exception = assertThrows( IllegalArgumentException.class, - () -> - dataSourceService.createDataSource(dataSourceMetadata)); - assertEquals("Missing properties field in datasource configuration. " + () -> dataSourceService.createDataSource(dataSourceMetadata)); + assertEquals( + "Missing properties field in datasource configuration. " + "Properties are required parameters.", exception.getMessage()); verify(dataSourceFactory, times(1)).getDataSourceType(); @@ -219,88 +220,99 @@ void testGetDataSourceMetadataSet() { properties.put("prometheus.auth.type", "basicauth"); properties.put("prometheus.auth.username", "username"); properties.put("prometheus.auth.password", "password"); - when(dataSourceMetadataStorage.getDataSourceMetadata()).thenReturn(new ArrayList<>() { - { - add(metadata("testDS", DataSourceType.PROMETHEUS, Collections.emptyList(), - properties)); - } - }); - Set dataSourceMetadataSet - = dataSourceService.getDataSourceMetadata(false); + when(dataSourceMetadataStorage.getDataSourceMetadata()) + .thenReturn( + new ArrayList<>() { + { + add( + metadata( + "testDS", DataSourceType.PROMETHEUS, Collections.emptyList(), properties)); + } + }); + Set dataSourceMetadataSet = dataSourceService.getDataSourceMetadata(false); assertEquals(1, dataSourceMetadataSet.size()); DataSourceMetadata dataSourceMetadata = dataSourceMetadataSet.iterator().next(); assertTrue(dataSourceMetadata.getProperties().containsKey("prometheus.uri")); assertFalse(dataSourceMetadata.getProperties().containsKey("prometheus.auth.type")); assertFalse(dataSourceMetadata.getProperties().containsKey("prometheus.auth.username")); assertFalse(dataSourceMetadata.getProperties().containsKey("prometheus.auth.password")); - assertFalse(dataSourceMetadataSet - .contains(DataSourceMetadata.defaultOpenSearchDataSourceMetadata())); + assertFalse( + dataSourceMetadataSet.contains(DataSourceMetadata.defaultOpenSearchDataSourceMetadata())); verify(dataSourceMetadataStorage, times(1)).getDataSourceMetadata(); } @Test void testGetDataSourceMetadataSetWithDefaultDatasource() { - when(dataSourceMetadataStorage.getDataSourceMetadata()).thenReturn(new ArrayList<>() { - { - add(metadata("testDS", DataSourceType.PROMETHEUS, Collections.emptyList(), - ImmutableMap.of())); - } - }); - Set dataSourceMetadataSet - = dataSourceService.getDataSourceMetadata(true); + when(dataSourceMetadataStorage.getDataSourceMetadata()) + .thenReturn( + new ArrayList<>() { + { + add( + metadata( + "testDS", + DataSourceType.PROMETHEUS, + Collections.emptyList(), + ImmutableMap.of())); + } + }); + Set dataSourceMetadataSet = dataSourceService.getDataSourceMetadata(true); assertEquals(2, dataSourceMetadataSet.size()); - assertTrue(dataSourceMetadataSet - .contains(DataSourceMetadata.defaultOpenSearchDataSourceMetadata())); + assertTrue( + dataSourceMetadataSet.contains(DataSourceMetadata.defaultOpenSearchDataSourceMetadata())); verify(dataSourceMetadataStorage, times(1)).getDataSourceMetadata(); } @Test void testUpdateDataSourceSuccessCase() { - DataSourceMetadata dataSourceMetadata = metadata("testDS", DataSourceType.OPENSEARCH, - Collections.emptyList(), ImmutableMap.of()); + DataSourceMetadata dataSourceMetadata = + metadata("testDS", DataSourceType.OPENSEARCH, Collections.emptyList(), ImmutableMap.of()); dataSourceService.updateDataSource(dataSourceMetadata); - verify(dataSourceMetadataStorage, times(1)) - .updateDataSourceMetadata(dataSourceMetadata); - verify(dataSourceFactory, times(1)) - .createDataSource(dataSourceMetadata); + verify(dataSourceMetadataStorage, times(1)).updateDataSourceMetadata(dataSourceMetadata); + verify(dataSourceFactory, times(1)).createDataSource(dataSourceMetadata); } @Test void testUpdateDefaultDataSource() { - DataSourceMetadata dataSourceMetadata = metadata(DEFAULT_DATASOURCE_NAME, - DataSourceType.OPENSEARCH, Collections.emptyList(), ImmutableMap.of()); - UnsupportedOperationException unsupportedOperationException - = assertThrows(UnsupportedOperationException.class, + DataSourceMetadata dataSourceMetadata = + metadata( + DEFAULT_DATASOURCE_NAME, + DataSourceType.OPENSEARCH, + Collections.emptyList(), + ImmutableMap.of()); + UnsupportedOperationException unsupportedOperationException = + assertThrows( + UnsupportedOperationException.class, () -> dataSourceService.updateDataSource(dataSourceMetadata)); - assertEquals("Not allowed to update default datasource :" + DEFAULT_DATASOURCE_NAME, + assertEquals( + "Not allowed to update default datasource :" + DEFAULT_DATASOURCE_NAME, unsupportedOperationException.getMessage()); } @Test void testDeleteDatasource() { dataSourceService.deleteDataSource("testDS"); - verify(dataSourceMetadataStorage, times(1)) - .deleteDataSourceMetadata("testDS"); + verify(dataSourceMetadataStorage, times(1)).deleteDataSourceMetadata("testDS"); } @Test void testDeleteDefaultDatasource() { - UnsupportedOperationException unsupportedOperationException - = assertThrows(UnsupportedOperationException.class, - () -> dataSourceService.deleteDataSource(DEFAULT_DATASOURCE_NAME)); - assertEquals("Not allowed to delete default datasource :" + DEFAULT_DATASOURCE_NAME, + UnsupportedOperationException unsupportedOperationException = + assertThrows( + UnsupportedOperationException.class, + () -> dataSourceService.deleteDataSource(DEFAULT_DATASOURCE_NAME)); + assertEquals( + "Not allowed to delete default datasource :" + DEFAULT_DATASOURCE_NAME, unsupportedOperationException.getMessage()); } @Test void testDataSourceExists() { - when(dataSourceMetadataStorage.getDataSourceMetadata("test")) - .thenReturn(Optional.empty()); + when(dataSourceMetadataStorage.getDataSourceMetadata("test")).thenReturn(Optional.empty()); Assertions.assertFalse(dataSourceService.dataSourceExists("test")); when(dataSourceMetadataStorage.getDataSourceMetadata("test")) - .thenReturn(Optional.of(metadata("test", DataSourceType.PROMETHEUS, - List.of(), ImmutableMap.of()))); + .thenReturn( + Optional.of(metadata("test", DataSourceType.PROMETHEUS, List.of(), ImmutableMap.of()))); Assertions.assertTrue(dataSourceService.dataSourceExists("test")); } @@ -310,9 +322,8 @@ void testDataSourceExistsForDefaultDataSource() { verifyNoInteractions(dataSourceMetadataStorage); } - DataSourceMetadata metadata(String name, DataSourceType type, - List allowedRoles, - Map properties) { + DataSourceMetadata metadata( + String name, DataSourceType type, List allowedRoles, Map properties) { DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); dataSourceMetadata.setName(name); dataSourceMetadata.setConnector(type); @@ -329,13 +340,15 @@ void testRemovalOfAuthorizationInfo() { properties.put("prometheus.auth.username", "username"); properties.put("prometheus.auth.password", "password"); DataSourceMetadata dataSourceMetadata = - new DataSourceMetadata("testDS", DataSourceType.PROMETHEUS, - Collections.singletonList("prometheus_access"), properties); + new DataSourceMetadata( + "testDS", + DataSourceType.PROMETHEUS, + Collections.singletonList("prometheus_access"), + properties); when(dataSourceMetadataStorage.getDataSourceMetadata("testDS")) .thenReturn(Optional.of(dataSourceMetadata)); - DataSourceMetadata dataSourceMetadata1 - = dataSourceService.getDataSourceMetadata("testDS"); + DataSourceMetadata dataSourceMetadata1 = dataSourceService.getDataSourceMetadata("testDS"); assertEquals("testDS", dataSourceMetadata1.getName()); assertEquals(DataSourceType.PROMETHEUS, dataSourceMetadata1.getConnector()); assertFalse(dataSourceMetadata1.getProperties().containsKey("prometheus.auth.type")); @@ -345,10 +358,11 @@ void testRemovalOfAuthorizationInfo() { @Test void testGetDataSourceMetadataForNonExistingDataSource() { - when(dataSourceMetadataStorage.getDataSourceMetadata("testDS")) - .thenReturn(Optional.empty()); - IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, - () -> dataSourceService.getDataSourceMetadata("testDS")); + when(dataSourceMetadataStorage.getDataSourceMetadata("testDS")).thenReturn(Optional.empty()); + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> dataSourceService.getDataSourceMetadata("testDS")); assertEquals("DataSource with name: testDS doesn't exist.", exception.getMessage()); } @@ -360,16 +374,15 @@ void testGetDataSourceMetadataForSpecificDataSourceName() { properties.put("prometheus.auth.username", "username"); properties.put("prometheus.auth.password", "password"); when(dataSourceMetadataStorage.getDataSourceMetadata("testDS")) - .thenReturn(Optional.ofNullable( - metadata("testDS", DataSourceType.PROMETHEUS, Collections.emptyList(), - properties))); - DataSourceMetadata dataSourceMetadata - = this.dataSourceService.getDataSourceMetadata("testDS"); + .thenReturn( + Optional.ofNullable( + metadata( + "testDS", DataSourceType.PROMETHEUS, Collections.emptyList(), properties))); + DataSourceMetadata dataSourceMetadata = this.dataSourceService.getDataSourceMetadata("testDS"); assertTrue(dataSourceMetadata.getProperties().containsKey("prometheus.uri")); assertFalse(dataSourceMetadata.getProperties().containsKey("prometheus.auth.type")); assertFalse(dataSourceMetadata.getProperties().containsKey("prometheus.auth.username")); assertFalse(dataSourceMetadata.getProperties().containsKey("prometheus.auth.password")); verify(dataSourceMetadataStorage, times(1)).getDataSourceMetadata("testDS"); } - } diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorageTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorageTest.java index b58ef3ea1e9..7d41737b2d3 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorageTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/storage/OpenSearchDataSourceMetadataStorageTest.java @@ -52,33 +52,25 @@ public class OpenSearchDataSourceMetadataStorageTest { @Mock(answer = Answers.RETURNS_DEEP_STUBS) private Client client; + @Mock(answer = Answers.RETURNS_DEEP_STUBS) private ClusterService clusterService; - @Mock - private Encryptor encryptor; + + @Mock private Encryptor encryptor; + @Mock(answer = Answers.RETURNS_DEEP_STUBS) private SearchResponse searchResponse; - @Mock - private ActionFuture searchResponseActionFuture; - @Mock - private ActionFuture createIndexResponseActionFuture; - @Mock - private ActionFuture indexResponseActionFuture; - @Mock - private IndexResponse indexResponse; - @Mock - private ActionFuture updateResponseActionFuture; - @Mock - private UpdateResponse updateResponse; - @Mock - private ActionFuture deleteResponseActionFuture; - @Mock - private DeleteResponse deleteResponse; - @Mock - private SearchHit searchHit; - @InjectMocks - private OpenSearchDataSourceMetadataStorage openSearchDataSourceMetadataStorage; + @Mock private ActionFuture searchResponseActionFuture; + @Mock private ActionFuture createIndexResponseActionFuture; + @Mock private ActionFuture indexResponseActionFuture; + @Mock private IndexResponse indexResponse; + @Mock private ActionFuture updateResponseActionFuture; + @Mock private UpdateResponse updateResponse; + @Mock private ActionFuture deleteResponseActionFuture; + @Mock private DeleteResponse deleteResponse; + @Mock private SearchHit searchHit; + @InjectMocks private OpenSearchDataSourceMetadataStorage openSearchDataSourceMetadataStorage; @SneakyThrows @Test @@ -91,28 +83,24 @@ public void testGetDataSourceMetadata() { Mockito.when(searchResponse.getHits()) .thenReturn( new SearchHits( - new SearchHit[] {searchHit}, - new TotalHits(21, TotalHits.Relation.EQUAL_TO), - 1.0F)); - Mockito.when(searchHit.getSourceAsString()) - .thenReturn(getBasicDataSourceMetadataString()); + new SearchHit[] {searchHit}, new TotalHits(21, TotalHits.Relation.EQUAL_TO), 1.0F)); + Mockito.when(searchHit.getSourceAsString()).thenReturn(getBasicDataSourceMetadataString()); Mockito.when(encryptor.decrypt("password")).thenReturn("password"); Mockito.when(encryptor.decrypt("username")).thenReturn("username"); - Optional dataSourceMetadataOptional - = openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); - + Optional dataSourceMetadataOptional = + openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); Assertions.assertFalse(dataSourceMetadataOptional.isEmpty()); DataSourceMetadata dataSourceMetadata = dataSourceMetadataOptional.get(); Assertions.assertEquals(TEST_DATASOURCE_INDEX_NAME, dataSourceMetadata.getName()); Assertions.assertEquals(DataSourceType.PROMETHEUS, dataSourceMetadata.getConnector()); - Assertions.assertEquals("password", - dataSourceMetadata.getProperties().get("prometheus.auth.password")); - Assertions.assertEquals("username", - dataSourceMetadata.getProperties().get("prometheus.auth.username")); - Assertions.assertEquals("basicauth", - dataSourceMetadata.getProperties().get("prometheus.auth.type")); + Assertions.assertEquals( + "password", dataSourceMetadata.getProperties().get("prometheus.auth.password")); + Assertions.assertEquals( + "username", dataSourceMetadata.getProperties().get("prometheus.auth.username")); + Assertions.assertEquals( + "basicauth", dataSourceMetadata.getProperties().get("prometheus.auth.type")); } @SneakyThrows @@ -124,9 +112,12 @@ public void testGetDataSourceMetadataWith404SearchResponse() { Mockito.when(searchResponseActionFuture.actionGet()).thenReturn(searchResponse); Mockito.when(searchResponse.status()).thenReturn(RestStatus.NOT_FOUND); - RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, - () -> openSearchDataSourceMetadataStorage.getDataSourceMetadata( - TEST_DATASOURCE_INDEX_NAME)); + RuntimeException runtimeException = + Assertions.assertThrows( + RuntimeException.class, + () -> + openSearchDataSourceMetadataStorage.getDataSourceMetadata( + TEST_DATASOURCE_INDEX_NAME)); Assertions.assertEquals( "Fetching dataSource metadata information failed with status : NOT_FOUND", runtimeException.getMessage()); @@ -143,15 +134,13 @@ public void testGetDataSourceMetadataWithParsingFailed() { Mockito.when(searchResponse.getHits()) .thenReturn( new SearchHits( - new SearchHit[] {searchHit}, - new TotalHits(21, TotalHits.Relation.EQUAL_TO), - 1.0F)); - Mockito.when(searchHit.getSourceAsString()) - .thenReturn("..testDs"); + new SearchHit[] {searchHit}, new TotalHits(21, TotalHits.Relation.EQUAL_TO), 1.0F)); + Mockito.when(searchHit.getSourceAsString()).thenReturn("..testDs"); - Assertions.assertThrows(RuntimeException.class, - () -> openSearchDataSourceMetadataStorage.getDataSourceMetadata( - TEST_DATASOURCE_INDEX_NAME)); + Assertions.assertThrows( + RuntimeException.class, + () -> + openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME)); } @SneakyThrows @@ -165,28 +154,24 @@ public void testGetDataSourceMetadataWithAWSSigV4() { Mockito.when(searchResponse.getHits()) .thenReturn( new SearchHits( - new SearchHit[] {searchHit}, - new TotalHits(21, TotalHits.Relation.EQUAL_TO), - 1.0F)); - Mockito.when(searchHit.getSourceAsString()) - .thenReturn(getAWSSigv4DataSourceMetadataString()); + new SearchHit[] {searchHit}, new TotalHits(21, TotalHits.Relation.EQUAL_TO), 1.0F)); + Mockito.when(searchHit.getSourceAsString()).thenReturn(getAWSSigv4DataSourceMetadataString()); Mockito.when(encryptor.decrypt("secret_key")).thenReturn("secret_key"); Mockito.when(encryptor.decrypt("access_key")).thenReturn("access_key"); - Optional dataSourceMetadataOptional - = openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); - + Optional dataSourceMetadataOptional = + openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); Assertions.assertFalse(dataSourceMetadataOptional.isEmpty()); DataSourceMetadata dataSourceMetadata = dataSourceMetadataOptional.get(); Assertions.assertEquals(TEST_DATASOURCE_INDEX_NAME, dataSourceMetadata.getName()); Assertions.assertEquals(DataSourceType.PROMETHEUS, dataSourceMetadata.getConnector()); - Assertions.assertEquals("secret_key", - dataSourceMetadata.getProperties().get("prometheus.auth.secret_key")); - Assertions.assertEquals("access_key", - dataSourceMetadata.getProperties().get("prometheus.auth.access_key")); - Assertions.assertEquals("awssigv4", - dataSourceMetadata.getProperties().get("prometheus.auth.type")); + Assertions.assertEquals( + "secret_key", dataSourceMetadata.getProperties().get("prometheus.auth.secret_key")); + Assertions.assertEquals( + "access_key", dataSourceMetadata.getProperties().get("prometheus.auth.access_key")); + Assertions.assertEquals( + "awssigv4", dataSourceMetadata.getProperties().get("prometheus.auth.type")); } @SneakyThrows @@ -200,31 +185,27 @@ public void testGetDataSourceMetadataWithBasicAuth() { Mockito.when(searchResponse.getHits()) .thenReturn( new SearchHits( - new SearchHit[] {searchHit}, - new TotalHits(21, TotalHits.Relation.EQUAL_TO), - 1.0F)); + new SearchHit[] {searchHit}, new TotalHits(21, TotalHits.Relation.EQUAL_TO), 1.0F)); Mockito.when(searchHit.getSourceAsString()) .thenReturn(getDataSourceMetadataStringWithBasicAuthentication()); Mockito.when(encryptor.decrypt("username")).thenReturn("username"); Mockito.when(encryptor.decrypt("password")).thenReturn("password"); - Optional dataSourceMetadataOptional - = openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); - + Optional dataSourceMetadataOptional = + openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); Assertions.assertFalse(dataSourceMetadataOptional.isEmpty()); DataSourceMetadata dataSourceMetadata = dataSourceMetadataOptional.get(); Assertions.assertEquals(TEST_DATASOURCE_INDEX_NAME, dataSourceMetadata.getName()); Assertions.assertEquals(DataSourceType.PROMETHEUS, dataSourceMetadata.getConnector()); - Assertions.assertEquals("username", - dataSourceMetadata.getProperties().get("prometheus.auth.username")); - Assertions.assertEquals("password", - dataSourceMetadata.getProperties().get("prometheus.auth.password")); - Assertions.assertEquals("basicauth", - dataSourceMetadata.getProperties().get("prometheus.auth.type")); + Assertions.assertEquals( + "username", dataSourceMetadata.getProperties().get("prometheus.auth.username")); + Assertions.assertEquals( + "password", dataSourceMetadata.getProperties().get("prometheus.auth.password")); + Assertions.assertEquals( + "basicauth", dataSourceMetadata.getProperties().get("prometheus.auth.type")); } - @SneakyThrows @Test public void testGetDataSourceMetadataList() { @@ -236,15 +217,12 @@ public void testGetDataSourceMetadataList() { Mockito.when(searchResponse.getHits()) .thenReturn( new SearchHits( - new SearchHit[] {searchHit}, - new TotalHits(21, TotalHits.Relation.EQUAL_TO), - 1.0F)); + new SearchHit[] {searchHit}, new TotalHits(21, TotalHits.Relation.EQUAL_TO), 1.0F)); Mockito.when(searchHit.getSourceAsString()) .thenReturn(getDataSourceMetadataStringWithNoAuthentication()); - List dataSourceMetadataList - = openSearchDataSourceMetadataStorage.getDataSourceMetadata(); - + List dataSourceMetadataList = + openSearchDataSourceMetadataStorage.getDataSourceMetadata(); Assertions.assertEquals(1, dataSourceMetadataList.size()); DataSourceMetadata dataSourceMetadata = dataSourceMetadataList.get(0); @@ -252,7 +230,6 @@ public void testGetDataSourceMetadataList() { Assertions.assertEquals(DataSourceType.PROMETHEUS, dataSourceMetadata.getConnector()); } - @SneakyThrows @Test public void testGetDataSourceMetadataListWithNoIndex() { @@ -264,8 +241,8 @@ public void testGetDataSourceMetadataListWithNoIndex() { .thenReturn(new CreateIndexResponse(true, true, DATASOURCE_INDEX_NAME)); Mockito.when(client.index(ArgumentMatchers.any())).thenReturn(indexResponseActionFuture); - List dataSourceMetadataList - = openSearchDataSourceMetadataStorage.getDataSourceMetadata(); + List dataSourceMetadataList = + openSearchDataSourceMetadataStorage.getDataSourceMetadata(); Assertions.assertEquals(0, dataSourceMetadataList.size()); } @@ -281,8 +258,8 @@ public void testGetDataSourceMetadataWithNoIndex() { .thenReturn(new CreateIndexResponse(true, true, DATASOURCE_INDEX_NAME)); Mockito.when(client.index(ArgumentMatchers.any())).thenReturn(indexResponseActionFuture); - Optional dataSourceMetadataOptional - = openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); + Optional dataSourceMetadataOptional = + openSearchDataSourceMetadataStorage.getDataSourceMetadata(TEST_DATASOURCE_INDEX_NAME); Assertions.assertFalse(dataSourceMetadataOptional.isPresent()); } @@ -310,8 +287,6 @@ public void testCreateDataSourceMetadata() { Mockito.verify(client.admin().indices(), Mockito.times(1)).create(ArgumentMatchers.any()); Mockito.verify(client, Mockito.times(1)).index(ArgumentMatchers.any()); Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(2)).stashContext(); - - } @Test @@ -334,7 +309,6 @@ public void testCreateDataSourceMetadataWithOutCreatingIndex() { Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); } - @Test public void testCreateDataSourceMetadataFailedWithNotFoundResponse() { @@ -351,10 +325,14 @@ public void testCreateDataSourceMetadataFailedWithNotFoundResponse() { Mockito.when(indexResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); - RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, - () -> this.openSearchDataSourceMetadataStorage.createDataSourceMetadata( - dataSourceMetadata)); - Assertions.assertEquals("Saving dataSource metadata information failed with result : not_found", + RuntimeException runtimeException = + Assertions.assertThrows( + RuntimeException.class, + () -> + this.openSearchDataSourceMetadataStorage.createDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals( + "Saving dataSource metadata information failed with result : not_found", runtimeException.getMessage()); Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); @@ -362,8 +340,6 @@ public void testCreateDataSourceMetadataFailedWithNotFoundResponse() { Mockito.verify(client.admin().indices(), Mockito.times(1)).create(ArgumentMatchers.any()); Mockito.verify(client, Mockito.times(1)).index(ArgumentMatchers.any()); Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(2)).stashContext(); - - } @Test @@ -381,20 +357,19 @@ public void testCreateDataSourceMetadataWithVersionConflict() { .thenThrow(VersionConflictEngineException.class); DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); IllegalArgumentException illegalArgumentException = - Assertions.assertThrows(IllegalArgumentException.class, - () -> this.openSearchDataSourceMetadataStorage.createDataSourceMetadata( - dataSourceMetadata)); - Assertions.assertEquals("A datasource already exists with name: testDS", - illegalArgumentException.getMessage()); - + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + this.openSearchDataSourceMetadataStorage.createDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals( + "A datasource already exists with name: testDS", illegalArgumentException.getMessage()); Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); Mockito.verify(client.admin().indices(), Mockito.times(1)).create(ArgumentMatchers.any()); Mockito.verify(client, Mockito.times(1)).index(ArgumentMatchers.any()); Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(2)).stashContext(); - - } @Test @@ -412,19 +387,20 @@ public void testCreateDataSourceMetadataWithException() { .thenThrow(new RuntimeException("error while indexing")); DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); - RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, - () -> this.openSearchDataSourceMetadataStorage.createDataSourceMetadata( - dataSourceMetadata)); - Assertions.assertEquals("java.lang.RuntimeException: error while indexing", - runtimeException.getMessage()); + RuntimeException runtimeException = + Assertions.assertThrows( + RuntimeException.class, + () -> + this.openSearchDataSourceMetadataStorage.createDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals( + "java.lang.RuntimeException: error while indexing", runtimeException.getMessage()); Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); Mockito.verify(client.admin().indices(), Mockito.times(1)).create(ArgumentMatchers.any()); Mockito.verify(client, Mockito.times(1)).index(ArgumentMatchers.any()); Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(2)).stashContext(); - - } @Test @@ -440,9 +416,12 @@ public void testCreateDataSourceMetadataWithIndexCreationFailed() { .thenReturn(new CreateIndexResponse(false, false, DATASOURCE_INDEX_NAME)); DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); - RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, - () -> this.openSearchDataSourceMetadataStorage.createDataSourceMetadata( - dataSourceMetadata)); + RuntimeException runtimeException = + Assertions.assertThrows( + RuntimeException.class, + () -> + this.openSearchDataSourceMetadataStorage.createDataSourceMetadata( + dataSourceMetadata)); Assertions.assertEquals( "Internal server error while creating.ql-datasources index:: " + "Index creation is not acknowledged.", @@ -470,7 +449,6 @@ public void testUpdateDataSourceMetadata() { Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); Mockito.verify(client, Mockito.times(1)).update(ArgumentMatchers.any()); Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); - } @Test @@ -500,10 +478,14 @@ public void testUpdateDataSourceMetadataWithNotFoundResult() { Mockito.when(updateResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); - RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, - () -> this.openSearchDataSourceMetadataStorage.updateDataSourceMetadata( - dataSourceMetadata)); - Assertions.assertEquals("Saving dataSource metadata information failed with result : not_found", + RuntimeException runtimeException = + Assertions.assertThrows( + RuntimeException.class, + () -> + this.openSearchDataSourceMetadataStorage.updateDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals( + "Saving dataSource metadata information failed with result : not_found", runtimeException.getMessage()); Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); @@ -511,32 +493,31 @@ public void testUpdateDataSourceMetadataWithNotFoundResult() { Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); Mockito.verify(client, Mockito.times(1)).update(ArgumentMatchers.any()); Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); - } @Test public void testUpdateDataSourceMetadataWithDocumentMissingException() { Mockito.when(encryptor.encrypt("secret_key")).thenReturn("secret_key"); Mockito.when(encryptor.encrypt("access_key")).thenReturn("access_key"); - Mockito.when(client.update(ArgumentMatchers.any())).thenThrow(new DocumentMissingException( - ShardId.fromString("[2][2]"), "testDS")); + Mockito.when(client.update(ArgumentMatchers.any())) + .thenThrow(new DocumentMissingException(ShardId.fromString("[2][2]"), "testDS")); DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); dataSourceMetadata.setName("testDS"); - DataSourceNotFoundException dataSourceNotFoundException = - Assertions.assertThrows(DataSourceNotFoundException.class, - () -> this.openSearchDataSourceMetadataStorage.updateDataSourceMetadata( - dataSourceMetadata)); - Assertions.assertEquals("Datasource with name: testDS doesn't exist", - dataSourceNotFoundException.getMessage()); + Assertions.assertThrows( + DataSourceNotFoundException.class, + () -> + this.openSearchDataSourceMetadataStorage.updateDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals( + "Datasource with name: testDS doesn't exist", dataSourceNotFoundException.getMessage()); Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); Mockito.verify(client, Mockito.times(1)).update(ArgumentMatchers.any()); Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); - } @Test @@ -548,19 +529,20 @@ public void testUpdateDataSourceMetadataWithRuntimeException() { DataSourceMetadata dataSourceMetadata = getDataSourceMetadata(); dataSourceMetadata.setName("testDS"); - - RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, - () -> this.openSearchDataSourceMetadataStorage.updateDataSourceMetadata( - dataSourceMetadata)); - Assertions.assertEquals("java.lang.RuntimeException: error message", - runtimeException.getMessage()); + RuntimeException runtimeException = + Assertions.assertThrows( + RuntimeException.class, + () -> + this.openSearchDataSourceMetadataStorage.updateDataSourceMetadata( + dataSourceMetadata)); + Assertions.assertEquals( + "java.lang.RuntimeException: error message", runtimeException.getMessage()); Mockito.verify(encryptor, Mockito.times(1)).encrypt("secret_key"); Mockito.verify(encryptor, Mockito.times(1)).encrypt("access_key"); Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); Mockito.verify(client, Mockito.times(1)).update(ArgumentMatchers.any()); Mockito.verify(client.threadPool().getThreadContext(), Mockito.times(1)).stashContext(); - } @Test @@ -584,11 +566,11 @@ public void testDeleteDataSourceMetadataWhichisAlreadyDeleted() { Mockito.when(deleteResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); DataSourceNotFoundException dataSourceNotFoundException = - Assertions.assertThrows(DataSourceNotFoundException.class, + Assertions.assertThrows( + DataSourceNotFoundException.class, () -> this.openSearchDataSourceMetadataStorage.deleteDataSourceMetadata("testDS")); - Assertions.assertEquals("Datasource with name: testDS doesn't exist", - dataSourceNotFoundException.getMessage()); - + Assertions.assertEquals( + "Datasource with name: testDS doesn't exist", dataSourceNotFoundException.getMessage()); Mockito.verifyNoInteractions(encryptor); Mockito.verify(client.admin().indices(), Mockito.times(0)).create(ArgumentMatchers.any()); @@ -602,9 +584,12 @@ public void testDeleteDataSourceMetadataWithUnexpectedResult() { Mockito.when(deleteResponseActionFuture.actionGet()).thenReturn(deleteResponse); Mockito.when(deleteResponse.getResult()).thenReturn(DocWriteResponse.Result.NOOP); - RuntimeException runtimeException = Assertions.assertThrows(RuntimeException.class, - () -> this.openSearchDataSourceMetadataStorage.deleteDataSourceMetadata("testDS")); - Assertions.assertEquals("Deleting dataSource metadata information failed with result : noop", + RuntimeException runtimeException = + Assertions.assertThrows( + RuntimeException.class, + () -> this.openSearchDataSourceMetadataStorage.deleteDataSourceMetadata("testDS")); + Assertions.assertEquals( + "Deleting dataSource metadata information failed with result : noop", runtimeException.getMessage()); Mockito.verifyNoInteractions(encryptor); @@ -684,5 +669,4 @@ private DataSourceMetadata getDataSourceMetadata() { dataSourceMetadata.setProperties(properties); return dataSourceMetadata; } - } diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceActionTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceActionTest.java index ccae5de2a9d..f1a3a2875e7 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceActionTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportCreateDataSourceActionTest.java @@ -27,27 +27,23 @@ @ExtendWith(MockitoExtension.class) public class TransportCreateDataSourceActionTest { - @Mock - private TransportService transportService; - @Mock - private TransportCreateDataSourceAction action; - @Mock - private DataSourceServiceImpl dataSourceService; - @Mock - private Task task; - @Mock - private ActionListener actionListener; + @Mock private TransportService transportService; + @Mock private TransportCreateDataSourceAction action; + @Mock private DataSourceServiceImpl dataSourceService; + @Mock private Task task; + @Mock private ActionListener actionListener; + @Captor private ArgumentCaptor createDataSourceActionResponseArgumentCaptor; - @Captor - private ArgumentCaptor exceptionArgumentCaptor; + @Captor private ArgumentCaptor exceptionArgumentCaptor; @BeforeEach public void setUp() { - action = new TransportCreateDataSourceAction(transportService, - new ActionFilters(new HashSet<>()), dataSourceService); + action = + new TransportCreateDataSourceAction( + transportService, new ActionFilters(new HashSet<>()), dataSourceService); } @Test @@ -61,10 +57,10 @@ public void testDoExecute() { verify(dataSourceService, times(1)).createDataSource(dataSourceMetadata); Mockito.verify(actionListener) .onResponse(createDataSourceActionResponseArgumentCaptor.capture()); - CreateDataSourceActionResponse createDataSourceActionResponse - = createDataSourceActionResponseArgumentCaptor.getValue(); - Assertions.assertEquals("Created DataSource with name test_datasource", - createDataSourceActionResponse.getResult()); + CreateDataSourceActionResponse createDataSourceActionResponse = + createDataSourceActionResponseArgumentCaptor.getValue(); + Assertions.assertEquals( + "Created DataSource with name test_datasource", createDataSourceActionResponse.getResult()); } @Test @@ -72,7 +68,8 @@ public void testDoExecuteWithException() { DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); dataSourceMetadata.setName("test_datasource"); dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); - doThrow(new RuntimeException("Error")).when(dataSourceService) + doThrow(new RuntimeException("Error")) + .when(dataSourceService) .createDataSource(dataSourceMetadata); CreateDataSourceActionRequest request = new CreateDataSourceActionRequest(dataSourceMetadata); action.doExecute(task, request, actionListener); @@ -80,7 +77,6 @@ public void testDoExecuteWithException() { Mockito.verify(actionListener).onFailure(exceptionArgumentCaptor.capture()); Exception exception = exceptionArgumentCaptor.getValue(); Assertions.assertTrue(exception instanceof RuntimeException); - Assertions.assertEquals("Error", - exception.getMessage()); + Assertions.assertEquals("Error", exception.getMessage()); } } diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceActionTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceActionTest.java index e97e7d1a650..ea581de20c3 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceActionTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportDeleteDataSourceActionTest.java @@ -25,28 +25,23 @@ @ExtendWith(MockitoExtension.class) public class TransportDeleteDataSourceActionTest { - @Mock - private TransportService transportService; - @Mock - private TransportDeleteDataSourceAction action; - @Mock - private DataSourceServiceImpl dataSourceService; - @Mock - private Task task; - @Mock - private ActionListener actionListener; + @Mock private TransportService transportService; + @Mock private TransportDeleteDataSourceAction action; + @Mock private DataSourceServiceImpl dataSourceService; + @Mock private Task task; + @Mock private ActionListener actionListener; @Captor private ArgumentCaptor deleteDataSourceActionResponseArgumentCaptor; - @Captor - private ArgumentCaptor exceptionArgumentCaptor; + @Captor private ArgumentCaptor exceptionArgumentCaptor; @BeforeEach public void setUp() { - action = new TransportDeleteDataSourceAction(transportService, - new ActionFilters(new HashSet<>()), dataSourceService); + action = + new TransportDeleteDataSourceAction( + transportService, new ActionFilters(new HashSet<>()), dataSourceService); } @Test @@ -57,10 +52,10 @@ public void testDoExecute() { verify(dataSourceService, times(1)).deleteDataSource("test_datasource"); Mockito.verify(actionListener) .onResponse(deleteDataSourceActionResponseArgumentCaptor.capture()); - DeleteDataSourceActionResponse deleteDataSourceActionResponse - = deleteDataSourceActionResponseArgumentCaptor.getValue(); - Assertions.assertEquals("Deleted DataSource with name test_datasource", - deleteDataSourceActionResponse.getResult()); + DeleteDataSourceActionResponse deleteDataSourceActionResponse = + deleteDataSourceActionResponseArgumentCaptor.getValue(); + Assertions.assertEquals( + "Deleted DataSource with name test_datasource", deleteDataSourceActionResponse.getResult()); } @Test @@ -72,7 +67,6 @@ public void testDoExecuteWithException() { Mockito.verify(actionListener).onFailure(exceptionArgumentCaptor.capture()); Exception exception = exceptionArgumentCaptor.getValue(); Assertions.assertTrue(exception instanceof RuntimeException); - Assertions.assertEquals("Error", - exception.getMessage()); + Assertions.assertEquals("Error", exception.getMessage()); } } diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceActionTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceActionTest.java index fc4439470e1..4f04afd667d 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceActionTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportGetDataSourceActionTest.java @@ -34,27 +34,22 @@ @ExtendWith(MockitoExtension.class) public class TransportGetDataSourceActionTest { - @Mock - private TransportService transportService; - @Mock - private TransportGetDataSourceAction action; - @Mock - private DataSourceServiceImpl dataSourceService; - @Mock - private Task task; - @Mock - private ActionListener actionListener; + @Mock private TransportService transportService; + @Mock private TransportGetDataSourceAction action; + @Mock private DataSourceServiceImpl dataSourceService; + @Mock private Task task; + @Mock private ActionListener actionListener; @Captor private ArgumentCaptor getDataSourceActionResponseArgumentCaptor; - @Captor - private ArgumentCaptor exceptionArgumentCaptor; + @Captor private ArgumentCaptor exceptionArgumentCaptor; @BeforeEach public void setUp() { - action = new TransportGetDataSourceAction(transportService, - new ActionFilters(new HashSet<>()), dataSourceService); + action = + new TransportGetDataSourceAction( + transportService, new ActionFilters(new HashSet<>()), dataSourceService); } @Test @@ -63,23 +58,22 @@ public void testDoExecute() { dataSourceMetadata.setName("test_datasource"); dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); GetDataSourceActionRequest request = new GetDataSourceActionRequest("test_datasource"); - when(dataSourceService.getDataSourceMetadata("test_datasource")) - .thenReturn(dataSourceMetadata); + when(dataSourceService.getDataSourceMetadata("test_datasource")).thenReturn(dataSourceMetadata); action.doExecute(task, request, actionListener); verify(dataSourceService, times(1)).getDataSourceMetadata("test_datasource"); Mockito.verify(actionListener).onResponse(getDataSourceActionResponseArgumentCaptor.capture()); - GetDataSourceActionResponse getDataSourceActionResponse - = getDataSourceActionResponseArgumentCaptor.getValue(); + GetDataSourceActionResponse getDataSourceActionResponse = + getDataSourceActionResponseArgumentCaptor.getValue(); JsonResponseFormatter dataSourceMetadataJsonResponseFormatter = - new JsonResponseFormatter<>( - JsonResponseFormatter.Style.PRETTY) { + new JsonResponseFormatter<>(JsonResponseFormatter.Style.PRETTY) { @Override protected Object buildJsonObject(DataSourceMetadata response) { return response; } }; - Assertions.assertEquals(dataSourceMetadataJsonResponseFormatter.format(dataSourceMetadata), + Assertions.assertEquals( + dataSourceMetadataJsonResponseFormatter.format(dataSourceMetadata), getDataSourceActionResponse.getResult()); DataSourceMetadata result = new Gson().fromJson(getDataSourceActionResponse.getResult(), DataSourceMetadata.class); @@ -100,18 +94,16 @@ public void testDoExecuteForGetAllDataSources() { action.doExecute(task, request, actionListener); verify(dataSourceService, times(1)).getDataSourceMetadata(false); Mockito.verify(actionListener).onResponse(getDataSourceActionResponseArgumentCaptor.capture()); - GetDataSourceActionResponse getDataSourceActionResponse - = getDataSourceActionResponseArgumentCaptor.getValue(); + GetDataSourceActionResponse getDataSourceActionResponse = + getDataSourceActionResponseArgumentCaptor.getValue(); JsonResponseFormatter> dataSourceMetadataJsonResponseFormatter = - new JsonResponseFormatter<>( - JsonResponseFormatter.Style.PRETTY) { + new JsonResponseFormatter<>(JsonResponseFormatter.Style.PRETTY) { @Override protected Object buildJsonObject(Set response) { return response; } }; - Type setType = new TypeToken>() { - }.getType(); + Type setType = new TypeToken>() {}.getType(); Assertions.assertEquals( dataSourceMetadataJsonResponseFormatter.format(Collections.singleton(dataSourceMetadata)), getDataSourceActionResponse.getResult()); @@ -131,7 +123,6 @@ public void testDoExecuteWithException() { Mockito.verify(actionListener).onFailure(exceptionArgumentCaptor.capture()); Exception exception = exceptionArgumentCaptor.getValue(); Assertions.assertTrue(exception instanceof RuntimeException); - Assertions.assertEquals("Error", - exception.getMessage()); + Assertions.assertEquals("Error", exception.getMessage()); } } diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceActionTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceActionTest.java index 4b5a6e0f573..998a1aa7b21 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceActionTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/transport/TransportUpdateDataSourceActionTest.java @@ -27,28 +27,23 @@ @ExtendWith(MockitoExtension.class) public class TransportUpdateDataSourceActionTest { - @Mock - private TransportService transportService; - @Mock - private TransportUpdateDataSourceAction action; - @Mock - private DataSourceServiceImpl dataSourceService; - @Mock - private Task task; - @Mock - private ActionListener actionListener; + @Mock private TransportService transportService; + @Mock private TransportUpdateDataSourceAction action; + @Mock private DataSourceServiceImpl dataSourceService; + @Mock private Task task; + @Mock private ActionListener actionListener; @Captor private ArgumentCaptor updateDataSourceActionResponseArgumentCaptor; - @Captor - private ArgumentCaptor exceptionArgumentCaptor; + @Captor private ArgumentCaptor exceptionArgumentCaptor; @BeforeEach public void setUp() { - action = new TransportUpdateDataSourceAction(transportService, - new ActionFilters(new HashSet<>()), dataSourceService); + action = + new TransportUpdateDataSourceAction( + transportService, new ActionFilters(new HashSet<>()), dataSourceService); } @Test @@ -62,10 +57,10 @@ public void testDoExecute() { verify(dataSourceService, times(1)).updateDataSource(dataSourceMetadata); Mockito.verify(actionListener) .onResponse(updateDataSourceActionResponseArgumentCaptor.capture()); - UpdateDataSourceActionResponse updateDataSourceActionResponse - = updateDataSourceActionResponseArgumentCaptor.getValue(); - Assertions.assertEquals("Updated DataSource with name test_datasource", - updateDataSourceActionResponse.getResult()); + UpdateDataSourceActionResponse updateDataSourceActionResponse = + updateDataSourceActionResponseArgumentCaptor.getValue(); + Assertions.assertEquals( + "Updated DataSource with name test_datasource", updateDataSourceActionResponse.getResult()); } @Test @@ -73,7 +68,8 @@ public void testDoExecuteWithException() { DataSourceMetadata dataSourceMetadata = new DataSourceMetadata(); dataSourceMetadata.setName("test_datasource"); dataSourceMetadata.setConnector(DataSourceType.PROMETHEUS); - doThrow(new RuntimeException("Error")).when(dataSourceService) + doThrow(new RuntimeException("Error")) + .when(dataSourceService) .updateDataSource(dataSourceMetadata); UpdateDataSourceActionRequest request = new UpdateDataSourceActionRequest(dataSourceMetadata); action.doExecute(task, request, actionListener); @@ -81,7 +77,6 @@ public void testDoExecuteWithException() { Mockito.verify(actionListener).onFailure(exceptionArgumentCaptor.capture()); Exception exception = exceptionArgumentCaptor.getValue(); Assertions.assertTrue(exception instanceof RuntimeException); - Assertions.assertEquals("Error", - exception.getMessage()); + Assertions.assertEquals("Error", exception.getMessage()); } } diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/utils/SchedulerTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/utils/SchedulerTest.java index e3dac306cd4..ff23cdcabbd 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/utils/SchedulerTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/utils/SchedulerTest.java @@ -19,27 +19,24 @@ @ExtendWith(MockitoExtension.class) public class SchedulerTest { - @Mock - private NodeClient nodeClient; + @Mock private NodeClient nodeClient; - @Mock - private ThreadPool threadPool; + @Mock private ThreadPool threadPool; @Test public void testSchedule() { Mockito.when(nodeClient.threadPool()).thenReturn(threadPool); Mockito.doAnswer( - invocation -> { - Runnable task = invocation.getArgument(0); - task.run(); - return null; - }) + invocation -> { + Runnable task = invocation.getArgument(0); + task.run(); + return null; + }) .when(threadPool) .schedule(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any()); AtomicBoolean isRun = new AtomicBoolean(false); Scheduler.schedule(nodeClient, () -> isRun.set(true)); Assert.assertTrue(isRun.get()); } - } diff --git a/datasources/src/test/java/org/opensearch/sql/datasources/utils/XContentParserUtilsTest.java b/datasources/src/test/java/org/opensearch/sql/datasources/utils/XContentParserUtilsTest.java index f47d0503e7e..c0c05c02821 100644 --- a/datasources/src/test/java/org/opensearch/sql/datasources/utils/XContentParserUtilsTest.java +++ b/datasources/src/test/java/org/opensearch/sql/datasources/utils/XContentParserUtilsTest.java @@ -30,7 +30,8 @@ public void testConvertToXContent() { XContentBuilder contentBuilder = XContentParserUtils.convertToXContent(dataSourceMetadata); String contentString = BytesReference.bytes(contentBuilder).utf8ToString(); - Assertions.assertEquals("{\"name\":\"testDS\",\"connector\":\"PROMETHEUS\",\"allowedRoles\":[\"prometheus_access\"],\"properties\":{\"prometheus.uri\":\"https://localhost:9090\"}}", + Assertions.assertEquals( + "{\"name\":\"testDS\",\"connector\":\"PROMETHEUS\",\"allowedRoles\":[\"prometheus_access\"],\"properties\":{\"prometheus.uri\":\"https://localhost:9090\"}}", contentString); } @@ -49,7 +50,6 @@ public void testToDataSourceMetadataFromJson() { Assertions.assertEquals(retrievedMetadata, dataSourceMetadata); Assertions.assertEquals("prometheus_access", retrievedMetadata.getAllowedRoles().get(0)); - } @SneakyThrows @@ -62,9 +62,12 @@ public void testToDataSourceMetadataFromJsonWithoutName() { Gson gson = new Gson(); String json = gson.toJson(dataSourceMetadata); - IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> { - XContentParserUtils.toDataSourceMetadata(json); - }); + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> { + XContentParserUtils.toDataSourceMetadata(json); + }); Assertions.assertEquals("name and connector are required fields.", exception.getMessage()); } @@ -78,9 +81,12 @@ public void testToDataSourceMetadataFromJsonWithoutConnector() { Gson gson = new Gson(); String json = gson.toJson(dataSourceMetadata); - IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> { - XContentParserUtils.toDataSourceMetadata(json); - }); + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> { + XContentParserUtils.toDataSourceMetadata(json); + }); Assertions.assertEquals("name and connector are required fields.", exception.getMessage()); } @@ -92,10 +98,12 @@ public void testToDataSourceMetadataFromJsonUsingUnknownObject() { Gson gson = new Gson(); String json = gson.toJson(hashMap); - IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> { - XContentParserUtils.toDataSourceMetadata(json); - }); + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> { + XContentParserUtils.toDataSourceMetadata(json); + }); Assertions.assertEquals("Unknown field: test", exception.getMessage()); } - }