diff --git a/contrib/storage-http/README.md b/contrib/storage-http/README.md index 0f188ba0cc5..d384bedd117 100644 --- a/contrib/storage-http/README.md +++ b/contrib/storage-http/README.md @@ -306,6 +306,9 @@ on 400 series errors, `false` will return an empty result set (with implicit fie Default is `true`, but when set to false, Drill will trust all SSL certificates. Useful for debugging or on internal corporate networks using self-signed certificates or private certificate authorities. +#### caseSensitiveFilters +Some APIs are case sensitive with the fields which are pushed down. If the endpoint that you are working with is in fact case sensitive, simply set this to `true`. Defaults to `false`. + ## Usage This plugin is different from other plugins in that it the table component of the `FROM` clause @@ -577,9 +580,9 @@ ORDER BY issue_count DESC you may encounter errors or empty responses if you are counting on the endpoint for redirection. -2. At this time, the plugin does not support any authentication other than basic authentication. +~~2. At this time, the plugin does not support any authentication other than basic authentication.~~ -3. This plugin does not implement join filter pushdowns (only constant plushdowns are +3. This plugin does not implement join filter pushdowns (only constant pushdowns are supported). Join pushdown has the potential to improve performance if you use the HTTP service joined to another table. diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java index 581b61cfccc..91af33e36f7 100644 --- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java +++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java @@ -104,6 +104,9 @@ public class HttpApiConfig { @JsonProperty private final boolean errorOn400; + @JsonProperty + private final boolean caseSensitiveFilters; + // Enables the user to configure JSON options at the connection level rather than globally. @JsonProperty private final HttpJsonOptions jsonOptions; @@ -157,6 +160,10 @@ public String inputType() { return this.inputType; } + public boolean caseSensitiveFilters() { + return this.caseSensitiveFilters; + } + public int xmlDataLevel() { return this.xmlDataLevel; } @@ -199,6 +206,7 @@ public boolean equals(Object o) { && errorOn400 == that.errorOn400 && verifySSLCert == that.verifySSLCert && directCredentials == that.directCredentials + && caseSensitiveFilters == that.caseSensitiveFilters && Objects.equals(url, that.url) && Objects.equals(method, that.method) && Objects.equals(postBody, that.postBody) @@ -218,7 +226,7 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(url, requireTail, method, postBody, headers, params, dataPath, authType, inputType, xmlDataLevel, limitQueryParam, errorOn400, jsonOptions, verifySSLCert, - credentialsProvider, paginator, directCredentials, postParameterLocation); + credentialsProvider, paginator, directCredentials, postParameterLocation, caseSensitiveFilters); } @Override @@ -232,6 +240,7 @@ public String toString() { .field("headers", headers) .field("params", params) .field("dataPath", dataPath) + .field("caseSensitiveFilters", caseSensitiveFilters) .field("authType", authType) .field("inputType", inputType) .field("xmlDataLevel", xmlDataLevel) @@ -328,6 +337,7 @@ private HttpApiConfig(HttpApiConfig.HttpApiConfigBuilder builder) { this.xmlDataLevel = Math.max(1, builder.xmlDataLevel); this.errorOn400 = builder.errorOn400; + this.caseSensitiveFilters = builder.caseSensitiveFilters; this.credentialsProvider = CredentialProviderUtils.getCredentialsProvider(builder.userName, builder.password, builder.credentialsProvider); this.directCredentials = builder.credentialsProvider == null; @@ -422,7 +432,7 @@ public static class HttpApiConfigBuilder { private String authType; private int xmlDataLevel; - + private boolean caseSensitiveFilters; private String limitQueryParam; private boolean errorOn400; @@ -489,6 +499,11 @@ public HttpApiConfigBuilder url(String url) { return this; } + public HttpApiConfigBuilder caseSensitiveFilters(boolean caseSensitiveFilters) { + this.caseSensitiveFilters = caseSensitiveFilters; + return this; + } + public HttpApiConfigBuilder method(String method) { this.method = method; return this; diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java index c4f55967e69..30f4be57fab 100644 --- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java +++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java @@ -35,6 +35,7 @@ import org.apache.drill.exec.store.http.util.SimpleHttp; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -175,7 +176,13 @@ private boolean acceptType(MinorType type) { */ @Override public Pair> transform(AndNode andNode) { - Map filters = CaseInsensitiveMap.newHashMap(); + Map filters; + if (groupScan.getHttpConfig().caseSensitiveFilters()) { + filters = new HashMap<>(); + } else { + filters = CaseInsensitiveMap.newHashMap(); + } + double selectivity = 1; for (ExprNode expr : andNode.children) { ColRelOpConstNode relOp = (ColRelOpConstNode) expr; diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java index e5f0ce70d5f..ee5b285f1ef 100644 --- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java +++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java @@ -35,7 +35,6 @@ import org.apache.drill.common.exceptions.EmptyErrorContext; import org.apache.drill.common.logical.OAuthConfig; import org.apache.drill.common.logical.StoragePluginConfig.AuthMode; -import org.apache.drill.common.map.CaseInsensitiveMap; import org.apache.drill.common.exceptions.CustomErrorContext; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.ExecConstants; @@ -712,7 +711,6 @@ public static String mapURLParameters(HttpUrl url, Map filters) .message("API Query with URL Parameters must be populated.") .build(logger); } - CaseInsensitiveMapcaseInsensitiveFilterMap = (CaseInsensitiveMap)filters; List params = SimpleHttp.getURLParameters(url); String tempUrl = SimpleHttp.decodedURL(url); @@ -726,7 +724,7 @@ public static String mapURLParameters(HttpUrl url, Map filters) // of providing helpful errors in strange cases, it is there. - String value = caseInsensitiveFilterMap.get(param); + String value = filters.get(param); // Check and see if there is a default for this parameter. If not throw an error. if (StringUtils.isEmpty(value)) {