From abcaef4cff98778f0d4db78265d2caed7988d5ff Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Sat, 3 Sep 2022 23:10:21 -0400 Subject: [PATCH 1/3] DRILL-8291: Allow Case Sensitive Filters in HTTP Plugin --- .../drill/exec/store/http/HttpApiConfig.java | 19 +++++++++++++++++-- .../exec/store/http/HttpPushDownListener.java | 9 ++++++++- .../exec/store/http/util/SimpleHttp.java | 3 +-- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java index 581b61cfccc..91af33e36f7 100644 --- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java +++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpApiConfig.java @@ -104,6 +104,9 @@ public class HttpApiConfig { @JsonProperty private final boolean errorOn400; + @JsonProperty + private final boolean caseSensitiveFilters; + // Enables the user to configure JSON options at the connection level rather than globally. @JsonProperty private final HttpJsonOptions jsonOptions; @@ -157,6 +160,10 @@ public String inputType() { return this.inputType; } + public boolean caseSensitiveFilters() { + return this.caseSensitiveFilters; + } + public int xmlDataLevel() { return this.xmlDataLevel; } @@ -199,6 +206,7 @@ public boolean equals(Object o) { && errorOn400 == that.errorOn400 && verifySSLCert == that.verifySSLCert && directCredentials == that.directCredentials + && caseSensitiveFilters == that.caseSensitiveFilters && Objects.equals(url, that.url) && Objects.equals(method, that.method) && Objects.equals(postBody, that.postBody) @@ -218,7 +226,7 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(url, requireTail, method, postBody, headers, params, dataPath, authType, inputType, xmlDataLevel, limitQueryParam, errorOn400, jsonOptions, verifySSLCert, - credentialsProvider, paginator, directCredentials, postParameterLocation); + credentialsProvider, paginator, directCredentials, postParameterLocation, caseSensitiveFilters); } @Override @@ -232,6 +240,7 @@ public String toString() { .field("headers", headers) .field("params", params) .field("dataPath", dataPath) + .field("caseSensitiveFilters", caseSensitiveFilters) .field("authType", authType) .field("inputType", inputType) .field("xmlDataLevel", xmlDataLevel) @@ -328,6 +337,7 @@ private HttpApiConfig(HttpApiConfig.HttpApiConfigBuilder builder) { this.xmlDataLevel = Math.max(1, builder.xmlDataLevel); this.errorOn400 = builder.errorOn400; + this.caseSensitiveFilters = builder.caseSensitiveFilters; this.credentialsProvider = CredentialProviderUtils.getCredentialsProvider(builder.userName, builder.password, builder.credentialsProvider); this.directCredentials = builder.credentialsProvider == null; @@ -422,7 +432,7 @@ public static class HttpApiConfigBuilder { private String authType; private int xmlDataLevel; - + private boolean caseSensitiveFilters; private String limitQueryParam; private boolean errorOn400; @@ -489,6 +499,11 @@ public HttpApiConfigBuilder url(String url) { return this; } + public HttpApiConfigBuilder caseSensitiveFilters(boolean caseSensitiveFilters) { + this.caseSensitiveFilters = caseSensitiveFilters; + return this; + } + public HttpApiConfigBuilder method(String method) { this.method = method; return this; diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java index c4f55967e69..30f4be57fab 100644 --- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java +++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/HttpPushDownListener.java @@ -35,6 +35,7 @@ import org.apache.drill.exec.store.http.util.SimpleHttp; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -175,7 +176,13 @@ private boolean acceptType(MinorType type) { */ @Override public Pair> transform(AndNode andNode) { - Map filters = CaseInsensitiveMap.newHashMap(); + Map filters; + if (groupScan.getHttpConfig().caseSensitiveFilters()) { + filters = new HashMap<>(); + } else { + filters = CaseInsensitiveMap.newHashMap(); + } + double selectivity = 1; for (ExprNode expr : andNode.children) { ColRelOpConstNode relOp = (ColRelOpConstNode) expr; diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java index e5f0ce70d5f..c4ab65ada08 100644 --- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java +++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java @@ -712,7 +712,6 @@ public static String mapURLParameters(HttpUrl url, Map filters) .message("API Query with URL Parameters must be populated.") .build(logger); } - CaseInsensitiveMapcaseInsensitiveFilterMap = (CaseInsensitiveMap)filters; List params = SimpleHttp.getURLParameters(url); String tempUrl = SimpleHttp.decodedURL(url); @@ -726,7 +725,7 @@ public static String mapURLParameters(HttpUrl url, Map filters) // of providing helpful errors in strange cases, it is there. - String value = caseInsensitiveFilterMap.get(param); + String value = filters.get(param); // Check and see if there is a default for this parameter. If not throw an error. if (StringUtils.isEmpty(value)) { From 45a6bb069ffce2e8c47027a45246bc734fd34368 Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Sun, 4 Sep 2022 08:38:37 -0400 Subject: [PATCH 2/3] Fixed imports --- .../java/org/apache/drill/exec/store/http/util/SimpleHttp.java | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java index c4ab65ada08..ee5b285f1ef 100644 --- a/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java +++ b/contrib/storage-http/src/main/java/org/apache/drill/exec/store/http/util/SimpleHttp.java @@ -35,7 +35,6 @@ import org.apache.drill.common.exceptions.EmptyErrorContext; import org.apache.drill.common.logical.OAuthConfig; import org.apache.drill.common.logical.StoragePluginConfig.AuthMode; -import org.apache.drill.common.map.CaseInsensitiveMap; import org.apache.drill.common.exceptions.CustomErrorContext; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.ExecConstants; From af07944b1b48e3b88a7eee1361ebdc79172281c2 Mon Sep 17 00:00:00 2001 From: Charles Givre Date: Sun, 4 Sep 2022 08:46:29 -0400 Subject: [PATCH 3/3] Updated docs --- contrib/storage-http/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/contrib/storage-http/README.md b/contrib/storage-http/README.md index 0f188ba0cc5..d384bedd117 100644 --- a/contrib/storage-http/README.md +++ b/contrib/storage-http/README.md @@ -306,6 +306,9 @@ on 400 series errors, `false` will return an empty result set (with implicit fie Default is `true`, but when set to false, Drill will trust all SSL certificates. Useful for debugging or on internal corporate networks using self-signed certificates or private certificate authorities. +#### caseSensitiveFilters +Some APIs are case sensitive with the fields which are pushed down. If the endpoint that you are working with is in fact case sensitive, simply set this to `true`. Defaults to `false`. + ## Usage This plugin is different from other plugins in that it the table component of the `FROM` clause @@ -577,9 +580,9 @@ ORDER BY issue_count DESC you may encounter errors or empty responses if you are counting on the endpoint for redirection. -2. At this time, the plugin does not support any authentication other than basic authentication. +~~2. At this time, the plugin does not support any authentication other than basic authentication.~~ -3. This plugin does not implement join filter pushdowns (only constant plushdowns are +3. This plugin does not implement join filter pushdowns (only constant pushdowns are supported). Join pushdown has the potential to improve performance if you use the HTTP service joined to another table.