diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 60cbe4edcb4d..f54e88c93044 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -418,14 +418,6 @@ com.google.protobuf protobuf-java - - org.apache.httpcomponents - httpcore - - - org.apache.httpcomponents - httpclient - diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlQueryTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlQueryTest.java index 46d6a938321e..05fca8cf3027 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlQueryTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlQueryTest.java @@ -22,23 +22,24 @@ import com.google.inject.Inject; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.java.util.http.client.Request; +import org.apache.druid.java.util.http.client.response.StatusResponseHandler; +import org.apache.druid.java.util.http.client.response.StatusResponseHolder; import org.apache.druid.testing.IntegrationTestingConfig; import org.apache.druid.testing.guice.DruidTestModuleFactory; +import org.apache.druid.testing.guice.TestClient; +import org.apache.druid.testing.utils.ITRetryUtil; import org.apache.druid.tests.TestNGGroup; -import org.apache.http.HttpEntity; -import org.apache.http.HttpStatus; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.util.EntityUtils; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.testng.Assert; import org.testng.annotations.Guice; import org.testng.annotations.Test; import javax.ws.rs.core.MediaType; import java.io.IOException; +import java.net.URL; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.function.Function; @@ -46,77 +47,75 @@ /** * Test the SQL endpoint with different Content-Type */ -@Test(groups = {TestNGGroup.QUERY, TestNGGroup.CENTRALIZED_DATASOURCE_SCHEMA}) +@Test(groups = {TestNGGroup.QUERY}) @Guice(moduleFactory = DruidTestModuleFactory.class) public class ITSqlQueryTest { - private static final Logger LOG = new Logger(ITSqlQueryTest.class); - @Inject IntegrationTestingConfig config; + @Inject + @TestClient + HttpClient httpClient; + interface IExecutable { - void execute(String endpoint) throws IOException; + void execute(String endpoint) throws Exception; } interface OnRequest { - void on(HttpPost request) throws IOException; + void on(Request request) throws IOException; } interface OnResponse { - void on(int statusCode, HttpEntity response) throws IOException; + void on(int statusCode, String response) throws IOException; } private void executeWithRetry(String endpoint, String contentType, IExecutable executable) { - Throwable lastException = null; - for (int i = 1; i <= 5; i++) { - LOG.info("Query to %s with Content-Type = %s, tries = %s", endpoint, contentType, i); - try { - executable.execute(endpoint); - return; - } - catch (IOException e) { - // Only catch IOException - lastException = e; - } - try { - Thread.sleep(200); - } - catch (InterruptedException ignored) { - break; - } - } - throw new ISE(contentType + " failed after 5 tries, last exception: " + lastException); + // Retry 5 times with 200 ms delay + ITRetryUtil.retryUntilEquals( + () -> { + executable.execute(endpoint); + return true; + }, true, + 200, + 5, + StringUtils.format("Query to %s with Content-Type = %s", endpoint, contentType) + ); } private void executeQuery( String contentType, + String query, OnRequest onRequest, OnResponse onResponse ) { IExecutable executable = (endpoint) -> { - try (CloseableHttpClient client = HttpClientBuilder.create().build()) { - HttpPost request = new HttpPost(endpoint); - if (contentType != null) { - request.addHeader("Content-Type", contentType); - } - onRequest.on(request); + Request request = new Request(HttpMethod.POST, new URL(endpoint)); + if (contentType != null) { + request.addHeader("Content-Type", contentType); + } - try (CloseableHttpResponse response = client.execute(request)) { - HttpEntity responseEntity = response.getEntity(); - assertNotNull(responseEntity); + if (query != null) { + request.setContent(query.getBytes(StandardCharsets.UTF_8)); + } - onResponse.on( - response.getStatusLine().getStatusCode(), - responseEntity - ); - } + if (onRequest != null) { + onRequest.on(request); } + + StatusResponseHolder response = httpClient.go(request, StatusResponseHandler.getInstance()) + .get(); + Assert.assertNotNull(response); + + onResponse.on( + response.getStatus().getCode(), + response.getContent().trim() + ); }; // Send query to broker to exeucte @@ -126,27 +125,6 @@ private void executeQuery( executeWithRetry(StringUtils.format("%s/druid/v2/sql/", config.getRouterUrl()), contentType, executable); } - private void assertEquals(String expected, String actual) - { - if (!expected.equals(actual)) { - throw new ISE("Expected [%s] but got [%s]", expected, actual); - } - } - - private void assertEquals(int expected, int actual) - { - if (expected != actual) { - throw new ISE("Expected [%d] but got [%d]", expected, actual); - } - } - - private void assertNotNull(Object object) - { - if (object == null) { - throw new ISE("Expected not null"); - } - } - private void assertStringCompare(String expected, String actual, Function predicate) { if (!predicate.apply(expected)) { @@ -159,13 +137,11 @@ public void testNullContentType() { executeQuery( null, + "select 1", (request) -> { - request.setEntity(new StringEntity("select 1")); }, - (statusCode, responseEntity) -> { - assertEquals(HttpStatus.SC_UNSUPPORTED_MEDIA_TYPE, statusCode); - - String responseBody = EntityUtils.toString(responseEntity).trim(); + (statusCode, responseBody) -> { + Assert.assertEquals(statusCode, HttpResponseStatus.UNSUPPORTED_MEDIA_TYPE.getCode(), responseBody); assertStringCompare("Unsupported Content-Type:", responseBody, responseBody::contains); } ); @@ -176,13 +152,11 @@ public void testUnsupportedContentType() { executeQuery( "application/xml", + "select 1", (request) -> { - request.setEntity(new StringEntity("select 1")); }, - (statusCode, responseEntity) -> { - assertEquals(HttpStatus.SC_UNSUPPORTED_MEDIA_TYPE, statusCode); - - String responseBody = EntityUtils.toString(responseEntity).trim(); + (statusCode, responseBody) -> { + Assert.assertEquals(statusCode, HttpResponseStatus.UNSUPPORTED_MEDIA_TYPE.getCode(), responseBody); assertStringCompare("Unsupported Content-Type:", responseBody, responseBody::contains); } ); @@ -193,14 +167,12 @@ public void testTextPlain() { executeQuery( MediaType.TEXT_PLAIN, + "select \n1", (request) -> { - request.setEntity(new StringEntity("select \n1")); }, - (statusCode, responseEntity) -> { - assertEquals(200, statusCode); - - String responseBody = EntityUtils.toString(responseEntity).trim(); - assertEquals("[{\"EXPR$0\":1}]", responseBody); + (statusCode, responseBody) -> { + Assert.assertEquals(statusCode, 200, responseBody); + Assert.assertEquals(responseBody, "[{\"EXPR$0\":1}]"); } ); } @@ -210,14 +182,12 @@ public void testFormURLEncoded() { executeQuery( MediaType.APPLICATION_FORM_URLENCODED, + URLEncoder.encode("select 'x % y'", StandardCharsets.UTF_8), (request) -> { - request.setEntity(new StringEntity(URLEncoder.encode("select 'x % y'", StandardCharsets.UTF_8))); }, - (statusCode, responseEntity) -> { - assertEquals(200, statusCode); - - String responseBody = EntityUtils.toString(responseEntity).trim(); - assertEquals("[{\"EXPR$0\":\"x % y\"}]", responseBody); + (statusCode, responseBody) -> { + Assert.assertEquals(statusCode, 200, responseBody); + Assert.assertEquals(responseBody, "[{\"EXPR$0\":\"x % y\"}]"); } ); } @@ -227,14 +197,12 @@ public void testFormURLEncoded_InvalidEncoding() { executeQuery( MediaType.APPLICATION_FORM_URLENCODED, + "select 'x % y'", (request) -> { - request.setEntity(new StringEntity("select 'x % y'")); }, - (statusCode, responseEntity) -> { - assertEquals(400, statusCode); - - String responseBody = EntityUtils.toString(responseEntity).trim(); - assertStringCompare("Unable to decoded", responseBody, responseBody::contains); + (statusCode, responseBody) -> { + Assert.assertEquals(statusCode, 400, responseBody); + assertStringCompare("Unable to decode", responseBody, responseBody::contains); } ); } @@ -244,14 +212,12 @@ public void testJSON() { executeQuery( MediaType.APPLICATION_JSON, + "{\"query\":\"select 567\"}", (request) -> { - request.setEntity(new StringEntity(StringUtils.format("{\"query\":\"select 567\"}"))); }, - (statusCode, responseEntity) -> { - assertEquals(200, statusCode); - - String responseBody = EntityUtils.toString(responseEntity).trim(); - assertEquals("[{\"EXPR$0\":567}]", responseBody); + (statusCode, responseBody) -> { + Assert.assertEquals(statusCode, 200, responseBody); + Assert.assertEquals(responseBody, "[{\"EXPR$0\":567}]"); } ); } @@ -261,13 +227,11 @@ public void testInvalidJSONFormat() { executeQuery( MediaType.APPLICATION_JSON, + "{\"query\":select 567}", (request) -> { - request.setEntity(new StringEntity(StringUtils.format("{\"query\":select 567}"))); }, - (statusCode, responseEntity) -> { - assertEquals(400, statusCode); - - String responseBody = EntityUtils.toString(responseEntity).trim(); + (statusCode, responseBody) -> { + Assert.assertEquals(statusCode, 400, responseBody); assertStringCompare("Malformed SQL query", responseBody, responseBody::contains); } ); @@ -278,13 +242,11 @@ public void testEmptyQuery_TextPlain() { executeQuery( MediaType.TEXT_PLAIN, + null, (request) -> { - // Empty query, DO NOTHING }, - (statusCode, responseEntity) -> { - assertEquals(400, statusCode); - - String responseBody = EntityUtils.toString(responseEntity).trim(); + (statusCode, responseBody) -> { + Assert.assertEquals(statusCode, 400, responseBody); assertStringCompare("Empty query", responseBody, responseBody::contains); } ); @@ -295,13 +257,11 @@ public void testEmptyQuery_UrlEncoded() { executeQuery( MediaType.APPLICATION_FORM_URLENCODED, + null, (request) -> { - // Empty query, DO NOTHING }, - (statusCode, responseEntity) -> { - assertEquals(400, statusCode); - - String responseBody = EntityUtils.toString(responseEntity).trim(); + (statusCode, responseBody) -> { + Assert.assertEquals(statusCode, 400, responseBody); assertStringCompare("Empty query", responseBody, responseBody::contains); } ); @@ -312,14 +272,11 @@ public void testBlankQuery_TextPlain() { executeQuery( MediaType.TEXT_PLAIN, + " ", (request) -> { - // an query with blank characters - request.setEntity(new StringEntity(" ")); }, - (statusCode, responseEntity) -> { - assertEquals(400, statusCode); - - String responseBody = EntityUtils.toString(responseEntity).trim(); + (statusCode, responseBody) -> { + Assert.assertEquals(statusCode, 400, responseBody); assertStringCompare("Empty query", responseBody, responseBody::contains); } ); @@ -330,36 +287,32 @@ public void testEmptyQuery_JSON() { executeQuery( MediaType.APPLICATION_JSON, + null, (request) -> { - // Empty query, DO NOTHING }, - (statusCode, responseEntity) -> { - assertEquals(400, statusCode); - - String responseBody = EntityUtils.toString(responseEntity).trim(); + (statusCode, responseBody) -> { + Assert.assertEquals(statusCode, 400, responseBody); assertStringCompare("Empty query", responseBody, responseBody::contains); } ); } /** - * When multiple Content-Type headers are set, the first one(in this case it's the text format) should be used. + * When multiple Content-Type headers are set, the first one (in this case, it's the text format) should be used. */ @Test public void testMultipleContentType() { executeQuery( MediaType.TEXT_PLAIN, + "SELECT 1", (request) -> { // Add one more Content-Type header request.addHeader("Content-Type", MediaType.APPLICATION_JSON); - request.setEntity(new StringEntity(StringUtils.format("SELECT 1"))); }, - (statusCode, responseEntity) -> { - assertEquals(200, statusCode); - - String responseBody = EntityUtils.toString(responseEntity).trim(); - assertEquals("[{\"EXPR$0\":1}]", responseBody); + (statusCode, responseBody) -> { + Assert.assertEquals(statusCode, 200, responseBody); + Assert.assertEquals(responseBody, "[{\"EXPR$0\":1}]"); } ); } diff --git a/services/pom.xml b/services/pom.xml index 570e48da1669..fad8933fb2c3 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -125,6 +125,10 @@ com.fasterxml.jackson.jaxrs jackson-jaxrs-smile-provider + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + com.opencsv opencsv diff --git a/services/src/main/java/org/apache/druid/server/AsyncQueryForwardingServlet.java b/services/src/main/java/org/apache/druid/server/AsyncQueryForwardingServlet.java index 5b8e9c17f393..dba842ec6896 100644 --- a/services/src/main/java/org/apache/druid/server/AsyncQueryForwardingServlet.java +++ b/services/src/main/java/org/apache/druid/server/AsyncQueryForwardingServlet.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.smile.SmileFactory; import com.fasterxml.jackson.jaxrs.smile.SmileMediaTypes; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; @@ -484,7 +485,7 @@ private void setProxyRequestContent(Request proxyRequest, HttpServletRequest cli byte[] bytes = objectMapper.writeValueAsBytes(content); proxyRequest.content(new BytesContentProvider(bytes)); proxyRequest.getHeaders().put(HttpHeader.CONTENT_LENGTH, String.valueOf(bytes.length)); - proxyRequest.getHeaders().put(HttpHeader.CONTENT_TYPE, MediaType.APPLICATION_JSON); + proxyRequest.getHeaders().put(HttpHeader.CONTENT_TYPE, objectMapper.getFactory() instanceof SmileFactory ? SmileMediaTypes.APPLICATION_JACKSON_SMILE : MediaType.APPLICATION_JSON); } catch (JsonProcessingException e) { throw new RuntimeException(e); diff --git a/sql/src/main/java/org/apache/druid/sql/http/SqlQuery.java b/sql/src/main/java/org/apache/druid/sql/http/SqlQuery.java index 71f9c14563ce..022a3c4ed54f 100644 --- a/sql/src/main/java/org/apache/druid/sql/http/SqlQuery.java +++ b/sql/src/main/java/org/apache/druid/sql/http/SqlQuery.java @@ -308,7 +308,7 @@ private static SqlQuery from( catch (IllegalArgumentException e) { throw new HttpException( Response.Status.BAD_REQUEST, - "Unable to decoded URL-Encoded SQL query: " + e.getMessage() + "Unable to decode URL-Encoded SQL query: " + e.getMessage() ); }