From 7f2b33cdcd97a09310407ed3580627cdedab49c1 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 1 Nov 2017 23:24:26 -0600 Subject: [PATCH 1/2] SQL: Add "array" result format, and document result formats. --- docs/content/querying/sql.md | 23 +++-- .../main/java/io/druid/sql/http/SqlQuery.java | 96 +++++++++++++++++-- .../java/io/druid/sql/http/SqlResource.java | 6 +- .../druid/sql/calcite/http/SqlQueryTest.java | 38 ++++++++ .../sql/calcite/http/SqlResourceTest.java | 65 +++++++++---- 5 files changed, 189 insertions(+), 39 deletions(-) create mode 100644 sql/src/test/java/io/druid/sql/calcite/http/SqlQueryTest.java diff --git a/docs/content/querying/sql.md b/docs/content/querying/sql.md index 8d6c5919b8b1..e08ea043cbb8 100644 --- a/docs/content/querying/sql.md +++ b/docs/content/querying/sql.md @@ -332,18 +332,25 @@ of configuration. ### JSON over HTTP You can make Druid SQL queries using JSON over HTTP by posting to the endpoint `/druid/v2/sql/`. The request should -be a JSON object with a "query" field, like `{"query" : "SELECT COUNT(*) FROM data_source WHERE foo = 'bar'"}`. You can -use _curl_ to send these queries from the command-line: +be a JSON object with a "query" field, like `{"query" : "SELECT COUNT(*) FROM data_source WHERE foo = 'bar'"}`. + +Results are available in two formats: "object" (the default; a JSON array of JSON objects), and "array" (a JSON array +of JSON arrays). In "object" form, each row's field names will match the column names from your SQL query. In "array" +form, each row's values are returned in the order specified in your SQL query. + +You can use _curl_ to send SQL queries from the command-line: ```bash $ cat query.json -{"query":"SELECT COUNT(*) FROM data_source"} +{"query":"SELECT COUNT(*) AS TheCount FROM data_source"} $ curl -XPOST -H'Content-Type: application/json' http://BROKER:8082/druid/v2/sql/ -d @query.json -[{"EXPR$0":24433}] +[{"TheCount":24433}] ``` -You can also provide [connection context parameters](#connection-context) by adding a "context" map, like: +Metadata is available over the HTTP API by querying the ["INFORMATION_SCHEMA" tables](#retrieving-metadata). + +Finally, you can also provide [connection context parameters](#connection-context) by adding a "context" map, like: ```json { @@ -354,8 +361,6 @@ You can also provide [connection context parameters](#connection-context) by add } ``` -Metadata is available over the HTTP API by querying the ["INFORMATION_SCHEMA" tables](#retrieving-metadata). - ### JDBC You can make Druid SQL queries using the [Avatica JDBC driver](https://calcite.apache.org/avatica/downloads/). Once @@ -404,6 +409,8 @@ Druid SQL supports setting connection parameters on the client. The parameters i All other context parameters you provide will be attached to Druid queries and can affect how they run. See [Query context](query-context.html) for details on the possible options. +Connection context can be specified as JDBC connection properties or as a "context" object in the JSON API. + |Parameter|Description|Default value| |---------|-----------|-------------| |`sqlTimeZone`|Sets the time zone for this connection, which will affect how time functions and timestamp literals behave. Should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|UTC| @@ -411,8 +418,6 @@ All other context parameters you provide will be attached to Druid queries and c |`useApproximateTopN`|Whether to use approximate [TopN queries](topnquery.html) when a SQL query could be expressed as such. If false, exact [GroupBy queries](groupbyquery.html) will be used instead.|druid.sql.planner.useApproximateTopN on the broker| |`useFallback`|Whether to evaluate operations on the broker when they cannot be expressed as Druid queries. This option is not recommended for production since it can generate unscalable query plans. If false, SQL queries that cannot be translated to Druid queries will fail.|druid.sql.planner.useFallback on the broker| -Connection context can be specified as JDBC connection properties or as a "context" object in the JSON API. - ### Retrieving metadata Druid brokers infer table and column metadata for each dataSource from segments loaded in the cluster, and use this to diff --git a/sql/src/main/java/io/druid/sql/http/SqlQuery.java b/sql/src/main/java/io/druid/sql/http/SqlQuery.java index 5c1c0a87737d..db0986dc364f 100644 --- a/sql/src/main/java/io/druid/sql/http/SqlQuery.java +++ b/sql/src/main/java/io/druid/sql/http/SqlQuery.java @@ -21,23 +21,99 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonGenerator; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import io.druid.java.util.common.StringUtils; +import javax.annotation.Nullable; +import java.io.IOException; import java.util.Map; +import java.util.Objects; public class SqlQuery { + public enum ResultFormat + { + ARRAY { + @Override + public void writeResultStart(final JsonGenerator jsonGenerator) throws IOException + { + jsonGenerator.writeStartArray(); + } + + @Override + public void writeResultField( + final JsonGenerator jsonGenerator, + final String name, + final Object value + ) throws IOException + { + jsonGenerator.writeObject(value); + } + + @Override + public void writeResultEnd(final JsonGenerator jsonGenerator) throws IOException + { + jsonGenerator.writeEndArray(); + } + }, + + OBJECT { + @Override + public void writeResultStart(final JsonGenerator jsonGenerator) throws IOException + { + jsonGenerator.writeStartObject(); + } + + @Override + public void writeResultField( + final JsonGenerator jsonGenerator, + final String name, + final Object value + ) throws IOException + { + jsonGenerator.writeFieldName(name); + jsonGenerator.writeObject(value); + } + + @Override + public void writeResultEnd(final JsonGenerator jsonGenerator) throws IOException + { + jsonGenerator.writeEndObject(); + } + }; + + public abstract void writeResultStart(final JsonGenerator jsonGenerator) throws IOException; + + public abstract void writeResultField(final JsonGenerator jsonGenerator, final String name, final Object value) + throws IOException; + + public abstract void writeResultEnd(final JsonGenerator jsonGenerator) throws IOException; + + @JsonCreator + public static ResultFormat fromString(@Nullable final String name) + { + if (name == null) { + return null; + } + return valueOf(StringUtils.toUpperCase(name)); + } + } + private final String query; + private final ResultFormat resultFormat; private final Map context; @JsonCreator public SqlQuery( @JsonProperty("query") final String query, + @JsonProperty("resultFormat") final ResultFormat resultFormat, @JsonProperty("context") final Map context ) { this.query = Preconditions.checkNotNull(query, "query"); + this.resultFormat = resultFormat == null ? ResultFormat.OBJECT : resultFormat; this.context = context == null ? ImmutableMap.of() : context; } @@ -47,6 +123,12 @@ public String getQuery() return query; } + @JsonProperty + public ResultFormat getResultFormat() + { + return resultFormat; + } + @JsonProperty public Map getContext() { @@ -62,21 +144,16 @@ public boolean equals(final Object o) if (o == null || getClass() != o.getClass()) { return false; } - final SqlQuery sqlQuery = (SqlQuery) o; - - if (query != null ? !query.equals(sqlQuery.query) : sqlQuery.query != null) { - return false; - } - return context != null ? context.equals(sqlQuery.context) : sqlQuery.context == null; + return Objects.equals(query, sqlQuery.query) && + resultFormat == sqlQuery.resultFormat && + Objects.equals(context, sqlQuery.context); } @Override public int hashCode() { - int result = query != null ? query.hashCode() : 0; - result = 31 * result + (context != null ? context.hashCode() : 0); - return result; + return Objects.hash(query, resultFormat, context); } @Override @@ -84,6 +161,7 @@ public String toString() { return "SqlQuery{" + "query='" + query + '\'' + + ", resultFormat=" + resultFormat + ", context=" + context + '}'; } diff --git a/sql/src/main/java/io/druid/sql/http/SqlResource.java b/sql/src/main/java/io/druid/sql/http/SqlResource.java index 8cc1d1f4d8d0..dd73cd44c375 100644 --- a/sql/src/main/java/io/druid/sql/http/SqlResource.java +++ b/sql/src/main/java/io/druid/sql/http/SqlResource.java @@ -114,7 +114,7 @@ public void write(final OutputStream outputStream) throws IOException, WebApplic while (!yielder.isDone()) { final Object[] row = yielder.get(); - jsonGenerator.writeStartObject(); + sqlQuery.getResultFormat().writeResultStart(jsonGenerator); for (int i = 0; i < fieldList.size(); i++) { final Object value; @@ -130,9 +130,9 @@ public void write(final OutputStream outputStream) throws IOException, WebApplic value = row[i]; } - jsonGenerator.writeObjectField(fieldList.get(i).getName(), value); + sqlQuery.getResultFormat().writeResultField(jsonGenerator, fieldList.get(i).getName(), value); } - jsonGenerator.writeEndObject(); + sqlQuery.getResultFormat().writeResultEnd(jsonGenerator); yielder = yielder.next(null); } diff --git a/sql/src/test/java/io/druid/sql/calcite/http/SqlQueryTest.java b/sql/src/test/java/io/druid/sql/calcite/http/SqlQueryTest.java new file mode 100644 index 000000000000..0445f54db102 --- /dev/null +++ b/sql/src/test/java/io/druid/sql/calcite/http/SqlQueryTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.sql.calcite.http; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import io.druid.segment.TestHelper; +import io.druid.sql.http.SqlQuery; +import org.junit.Assert; +import org.junit.Test; + +public class SqlQueryTest +{ + @Test + public void testSerde() throws Exception + { + final ObjectMapper jsonMapper = TestHelper.getJsonMapper(); + final SqlQuery query = new SqlQuery("SELECT 1", SqlQuery.ResultFormat.ARRAY, ImmutableMap.of("useCache", false)); + Assert.assertEquals(query, jsonMapper.readValue(jsonMapper.writeValueAsString(query), SqlQuery.class)); + } +} diff --git a/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java b/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java index 31db7ca651e3..aa3fa3ea1261 100644 --- a/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java +++ b/sql/src/test/java/io/druid/sql/calcite/http/SqlResourceTest.java @@ -126,12 +126,28 @@ public void tearDown() throws Exception public void testCountStar() throws Exception { final List> rows = doPost( - new SqlQuery("SELECT COUNT(*) AS cnt FROM druid.foo", null) + new SqlQuery("SELECT COUNT(*) AS cnt, 'foo' AS TheFoo FROM druid.foo", null, null) ).rhs; Assert.assertEquals( ImmutableList.of( - ImmutableMap.of("cnt", 6) + ImmutableMap.of("cnt", 6, "TheFoo", "foo") + ), + rows + ); + } + + @Test + public void testCountStarAsArray() throws Exception + { + final List> rows = doPost( + new SqlQuery("SELECT COUNT(*), 'foo' FROM druid.foo", SqlQuery.ResultFormat.ARRAY, null), + new TypeReference>>() {} + ).rhs; + + Assert.assertEquals( + ImmutableList.of( + ImmutableList.of(6, "foo") ), rows ); @@ -141,7 +157,11 @@ public void testCountStar() throws Exception public void testTimestampsInResponse() throws Exception { final List> rows = doPost( - new SqlQuery("SELECT __time, CAST(__time AS DATE) AS t2 FROM druid.foo LIMIT 1", null) + new SqlQuery( + "SELECT __time, CAST(__time AS DATE) AS t2 FROM druid.foo LIMIT 1", + SqlQuery.ResultFormat.OBJECT, + null + ) ).rhs; Assert.assertEquals( @@ -158,6 +178,7 @@ public void testTimestampsInResponseLosAngelesTimeZone() throws Exception final List> rows = doPost( new SqlQuery( "SELECT __time, CAST(__time AS DATE) AS t2 FROM druid.foo LIMIT 1", + SqlQuery.ResultFormat.OBJECT, ImmutableMap.of(PlannerContext.CTX_SQL_TIME_ZONE, "America/Los_Angeles") ) ).rhs; @@ -174,7 +195,7 @@ public void testTimestampsInResponseLosAngelesTimeZone() throws Exception public void testFieldAliasingSelect() throws Exception { final List> rows = doPost( - new SqlQuery("SELECT dim2 \"x\", dim2 \"y\" FROM druid.foo LIMIT 1", null) + new SqlQuery("SELECT dim2 \"x\", dim2 \"y\" FROM druid.foo LIMIT 1", SqlQuery.ResultFormat.OBJECT, null) ).rhs; Assert.assertEquals( @@ -189,7 +210,7 @@ public void testFieldAliasingSelect() throws Exception public void testFieldAliasingGroupBy() throws Exception { final List> rows = doPost( - new SqlQuery("SELECT dim2 \"x\", dim2 \"y\" FROM druid.foo GROUP BY dim2", null) + new SqlQuery("SELECT dim2 \"x\", dim2 \"y\" FROM druid.foo GROUP BY dim2", SqlQuery.ResultFormat.OBJECT, null) ).rhs; Assert.assertEquals( @@ -206,7 +227,7 @@ public void testFieldAliasingGroupBy() throws Exception public void testExplainCountStar() throws Exception { final List> rows = doPost( - new SqlQuery("EXPLAIN PLAN FOR SELECT COUNT(*) AS cnt FROM druid.foo", null) + new SqlQuery("EXPLAIN PLAN FOR SELECT COUNT(*) AS cnt FROM druid.foo", SqlQuery.ResultFormat.OBJECT, null) ).rhs; Assert.assertEquals( @@ -223,7 +244,13 @@ public void testExplainCountStar() throws Exception @Test public void testCannotValidate() throws Exception { - final QueryInterruptedException exception = doPost(new SqlQuery("SELECT dim3 FROM druid.foo", null)).lhs; + final QueryInterruptedException exception = doPost( + new SqlQuery( + "SELECT dim3 FROM druid.foo", + SqlQuery.ResultFormat.OBJECT, + null + ) + ).lhs; Assert.assertNotNull(exception); Assert.assertEquals(QueryInterruptedException.UNKNOWN_EXCEPTION, exception.getErrorCode()); @@ -236,7 +263,7 @@ public void testCannotConvert() throws Exception { // SELECT + ORDER unsupported final QueryInterruptedException exception = doPost( - new SqlQuery("SELECT dim1 FROM druid.foo ORDER BY dim1", null) + new SqlQuery("SELECT dim1 FROM druid.foo ORDER BY dim1", SqlQuery.ResultFormat.OBJECT, null) ).lhs; Assert.assertNotNull(exception); @@ -254,9 +281,8 @@ public void testResourceLimitExceeded() throws Exception final QueryInterruptedException exception = doPost( new SqlQuery( "SELECT DISTINCT dim1 FROM foo", - ImmutableMap.of( - "maxMergingDictionarySize", 1 - ) + SqlQuery.ResultFormat.OBJECT, + ImmutableMap.of("maxMergingDictionarySize", 1) ) ).lhs; @@ -266,7 +292,10 @@ public void testResourceLimitExceeded() throws Exception } // Returns either an error or a result. - private Pair>> doPost(final SqlQuery query) throws Exception + private Pair doPost( + final SqlQuery query, + final TypeReference typeReference + ) throws Exception { final Response response = resource.doPost(query, req); if (response.getStatus() == 200) { @@ -275,12 +304,7 @@ private Pair>> doPost(final output.write(baos); return Pair.of( null, - JSON_MAPPER.>>readValue( - baos.toByteArray(), - new TypeReference>>() - { - } - ) + JSON_MAPPER.readValue(baos.toByteArray(), typeReference) ); } else { return Pair.of( @@ -289,4 +313,9 @@ private Pair>> doPost(final ); } } + + private Pair>> doPost(final SqlQuery query) throws Exception + { + return doPost(query, new TypeReference>>() {}); + } } From 7674fce823b947e796e015858291e48e6b3db4b1 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Thu, 2 Nov 2017 01:10:25 -0600 Subject: [PATCH 2/2] Code style. --- sql/src/main/java/io/druid/sql/http/SqlQuery.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/src/main/java/io/druid/sql/http/SqlQuery.java b/sql/src/main/java/io/druid/sql/http/SqlQuery.java index db0986dc364f..ce8c377dafd2 100644 --- a/sql/src/main/java/io/druid/sql/http/SqlQuery.java +++ b/sql/src/main/java/io/druid/sql/http/SqlQuery.java @@ -84,12 +84,12 @@ public void writeResultEnd(final JsonGenerator jsonGenerator) throws IOException } }; - public abstract void writeResultStart(final JsonGenerator jsonGenerator) throws IOException; + public abstract void writeResultStart(JsonGenerator jsonGenerator) throws IOException; - public abstract void writeResultField(final JsonGenerator jsonGenerator, final String name, final Object value) + public abstract void writeResultField(JsonGenerator jsonGenerator, String name, Object value) throws IOException; - public abstract void writeResultEnd(final JsonGenerator jsonGenerator) throws IOException; + public abstract void writeResultEnd(JsonGenerator jsonGenerator) throws IOException; @JsonCreator public static ResultFormat fromString(@Nullable final String name)