Skip to content

Align ScanQuery column order with its desired signature #17457

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kgyrtkirk wants to merge 22 commits into from
Closed

Align ScanQuery column order with its desired signature #17457

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
2ee7005
fix and update1
kgyrtkirk Nov 6, 2024
44c098a
column order changes
kgyrtkirk Nov 6, 2024
579a848
sig changes
kgyrtkirk Nov 6, 2024
c96ef1a
column order in joinqueytest
kgyrtkirk Nov 6, 2024
fbc1bb5
column order in calcitequerytest
kgyrtkirk Nov 6, 2024
9a963c6
fix subq
kgyrtkirk Nov 6, 2024
ce123f8
fix a few more
kgyrtkirk Nov 6, 2024
a68f470
up
kgyrtkirk Nov 6, 2024
cb35cf6
updates
kgyrtkirk Nov 6, 2024
d0b25e5
up
kgyrtkirk Nov 6, 2024
ac0f5ea
fix
kgyrtkirk Nov 6, 2024
ad14eab
update
kgyrtkirk Nov 6, 2024
e67ff79
column order changes
kgyrtkirk Nov 6, 2024
387f0e7
order in join.iq
kgyrtkirk Nov 6, 2024
35c3772
fix test
kgyrtkirk Nov 7, 2024
2711a4f
update test
kgyrtkirk Nov 7, 2024
6ce1c0c
cleanupo/fix/etc
kgyrtkirk Nov 7, 2024
b8720b6
fix
kgyrtkirk Nov 7, 2024
63f0a43
make msqa pass
kgyrtkirk Nov 7, 2024
b4037a5
fix
kgyrtkirk Nov 7, 2024
33d57a6
fix onemore
kgyrtkirk Nov 7, 2024
f417893
update comment
kgyrtkirk Nov 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...he/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -398,7 +398,7 @@ public void testSubqueryWithNestedGroupBy()
new ExpressionVirtualColumn("v0", "946684800000", ColumnType.LONG, TestExprMacroTable.INSTANCE),
new ExpressionVirtualColumn("v1", "case_searched((\"j0.a0\" < \"_a0\"),'val2',((\"j0.a0\" >= \"_a0\") && (\"j0.a0\" < \"_a1\")),'val3',(\"j0.a0\" >= \"_a1\"),'val1',null)", ColumnType.STRING, TestExprMacroTable.INSTANCE)
)
.columns("j0.a0", "j0.d0", "v0", "v1")
.columns("v0", "j0.d0", "j0.a0", "v1")
.build()
),
expectedResults
Expand Down
20 changes: 6 additions & 14 deletions extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQArraysTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -758,13 +758,13 @@ public void testSelectOnArrays(String contextName, Map<String, Object> context,
.build();

RowSignature scanSignature = RowSignature.builder()
.add("arrayDouble", ColumnType.DOUBLE_ARRAY)
.add("arrayDoubleNulls", ColumnType.DOUBLE_ARRAY)
.add("arrayLong", ColumnType.LONG_ARRAY)
.add("arrayLongNulls", ColumnType.LONG_ARRAY)
.add("v0", ColumnType.LONG)
.add("arrayString", ColumnType.STRING_ARRAY)
.add("arrayStringNulls", ColumnType.STRING_ARRAY)
.add("v0", ColumnType.LONG)
.add("arrayLong", ColumnType.LONG_ARRAY)
.add("arrayLongNulls", ColumnType.LONG_ARRAY)
.add("arrayDouble", ColumnType.DOUBLE_ARRAY)
.add("arrayDoubleNulls", ColumnType.DOUBLE_ARRAY)
.build();

final Map<String, Object> adjustedContext = new HashMap<>(context);
Expand All @@ -773,15 +773,7 @@ public void testSelectOnArrays(String contextName, Map<String, Object> context,
Query<?> expectedQuery = newScanQueryBuilder()
.dataSource(dataFileExternalDataSource)
.intervals(querySegmentSpec(Filtration.eternity()))
.columns(
"arrayDouble",
"arrayDoubleNulls",
"arrayLong",
"arrayLongNulls",
"arrayString",
"arrayStringNulls",
"v0"
)
.columns(scanSignature.getColumnNames())
.virtualColumns(new ExpressionVirtualColumn(
"v0",
"timestamp_parse(\"timestamp\",null,'UTC')",
Expand Down
6 changes: 4 additions & 2 deletions ...ore/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQParseExceptionsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,7 @@ public void testIngestWithSanitizedNullByte() throws IOException
)
)
.columns("v0", "v1")
.columnTypes(ColumnType.LONG, ColumnType.STRING)
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.context(defaultScanQueryContext(
context,
Expand Down Expand Up @@ -272,13 +273,14 @@ public void testIngestWithSanitizedNullByteUsingContextParameter() throws IOExce
ColumnType.LONG
)
)
.columns("agent_category", "v0")
.columns("v0", "agent_category")
.columnTypes(ColumnType.LONG, ColumnType.STRING)
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.context(defaultScanQueryContext(
context,
RowSignature.builder()
.add("agent_category", ColumnType.STRING)
.add("v0", ColumnType.LONG)
.add("agent_category", ColumnType.STRING)
.build()
))
.build();
Expand Down
29 changes: 14 additions & 15 deletions extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,12 +254,12 @@ public void testSelectOnFoo2(String contextName, Map<String, Object> context)
.query(newScanQueryBuilder()
.dataSource(CalciteTests.DATASOURCE2)
.intervals(querySegmentSpec(Filtration.eternity()))
.columns("dim2", "m1")
.columns("m1", "dim2")
.context(defaultScanQueryContext(
context,
RowSignature.builder()
.add("dim2", ColumnType.STRING)
.add("m1", ColumnType.LONG)
.add("dim2", ColumnType.STRING)
.build()
))
.build())
Expand Down Expand Up @@ -1475,17 +1475,16 @@ public void testExternSelectWithMultipleWorkers(String contextName, Map<String,
ColumnType.LONG,
CalciteTests.createExprMacroTable()
)
).columns("user", "v0").filters(new LikeDimFilter("user", "%ot%", null, null))
.context(defaultScanQueryContext(multipleWorkerContext, RowSignature.builder()
.add(
"user",
ColumnType.STRING
)
.add(
"v0",
ColumnType.LONG
)
.build()))
)
.columns("v0", "user")
.filters(new LikeDimFilter("user", "%ot%", null, null))
.context(defaultScanQueryContext(
multipleWorkerContext,
RowSignature.builder()
.add("v0", ColumnType.LONG)
.add("user", ColumnType.STRING)
.build()
))
.build();

SelectTester selectTester = testSelectQuery()
Expand Down Expand Up @@ -1670,12 +1669,12 @@ public void testSelectOnUserDefinedSourceContainingWith(String contextName, Map<
newScanQueryBuilder()
.dataSource(CalciteTests.DATASOURCE2)
.intervals(querySegmentSpec(Filtration.eternity()))
.columns("dim2", "m1")
.columns("m1", "dim2")
.context(defaultScanQueryContext(
context,
RowSignature.builder()
.add("dim2", ColumnType.STRING)
.add("m1", ColumnType.LONG)
.add("dim2", ColumnType.STRING)
.build()
))
.build()
Expand Down
28 changes: 14 additions & 14 deletions extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQWindowTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -987,7 +987,7 @@ public void testWindowOnFooWithPartitionByOrderBYWithJoin()
.putAll(DEFAULT_MSQ_CONTEXT)
.put(
DruidQuery.CTX_SCAN_SIGNATURE,
"[{\"name\":\"j0.m2\",\"type\":\"DOUBLE\"},{\"name\":\"m1\",\"type\":\"FLOAT\"}]"
"[{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"j0.m2\",\"type\":\"DOUBLE\"}]"
)
.build();

Expand Down Expand Up @@ -1037,7 +1037,7 @@ public void testWindowOnFooWithPartitionByOrderBYWithJoin()
)
)
.intervals(querySegmentSpec(Filtration.eternity()))
.columns("j0.m2", "m1")
.columns("m1", "j0.m2")
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.context(contextWithRowSignature)
.build()),
Expand Down Expand Up @@ -1090,7 +1090,7 @@ public void testWindowOnFooWithEmptyOverWithJoin()
.putAll(DEFAULT_MSQ_CONTEXT)
.put(
DruidQuery.CTX_SCAN_SIGNATURE,
"[{\"name\":\"j0.m2\",\"type\":\"DOUBLE\"},{\"name\":\"m1\",\"type\":\"FLOAT\"}]"
"[{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"j0.m2\",\"type\":\"DOUBLE\"}]"
)
.build();

Expand Down Expand Up @@ -1140,7 +1140,7 @@ public void testWindowOnFooWithEmptyOverWithJoin()
)
)
.intervals(querySegmentSpec(Filtration.eternity()))
.columns("j0.m2", "m1")
.columns("m1", "j0.m2")
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.context(contextWithRowSignature)
.build()),
Expand Down Expand Up @@ -1269,7 +1269,7 @@ public void testWindowOnFooWithEmptyOverWithUnnest()
.putAll(DEFAULT_MSQ_CONTEXT)
.put(
DruidQuery.CTX_SCAN_SIGNATURE,
"[{\"name\":\"j0.unnest\",\"type\":\"STRING\"},{\"name\":\"m1\",\"type\":\"FLOAT\"}]"
"[{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"j0.unnest\",\"type\":\"STRING\"}]"
)
.build();

Expand Down Expand Up @@ -1297,7 +1297,7 @@ public void testWindowOnFooWithEmptyOverWithUnnest()
)
)
.intervals(querySegmentSpec(Filtration.eternity()))
.columns("j0.unnest", "m1")
.columns("m1", "j0.unnest")
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.context(contextWithRowSignature)
.build()),
Expand Down Expand Up @@ -1351,7 +1351,7 @@ public void testWindowOnFooWithPartitionByAndWithUnnest()
.putAll(DEFAULT_MSQ_CONTEXT)
.put(
DruidQuery.CTX_SCAN_SIGNATURE,
"[{\"name\":\"j0.unnest\",\"type\":\"STRING\"},{\"name\":\"m1\",\"type\":\"FLOAT\"}]"
"[{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"j0.unnest\",\"type\":\"STRING\"}]"
)
.build();

Expand Down Expand Up @@ -1379,7 +1379,7 @@ public void testWindowOnFooWithPartitionByAndWithUnnest()
)
)
.intervals(querySegmentSpec(Filtration.eternity()))
.columns("j0.unnest", "m1")
.columns("m1", "j0.unnest")
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.context(contextWithRowSignature)
.build()),
Expand Down Expand Up @@ -1740,7 +1740,7 @@ public void testSelectWithWikipedia()
.putAll(DEFAULT_MSQ_CONTEXT)
.put(
DruidQuery.CTX_SCAN_SIGNATURE,
"[{\"name\":\"added\",\"type\":\"LONG\"},{\"name\":\"cityName\",\"type\":\"STRING\"}]"
"[{\"name\":\"cityName\",\"type\":\"STRING\"},{\"name\":\"added\",\"type\":\"LONG\"}]"
)
.build();

Expand All @@ -1750,7 +1750,7 @@ public void testSelectWithWikipedia()
.dataSource(CalciteTests.WIKIPEDIA)
.intervals(querySegmentSpec(Filtration.eternity()))
.filters(in("cityName", ImmutableList.of("Ahmedabad", "Albuquerque")))
.columns("added", "cityName")
.columns("cityName", "added")
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.context(contextWithRowSignature)
.build()),
Expand Down Expand Up @@ -1829,7 +1829,7 @@ public void testSelectWithWikipediaWithPartitionKeyNotInSelect()
.putAll(DEFAULT_MSQ_CONTEXT)
.put(
DruidQuery.CTX_SCAN_SIGNATURE,
"[{\"name\":\"added\",\"type\":\"LONG\"},{\"name\":\"cityName\",\"type\":\"STRING\"},{\"name\":\"countryIsoCode\",\"type\":\"STRING\"}]"
"[{\"name\":\"cityName\",\"type\":\"STRING\"},{\"name\":\"countryIsoCode\",\"type\":\"STRING\"},{\"name\":\"added\",\"type\":\"LONG\"}]"
)
.build();

Expand All @@ -1839,7 +1839,7 @@ public void testSelectWithWikipediaWithPartitionKeyNotInSelect()
.dataSource(CalciteTests.WIKIPEDIA)
.intervals(querySegmentSpec(Filtration.eternity()))
.filters(notNull("cityName"))
.columns("added", "cityName", "countryIsoCode")
.columns("cityName", "countryIsoCode", "added")
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.context(innerContextWithRowSignature)
.build()),
Expand All @@ -1861,13 +1861,13 @@ public void testSelectWithWikipediaWithPartitionKeyNotInSelect()
.putAll(DEFAULT_MSQ_CONTEXT)
.put(
DruidQuery.CTX_SCAN_SIGNATURE,
"[{\"name\":\"added\",\"type\":\"LONG\"},{\"name\":\"cityName\",\"type\":\"STRING\"},{\"name\":\"w0\",\"type\":\"LONG\"}]"
"[{\"name\":\"cityName\",\"type\":\"STRING\"},{\"name\":\"added\",\"type\":\"LONG\"},{\"name\":\"w0\",\"type\":\"LONG\"}]"
)
.build();
final Query scanQuery = Druids.newScanQueryBuilder()
.dataSource(new QueryDataSource(query))
.intervals(querySegmentSpec(Filtration.eternity()))
.columns("added", "cityName", "w0")
.columns("cityName", "added", "w0")
.limit(5)
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.context(outerContextWithRowSignature)
Expand Down
2 changes: 1 addition & 1 deletion ...ery/src/test/java/org/apache/druid/msq/sql/resources/SqlMSQStatementResourcePostTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ public void testExplain() throws IOException
), SqlStatementResourceTest.makeOkRequest());

Assert.assertEquals(
"{PLAN=[{\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"__time\",\"cnt\",\"dim1\",\"dim2\",\"dim3\",\"m1\",\"m2\",\"unique_dim1\"],\"context\":{\"__resultFormat\":\"object\",\"executionMode\":\"ASYNC\",\"scanSignature\":\"[{\\\"name\\\":\\\"__time\\\",\\\"type\\\":\\\"LONG\\\"},{\\\"name\\\":\\\"cnt\\\",\\\"type\\\":\\\"LONG\\\"},{\\\"name\\\":\\\"dim1\\\",\\\"type\\\":\\\"STRING\\\"},{\\\"name\\\":\\\"dim2\\\",\\\"type\\\":\\\"STRING\\\"},{\\\"name\\\":\\\"dim3\\\",\\\"type\\\":\\\"STRING\\\"},{\\\"name\\\":\\\"m1\\\",\\\"type\\\":\\\"FLOAT\\\"},{\\\"name\\\":\\\"m2\\\",\\\"type\\\":\\\"DOUBLE\\\"},{\\\"name\\\":\\\"unique_dim1\\\",\\\"type\\\":\\\"COMPLEX<hyperUnique>\\\"}]\",\"sqlQueryId\":\"queryId\"},\"columnTypes\":[\"LONG\",\"LONG\",\"STRING\",\"STRING\",\"STRING\",\"FLOAT\",\"DOUBLE\",\"COMPLEX<hyperUnique>\"],\"granularity\":{\"type\":\"all\"},\"legacy\":false},\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX<hyperUnique>\"}],\"columnMappings\":[{\"queryColumn\":\"__time\",\"outputColumn\":\"__time\"},{\"queryColumn\":\"dim1\",\"outputColumn\":\"dim1\"},{\"queryColumn\":\"dim2\",\"outputColumn\":\"dim2\"},{\"queryColumn\":\"dim3\",\"outputColumn\":\"dim3\"},{\"queryColumn\":\"cnt\",\"outputColumn\":\"cnt\"},{\"queryColumn\":\"m1\",\"outputColumn\":\"m1\"},{\"queryColumn\":\"m2\",\"outputColumn\":\"m2\"},{\"queryColumn\":\"unique_dim1\",\"outputColumn\":\"unique_dim1\"}]}], RESOURCES=[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}], ATTRIBUTES={\"statementType\":\"SELECT\"}}",
"{PLAN=[{\"query\":{\"queryType\":\"scan\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"resultFormat\":\"compactedList\",\"columns\":[\"__time\",\"dim1\",\"dim2\",\"dim3\",\"cnt\",\"m1\",\"m2\",\"unique_dim1\"],\"context\":{\"__resultFormat\":\"object\",\"executionMode\":\"ASYNC\",\"scanSignature\":\"[{\\\"name\\\":\\\"__time\\\",\\\"type\\\":\\\"LONG\\\"},{\\\"name\\\":\\\"dim1\\\",\\\"type\\\":\\\"STRING\\\"},{\\\"name\\\":\\\"dim2\\\",\\\"type\\\":\\\"STRING\\\"},{\\\"name\\\":\\\"dim3\\\",\\\"type\\\":\\\"STRING\\\"},{\\\"name\\\":\\\"cnt\\\",\\\"type\\\":\\\"LONG\\\"},{\\\"name\\\":\\\"m1\\\",\\\"type\\\":\\\"FLOAT\\\"},{\\\"name\\\":\\\"m2\\\",\\\"type\\\":\\\"DOUBLE\\\"},{\\\"name\\\":\\\"unique_dim1\\\",\\\"type\\\":\\\"COMPLEX<hyperUnique>\\\"}]\",\"sqlQueryId\":\"queryId\"},\"columnTypes\":[\"LONG\",\"STRING\",\"STRING\",\"STRING\",\"LONG\",\"FLOAT\",\"DOUBLE\",\"COMPLEX<hyperUnique>\"],\"granularity\":{\"type\":\"all\"},\"legacy\":false},\"signature\":[{\"name\":\"__time\",\"type\":\"LONG\"},{\"name\":\"dim1\",\"type\":\"STRING\"},{\"name\":\"dim2\",\"type\":\"STRING\"},{\"name\":\"dim3\",\"type\":\"STRING\"},{\"name\":\"cnt\",\"type\":\"LONG\"},{\"name\":\"m1\",\"type\":\"FLOAT\"},{\"name\":\"m2\",\"type\":\"DOUBLE\"},{\"name\":\"unique_dim1\",\"type\":\"COMPLEX<hyperUnique>\"}],\"columnMappings\":[{\"queryColumn\":\"__time\",\"outputColumn\":\"__time\"},{\"queryColumn\":\"dim1\",\"outputColumn\":\"dim1\"},{\"queryColumn\":\"dim2\",\"outputColumn\":\"dim2\"},{\"queryColumn\":\"dim3\",\"outputColumn\":\"dim3\"},{\"queryColumn\":\"cnt\",\"outputColumn\":\"cnt\"},{\"queryColumn\":\"m1\",\"outputColumn\":\"m1\"},{\"queryColumn\":\"m2\",\"outputColumn\":\"m2\"},{\"queryColumn\":\"unique_dim1\",\"outputColumn\":\"unique_dim1\"}]}], RESOURCES=[{\"name\":\"foo\",\"type\":\"DATASOURCE\"}], ATTRIBUTES={\"statementType\":\"SELECT\"}}",
String.valueOf(SqlStatementResourceTest.getResultRowsFromResponse(response).get(0))
);
}
Expand Down
3 changes: 2 additions & 1 deletion processing/src/test/java/org/apache/druid/query/scan/ScanQueryLimitRowIteratorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.apache.druid.query.QueryPlus;
import org.apache.druid.query.QueryRunnerTestHelper;
import org.apache.druid.query.context.ResponseContext;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
Expand All @@ -39,7 +40,7 @@
import java.util.concurrent.ThreadLocalRandom;

@RunWith(Parameterized.class)
public class ScanQueryLimitRowIteratorTest
public class ScanQueryLimitRowIteratorTest extends InitializedNullHandlingTest
{
private static final int NUM_ELEMENTS = 1000;
private final int batchSize;
Expand Down
Loading