diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java index c83481757d8f..26d566b31e8c 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQSelectTest.java @@ -31,6 +31,7 @@ import org.apache.druid.msq.indexing.ColumnMappings; import org.apache.druid.msq.indexing.MSQSpec; import org.apache.druid.msq.indexing.MSQTuningConfig; +import org.apache.druid.msq.indexing.error.CannotParseExternalDataFault; import org.apache.druid.msq.shuffle.DurableStorageUtils; import org.apache.druid.msq.test.MSQTestBase; import org.apache.druid.query.InlineDataSource; @@ -1253,6 +1254,63 @@ public void testGroupByOnFooWithDurableStoragePathAssertions() throws IOExceptio .write(ArgumentMatchers.endsWith("__success")); } + @Test + public void testMultiValueStringWithIncorrectType() throws IOException + { + final File toRead = getResourceAsTemporaryFile("/unparseable-mv-string-array.json"); + final String toReadAsJson = queryFramework().queryJsonMapper().writeValueAsString(toRead.getAbsolutePath()); + + RowSignature rowSignature = RowSignature.builder() + .add("__time", ColumnType.LONG) + .add("language", ColumnType.STRING_ARRAY) + .build(); + + final GroupByQuery expectedQuery = + GroupByQuery.builder() + .setDataSource(CalciteTests.DATASOURCE1) + .setInterval(querySegmentSpec(Filtration.eternity())) + .setGranularity(Granularities.ALL) + .setDimensions(dimensions(new DefaultDimensionSpec("__time", "d0", ColumnType.LONG))) + .build(); + + + testSelectQuery() + .setSql("WITH\n" + + "kttm_data AS (\n" + + "SELECT * FROM TABLE(\n" + + " EXTERN(\n" + + " '{ \"files\": [" + toReadAsJson + "],\"type\":\"local\"}',\n" + + " '{\"type\":\"json\"}',\n" + + " '[{\"name\":\"timestamp\",\"type\":\"string\"},{\"name\":\"agent_category\",\"type\":\"string\"},{\"name\":\"agent_type\",\"type\":\"string\"},{\"name\":\"browser\",\"type\":\"string\"},{\"name\":\"browser_version\",\"type\":\"string\"},{\"name\":\"city\",\"type\":\"string\"},{\"name\":\"continent\",\"type\":\"string\"},{\"name\":\"country\",\"type\":\"string\"},{\"name\":\"version\",\"type\":\"string\"},{\"name\":\"event_type\",\"type\":\"string\"},{\"name\":\"event_subtype\",\"type\":\"string\"},{\"name\":\"loaded_image\",\"type\":\"string\"},{\"name\":\"adblock_list\",\"type\":\"string\"},{\"name\":\"forwarded_for\",\"type\":\"string\"},{\"name\":\"language\",\"type\":\"string\"},{\"name\":\"number\",\"type\":\"long\"},{\"name\":\"os\",\"type\":\"string\"},{\"name\":\"path\",\"type\":\"string\"},{\"name\":\"platform\",\"type\":\"string\"},{\"name\":\"referrer\",\"type\":\"string\"},{\"name\":\"referrer_host\",\"type\":\"string\"},{\"name\":\"region\",\"type\":\"string\"},{\"name\":\"remote_address\",\"type\":\"string\"},{\"name\":\"screen\",\"type\":\"string\"},{\"name\":\"session\",\"type\":\"string\"},{\"name\":\"session_length\",\"type\":\"long\"},{\"name\":\"timezone\",\"type\":\"string\"},{\"name\":\"timezone_offset\",\"type\":\"long\"},{\"name\":\"window\",\"type\":\"string\"}]'\n" + + " )\n" + + "))\n" + + "\n" + + "SELECT\n" + + " FLOOR(TIME_PARSE(\"timestamp\") TO MINUTE) AS __time,\n" + + " MV_TO_ARRAY(\"language\") AS \"language\"\n" + + "FROM kttm_data") + .setExpectedRowSignature(rowSignature) + .setExpectedResultRows(ImmutableList.of( + new Object[]{1566691200000L, ImmutableList.of("en")}, + new Object[]{1566691200000L, ImmutableList.of("en", "es", "es-419", "es-MX")}, + new Object[]{1566691200000L, ImmutableList.of("en", "es", "es-419", "es-US")} + )) + .setExpectedMSQSpec( + MSQSpec + .builder() + .query(expectedQuery) + .columnMappings(new ColumnMappings( + ImmutableList.of( + new ColumnMapping("d0", "__time"), + new ColumnMapping("a0", "cnt") + ) + )) + .tuningConfig(MSQTuningConfig.defaultConfig()) + .build()) + .setExpectedMSQFault(new CannotParseExternalDataFault( + "Unable to add the row to the frame. Type conversion might be required.")) + .verifyResults(); + } @Nonnull private List expectedMultiValueFooRowsGroup() diff --git a/extensions-core/multi-stage-query/src/test/resources/unparseable-mv-string-array.json b/extensions-core/multi-stage-query/src/test/resources/unparseable-mv-string-array.json new file mode 100644 index 000000000000..57b9a7709b09 --- /dev/null +++ b/extensions-core/multi-stage-query/src/test/resources/unparseable-mv-string-array.json @@ -0,0 +1,3 @@ +{"timestamp":"2019-08-25T00:00:00.031Z","agent_category":"Personal computer","agent_type":"Browser","browser":"Chrome","browser_version":"76.0.3809.100","city":"Rosario","continent":"South America","country":"Argentina","version":"1.9.6","event_type":"PercentClear","event_subtype":"55","loaded_image":"http://www.koalastothemax.com/img/koalas2.jpg","adblock_list":"NoAdblock","forwarded_for":"181.13.41.82","language":[{},{}],"number":"16","os":"Windows 7","path":"http://www.koalastothemax.com/","platform":"Windows","referrer":"Direct","referrer_host":"Direct","region":"Santa Fe","remote_address":"172.31.57.89","screen":"1680x1050","session":"S56194838","session_length":76261,"timezone":"N/A","timezone_offset":"180","window":"1680x939"} +{"timestamp":"2019-08-25T00:00:00.059Z","agent_category":"Smartphone","agent_type":"Mobile Browser","browser":"Chrome Mobile","browser_version":"50.0.2661.89","city":"Nuevo Casas Grandes","continent":"North America","country":"Mexico","version":"1.9.6","event_type":"PercentClear","event_subtype":"85","loaded_image":"https://koalastothemax.com/img/koalas1.jpg","adblock_list":"NoAdblock","forwarded_for":"177.242.100.0","language":["en","es","es-419","es-MX"],"number":"24","os":"Android","path":"https://koalastothemax.com/","platform":"Android","referrer":"https://www.google.com/","referrer_host":"www.google.com","region":"Chihuahua","remote_address":"172.31.11.5","screen":"320x570","session":"S46093731","session_length":252689,"timezone":"CDT","timezone_offset":"300","window":"540x743"} +{"timestamp":"2019-08-25T00:00:00.178Z","agent_category":"Personal computer","agent_type":"Browser","browser":"Chrome","browser_version":"76.0.3809.100","city":"Luis Guillon","continent":"South America","country":"Argentina","version":"1.9.6","event_type":"PercentClear","event_subtype":"90","loaded_image":"http://www.koalastothemax.com/img/koalas.jpg","adblock_list":"NoAdblock","forwarded_for":"181.46.136.44","language":["en","es","es-419","es-US"],"number":"24","os":"Windows 7","path":"http://www.koalastothemax.com/","platform":"Windows","referrer":"Direct","referrer_host":"Direct","region":"Buenos Aires","remote_address":"172.31.11.5","screen":"1366x768","session":"S13352079","session_length":1753602,"timezone":"N/A","timezone_offset":"180","window":"1366x652"} diff --git a/processing/src/main/java/org/apache/druid/frame/write/RowBasedFrameWriter.java b/processing/src/main/java/org/apache/druid/frame/write/RowBasedFrameWriter.java index 2bf4b343080c..2a8af4b9e8f8 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/RowBasedFrameWriter.java +++ b/processing/src/main/java/org/apache/druid/frame/write/RowBasedFrameWriter.java @@ -32,6 +32,7 @@ import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.java.util.common.parsers.ParseException; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.utils.CloseableUtils; @@ -118,8 +119,13 @@ public boolean addSelection() return false; } - if (!writeData()) { - return false; + try { + if (!writeData()) { + return false; + } + } + catch (Exception e) { + throw new ParseException("", e, "Unable to add the row to the frame. Type conversion might be required."); } final MemoryRange rowOffsetCursor = rowOffsetMemory.cursor(); diff --git a/processing/src/main/java/org/apache/druid/frame/write/columnar/ColumnarFrameWriter.java b/processing/src/main/java/org/apache/druid/frame/write/columnar/ColumnarFrameWriter.java index 76f5a17d27af..eff5213ae100 100644 --- a/processing/src/main/java/org/apache/druid/frame/write/columnar/ColumnarFrameWriter.java +++ b/processing/src/main/java/org/apache/druid/frame/write/columnar/ColumnarFrameWriter.java @@ -30,6 +30,7 @@ import org.apache.druid.frame.write.FrameWriter; import org.apache.druid.frame.write.FrameWriterUtils; import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.parsers.ParseException; import org.apache.druid.segment.column.RowSignature; import javax.annotation.Nullable; @@ -74,11 +75,16 @@ public boolean addSelection() } int i = 0; - for (; i < columnWriters.size(); i++) { - if (!columnWriters.get(i).addSelection()) { - break; + try { + for (; i < columnWriters.size(); i++) { + if (!columnWriters.get(i).addSelection()) { + break; + } } } + catch (Exception e) { + throw new ParseException("", e, "Unable to add the row to the frame. Type conversion might be required."); + } if (i < columnWriters.size()) { // Add failed, clean up.