From 6011fe232561263608e89afaf5e8d2ca90626fb4 Mon Sep 17 00:00:00 2001 From: zachjsh Date: Thu, 6 Apr 2023 17:18:59 -0400 Subject: [PATCH 1/6] * fix --- .../sql/calcite/external/SchemaAwareUserDefinedTableMacro.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/external/SchemaAwareUserDefinedTableMacro.java b/sql/src/main/java/org/apache/druid/sql/calcite/external/SchemaAwareUserDefinedTableMacro.java index b05c0a0cd868..9f8016976ce6 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/external/SchemaAwareUserDefinedTableMacro.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/external/SchemaAwareUserDefinedTableMacro.java @@ -176,8 +176,9 @@ public Set computeResources(final SqlCall call, final boolean in ResourceType.EXTERNAL, ((ExternalTable) table).getInputSourceType() ), Action.READ)); + } else { + resourceActions.addAll(base.computeResources(call, inputSourceTypeSecurityEnabled)); } - resourceActions.addAll(base.computeResources(call, inputSourceTypeSecurityEnabled)); return resourceActions; } } From 1d1671c1ca3c08b361fbc449e795dd7cb7bee5fd Mon Sep 17 00:00:00 2001 From: zachjsh Date: Thu, 6 Apr 2023 19:05:02 -0400 Subject: [PATCH 2/6] * add tests --- .../sql/calcite/IngestTableFunctionTest.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/IngestTableFunctionTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/IngestTableFunctionTest.java index 83bb87436be5..86136c700029 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/IngestTableFunctionTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/IngestTableFunctionTest.java @@ -35,6 +35,7 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.server.security.Access; +import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.ForbiddenException; import org.apache.druid.sql.calcite.external.ExternalDataSource; import org.apache.druid.sql.calcite.external.Externals; @@ -110,6 +111,65 @@ public void testHttpExtern() .verify(); } + /** + * Http function + */ + @Test + public void testHttpFunction() + { + String extern = "TABLE(http(" + + "userName => 'bob'," + + "password => 'secret'," + + "uris => ARRAY['http://foo.com/bar.csv']," + + "format => 'csv'))" + + " (x VARCHAR, y VARCHAR, z BIGINT)"; + testIngestionQuery() + .sql("INSERT INTO dst SELECT * FROM %s PARTITIONED BY ALL TIME", extern) + .authentication(CalciteTests.SUPER_USER_AUTH_RESULT) + .expectTarget("dst", httpDataSource.getSignature()) + .expectResources(dataSourceWrite("dst"), Externals.EXTERNAL_RESOURCE_ACTION) + .expectQuery( + newScanQueryBuilder() + .dataSource(httpDataSource) + .intervals(querySegmentSpec(Filtration.eternity())) + .columns("x", "y", "z") + .context(CalciteIngestionDmlTest.PARTITIONED_BY_ALL_TIME_QUERY_CONTEXT) + .build() + ) + .expectLogicalPlanFrom("httpExtern") + .verify(); + } + + /** + * Http function + */ + @Test + public void testHttpFunctionWithInputsourceSecurity() + { + String extern = "TABLE(http(" + + "userName => 'bob'," + + "password => 'secret'," + + "uris => ARRAY['http://foo.com/bar.csv']," + + "format => 'csv'))" + + " (x VARCHAR, y VARCHAR, z BIGINT)"; + testIngestionQuery() + .sql("INSERT INTO dst SELECT * FROM %s PARTITIONED BY ALL TIME", extern) + .authConfig(AuthConfig.newBuilder().setEnableInputSourceSecurity(true).build()) + .authentication(CalciteTests.SUPER_USER_AUTH_RESULT) + .expectTarget("dst", httpDataSource.getSignature()) + .expectResources(dataSourceWrite("dst"), externalRead("http")) + .expectQuery( + newScanQueryBuilder() + .dataSource(httpDataSource) + .intervals(querySegmentSpec(Filtration.eternity())) + .columns("x", "y", "z") + .context(CalciteIngestionDmlTest.PARTITIONED_BY_ALL_TIME_QUERY_CONTEXT) + .build() + ) + .expectLogicalPlanFrom("httpExtern") + .verify(); + } + protected String externSqlByName(final ExternalDataSource externalDataSource) { ObjectMapper queryJsonMapper = queryFramework().queryJsonMapper(); From 6e9913fd9c223233bb87a92f44aca29394bf9511 Mon Sep 17 00:00:00 2001 From: zachjsh Date: Fri, 7 Apr 2023 13:18:10 -0400 Subject: [PATCH 3/6] * fix --- .../model/table/S3InputSourceDefnTest.java | 22 +++--- .../model/table/BaseInputSourceDefn.java | 5 +- .../model/table/ExternalTableSpec.java | 9 ++- .../model/table/FormattedInputSourceDefn.java | 3 +- .../model/table/HttpInputSourceDefnTest.java | 10 +-- .../table/InlineInputSourceDefnTest.java | 6 +- .../model/table/LocalInputSourceDefnTest.java | 2 +- .../external/DruidExternTableMacro.java | 11 +-- .../external/ExternalOperatorConversion.java | 7 +- .../druid/sql/calcite/external/Externals.java | 8 ++- .../SchemaAwareUserDefinedTableMacro.java | 10 +-- .../sql/calcite/table/ExternalTable.java | 14 ++-- .../sql/calcite/CalciteIngestionDmlTest.java | 1 + .../sql/calcite/CalciteInsertDmlTest.java | 72 +++++++++++++++++++ 14 files changed, 133 insertions(+), 47 deletions(-) diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/catalog/model/table/S3InputSourceDefnTest.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/catalog/model/table/S3InputSourceDefnTest.java index 759cbc6875cd..e0c10358817a 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/catalog/model/table/S3InputSourceDefnTest.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/catalog/model/table/S3InputSourceDefnTest.java @@ -345,7 +345,7 @@ public void testAdHocUri() CatalogUtils.stringListToUriList(uris), s3InputSource.getUris() ); - assertEquals(S3InputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(S3InputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); // But, it fails if there are no columns. assertThrows(IAE.class, () -> fn.apply("x", args, Collections.emptyList(), mapper)); @@ -366,7 +366,7 @@ public void testMultipleAdHocUris() CatalogUtils.stringListToUriList(uris), s3InputSource.getUris() ); - assertEquals(S3InputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(S3InputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); } @Test @@ -399,7 +399,7 @@ public void testAdHocUriWithGlob() s3InputSource.getUris() ); assertEquals("*.csv", s3InputSource.getObjectGlob()); - assertEquals(S3InputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(S3InputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); } @Test @@ -419,7 +419,7 @@ public void testAdHocPrefix() CatalogUtils.stringListToUriList(prefixes), s3InputSource.getPrefixes() ); - assertEquals(S3InputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(S3InputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); // But, it fails if there are no columns. assertThrows(IAE.class, () -> fn.apply("x", args, Collections.emptyList(), mapper)); @@ -442,7 +442,7 @@ public void testMultipleAdHocPrefixes() CatalogUtils.stringListToUriList(prefixes), s3InputSource.getPrefixes() ); - assertEquals(S3InputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(S3InputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); } @Test @@ -462,7 +462,7 @@ public void testAdHocBucketAndPaths() CloudObjectLocation obj = s3InputSource.getObjects().get(0); assertEquals("foo.com", obj.getBucket()); assertEquals("bar/file.csv", obj.getPath()); - assertEquals(S3InputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(S3InputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); // But, it fails if there are no columns. assertThrows(IAE.class, () -> fn.apply("x", args, Collections.emptyList(), mapper)); @@ -510,7 +510,7 @@ public void testMultipleAdHocObjects() obj = s3InputSource.getObjects().get(1); assertEquals("foo.com", obj.getBucket()); assertEquals("mumble/file2.csv", obj.getPath()); - assertEquals(S3InputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(S3InputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); } @Test @@ -578,7 +578,7 @@ public void testFullTableSpecHappyPath() ExternalTableDefn externDefn = (ExternalTableDefn) resolved.defn(); ExternalTableSpec externSpec = externDefn.convert(resolved); assertEquals(s3InputSource, externSpec.inputSource); - assertEquals(S3InputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(S3InputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); // Get the partial table function TableFunction fn = externDefn.tableFn(resolved); @@ -613,7 +613,7 @@ public void testTableSpecWithoutConfig() ExternalTableDefn externDefn = (ExternalTableDefn) resolved.defn(); ExternalTableSpec externSpec = externDefn.convert(resolved); assertEquals(s3InputSource, externSpec.inputSource); - assertEquals(S3InputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(S3InputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); // Get the partial table function TableFunction fn = externDefn.tableFn(resolved); @@ -672,7 +672,7 @@ public void testTableSpecWithBucketAndFormat() CloudObjectLocation obj = s3InputSource.getObjects().get(0); assertEquals("foo.com", obj.getBucket()); assertEquals("bar/file.csv", obj.getPath()); - assertEquals(S3InputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(S3InputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); // But, it fails columns are provided since the table already has them. assertThrows(IAE.class, () -> fn.apply("x", args, COLUMNS, mapper)); @@ -714,7 +714,7 @@ public void testTableSpecAsConnection() assertEquals("foo.com", obj.getBucket()); assertEquals("bar/file.csv", obj.getPath()); assertTrue(externSpec.inputFormat instanceof CsvInputFormat); - assertEquals(S3InputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(S3InputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); // But, it fails columns are not provided since the table does not have them. assertThrows(IAE.class, () -> fn.apply("x", args, Collections.emptyList(), mapper)); diff --git a/server/src/main/java/org/apache/druid/catalog/model/table/BaseInputSourceDefn.java b/server/src/main/java/org/apache/druid/catalog/model/table/BaseInputSourceDefn.java index 83a085ead4ff..660e73c35239 100644 --- a/server/src/main/java/org/apache/druid/catalog/model/table/BaseInputSourceDefn.java +++ b/server/src/main/java/org/apache/druid/catalog/model/table/BaseInputSourceDefn.java @@ -29,6 +29,7 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.utils.CollectionUtils; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -152,7 +153,7 @@ protected ExternalTableSpec convertArgsToTable( convertArgsToSource(args, jsonMapper), convertArgsToFormat(args, columns, jsonMapper), Columns.convertSignature(columns), - typeValue() + () -> Collections.singleton(typeValue()) ); } @@ -209,7 +210,7 @@ public ExternalTableSpec convertTable(ResolvedExternalTable table) convertTableToSource(table), convertTableToFormat(table), Columns.convertSignature(table.resolvedTable().spec().columns()), - typeValue() + () -> Collections.singleton(typeValue()) ); } diff --git a/server/src/main/java/org/apache/druid/catalog/model/table/ExternalTableSpec.java b/server/src/main/java/org/apache/druid/catalog/model/table/ExternalTableSpec.java index a132ff67280e..92d782f8cf6c 100644 --- a/server/src/main/java/org/apache/druid/catalog/model/table/ExternalTableSpec.java +++ b/server/src/main/java/org/apache/druid/catalog/model/table/ExternalTableSpec.java @@ -19,11 +19,14 @@ package org.apache.druid.catalog.model.table; +import com.google.common.base.Function; +import com.google.common.base.Supplier; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.InputSource; import org.apache.druid.segment.column.RowSignature; import javax.annotation.Nullable; +import java.util.Set; /** * Catalog form of an external table specification used to pass along the three @@ -36,17 +39,17 @@ public class ExternalTableSpec public final InputSource inputSource; public final InputFormat inputFormat; @Nullable public final RowSignature signature; - public final String inputSourceType; + public final Supplier> inputSourceTypesSupplier; public ExternalTableSpec( final InputSource inputSource, final InputFormat inputFormat, final RowSignature signature, - final String inputSourceType) + final Supplier> inputSourceTypesSupplier) { this.inputSource = inputSource; this.inputFormat = inputFormat; this.signature = signature; - this.inputSourceType = inputSourceType; + this.inputSourceTypesSupplier = inputSourceTypesSupplier; } } diff --git a/server/src/main/java/org/apache/druid/catalog/model/table/FormattedInputSourceDefn.java b/server/src/main/java/org/apache/druid/catalog/model/table/FormattedInputSourceDefn.java index bf0a904bca01..a00e71b89174 100644 --- a/server/src/main/java/org/apache/druid/catalog/model/table/FormattedInputSourceDefn.java +++ b/server/src/main/java/org/apache/druid/catalog/model/table/FormattedInputSourceDefn.java @@ -33,6 +33,7 @@ import org.apache.druid.utils.CollectionUtils; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -194,7 +195,7 @@ protected ExternalTableSpec convertPartialFormattedTable( convertSource(sourceMap, jsonMapper), inputFormat, Columns.convertSignature(completedCols), - typeValue() + () -> Collections.singleton(typeValue()) ); } } diff --git a/server/src/test/java/org/apache/druid/catalog/model/table/HttpInputSourceDefnTest.java b/server/src/test/java/org/apache/druid/catalog/model/table/HttpInputSourceDefnTest.java index 21e6f14eaca2..67ceedf30391 100644 --- a/server/src/test/java/org/apache/druid/catalog/model/table/HttpInputSourceDefnTest.java +++ b/server/src/test/java/org/apache/druid/catalog/model/table/HttpInputSourceDefnTest.java @@ -398,7 +398,7 @@ public void testMultipleURIsInTableSpec() throws URISyntaxException CatalogUtils.stringListToUriList(Arrays.asList("http://foo.com/foo.csv", "http://foo.com/bar.csv")), sourceSpec.getUris() ); - assertEquals(HttpInputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(HttpInputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); } @Test @@ -442,7 +442,7 @@ public void testMultipleURIsWithTemplate() throws URISyntaxException CatalogUtils.stringListToUriList(Arrays.asList("http://foo.com/my.csv", "http://foo.com/bar.csv")), sourceSpec.getUris() ); - assertEquals(HttpInputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(HttpInputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); } @Test @@ -466,7 +466,7 @@ public void testMultipleURIsAdHoc() CatalogUtils.stringListToUriList(Arrays.asList("http://foo.com/foo.csv", "http://foo.com/bar.csv")), sourceSpec.getUris() ); - assertEquals(HttpInputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(HttpInputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); } @Test @@ -499,7 +499,7 @@ public void testEnvPassword() throws URISyntaxException "SECRET", ((EnvironmentVariablePasswordProvider) sourceSpec.getHttpAuthenticationPasswordProvider()).getVariable() ); - assertEquals(HttpInputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(HttpInputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); } private void validateHappyPath(ExternalTableSpec externSpec, boolean withUser) @@ -519,7 +519,7 @@ private void validateHappyPath(ExternalTableSpec externSpec, boolean withUser) assertEquals(Arrays.asList("x", "y"), sig.getColumnNames()); assertEquals(ColumnType.STRING, sig.getColumnType(0).get()); assertEquals(ColumnType.LONG, sig.getColumnType(1).get()); - assertEquals(HttpInputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(HttpInputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); } private Map httpToMap(HttpInputSource source) diff --git a/server/src/test/java/org/apache/druid/catalog/model/table/InlineInputSourceDefnTest.java b/server/src/test/java/org/apache/druid/catalog/model/table/InlineInputSourceDefnTest.java index daae0bccd05b..448e353664dc 100644 --- a/server/src/test/java/org/apache/druid/catalog/model/table/InlineInputSourceDefnTest.java +++ b/server/src/test/java/org/apache/druid/catalog/model/table/InlineInputSourceDefnTest.java @@ -144,7 +144,7 @@ public void testValidAdHocFn() CsvInputFormat format = (CsvInputFormat) extern.inputFormat; assertEquals(Arrays.asList("a", "b"), format.getColumns()); assertEquals(2, extern.signature.size()); - assertEquals(InlineInputSourceDefn.TYPE_KEY, extern.inputSourceType); + assertEquals(Collections.singleton(InlineInputSourceDefn.TYPE_KEY), extern.inputSourceTypesSupplier.get()); // Fails if no columns are provided. assertThrows(IAE.class, () -> fn.apply("x", new HashMap<>(), Collections.emptyList(), mapper)); @@ -179,7 +179,7 @@ public void testPartialTable() CsvInputFormat actualFormat = (CsvInputFormat) extern.inputFormat; assertEquals(Arrays.asList("a", "b"), actualFormat.getColumns()); assertEquals(2, extern.signature.size()); - assertEquals(InlineInputSourceDefn.TYPE_KEY, extern.inputSourceType); + assertEquals(Collections.singleton(InlineInputSourceDefn.TYPE_KEY), extern.inputSourceTypesSupplier.get()); // Cannot supply columns with the function List columns = Arrays.asList( @@ -215,6 +215,6 @@ public void testDefinedTable() CsvInputFormat actualFormat = (CsvInputFormat) extern.inputFormat; assertEquals(Arrays.asList("a", "b"), actualFormat.getColumns()); assertEquals(2, extern.signature.size()); - assertEquals(InlineInputSourceDefn.TYPE_KEY, extern.inputSourceType); + assertEquals(Collections.singleton(InlineInputSourceDefn.TYPE_KEY), extern.inputSourceTypesSupplier.get()); } } diff --git a/server/src/test/java/org/apache/druid/catalog/model/table/LocalInputSourceDefnTest.java b/server/src/test/java/org/apache/druid/catalog/model/table/LocalInputSourceDefnTest.java index a71a00f4b99d..1bbc74513b2f 100644 --- a/server/src/test/java/org/apache/druid/catalog/model/table/LocalInputSourceDefnTest.java +++ b/server/src/test/java/org/apache/druid/catalog/model/table/LocalInputSourceDefnTest.java @@ -536,6 +536,6 @@ private void validateFormat(ExternalTableSpec externSpec) assertEquals(Arrays.asList("x", "y"), sig.getColumnNames()); assertEquals(ColumnType.STRING, sig.getColumnType(0).get()); assertEquals(ColumnType.LONG, sig.getColumnType(1).get()); - assertEquals(LocalInputSourceDefn.TYPE_KEY, externSpec.inputSourceType); + assertEquals(Collections.singleton(LocalInputSourceDefn.TYPE_KEY), externSpec.inputSourceTypesSupplier.get()); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/external/DruidExternTableMacro.java b/sql/src/main/java/org/apache/druid/sql/calcite/external/DruidExternTableMacro.java index e8740ba5bb60..83521198f98d 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/external/DruidExternTableMacro.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/external/DruidExternTableMacro.java @@ -25,6 +25,7 @@ import org.apache.calcite.sql.SqlCharStringLiteral; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.util.NlsString; +import org.apache.druid.data.input.InputSource; import org.apache.druid.server.security.Action; import org.apache.druid.server.security.Resource; import org.apache.druid.server.security.ResourceAction; @@ -34,6 +35,7 @@ import javax.validation.constraints.NotNull; import java.util.Collections; import java.util.Set; +import java.util.stream.Collectors; /** * Used by {@link ExternalOperatorConversion} to generate a {@link DruidTable} @@ -55,11 +57,10 @@ public Set computeResources(final SqlCall call, boolean inputSou String inputSourceStr = getInputSourceArgument(call); try { - JsonNode jsonNode = ((DruidTableMacro) macro).getJsonMapper().readTree(inputSourceStr); - return Collections.singleton(new ResourceAction(new Resource( - ResourceType.EXTERNAL, - jsonNode.get("type").asText() - ), Action.READ)); + InputSource inputSource = ((DruidTableMacro) macro).getJsonMapper().readValue(inputSourceStr, InputSource.class); + return inputSource.getTypes().stream() + .map(inputSourceType -> new ResourceAction(new Resource(ResourceType.EXTERNAL, inputSourceType), Action.READ)) + .collect(Collectors.toSet()); } catch (JsonProcessingException e) { // this shouldn't happen, the input source paraemeter should have been validated before this diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/external/ExternalOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/external/ExternalOperatorConversion.java index 6b1b1fb62419..3dacbea66511 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/external/ExternalOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/external/ExternalOperatorConversion.java @@ -34,6 +34,7 @@ import org.apache.druid.segment.column.RowSignature; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -111,12 +112,12 @@ public ExternalTableSpec apply( } String inputSrcStr = CatalogUtils.getString(args, INPUT_SOURCE_PARAM); - String inputSrcType = jsonMapper.readTree(inputSrcStr).get("type").asText(); + InputSource inputSource = jsonMapper.readValue(inputSrcStr, InputSource.class); return new ExternalTableSpec( - jsonMapper.readValue(inputSrcStr, InputSource.class), + inputSource, jsonMapper.readValue(CatalogUtils.getString(args, INPUT_FORMAT_PARAM), InputFormat.class), rowSignature, - inputSrcType + inputSource::getTypes ); } catch (JsonProcessingException e) { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/external/Externals.java b/sql/src/main/java/org/apache/druid/sql/calcite/external/Externals.java index 595c82571815..ee0c650242ea 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/external/Externals.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/external/Externals.java @@ -20,6 +20,7 @@ package org.apache.druid.sql.calcite.external; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import org.apache.calcite.avatica.SqlType; import org.apache.calcite.rel.type.RelDataType; @@ -55,6 +56,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; /** @@ -297,7 +299,7 @@ public static ExternalTable buildExternalTable( + "Please change the column name to something other than __time"); } - return toExternalTable(spec, jsonMapper, spec.inputSourceType); + return toExternalTable(spec, jsonMapper, spec.inputSourceTypesSupplier); } public static ResourceAction externalRead(String name) @@ -308,7 +310,7 @@ public static ResourceAction externalRead(String name) public static ExternalTable toExternalTable( ExternalTableSpec spec, ObjectMapper jsonMapper, - String inputSourceType + Supplier> inputSourceTypesSupplier ) { return new ExternalTable( @@ -319,7 +321,7 @@ public static ExternalTable toExternalTable( ), spec.signature, jsonMapper, - inputSourceType + inputSourceTypesSupplier ); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/external/SchemaAwareUserDefinedTableMacro.java b/sql/src/main/java/org/apache/druid/sql/calcite/external/SchemaAwareUserDefinedTableMacro.java index 9f8016976ce6..28c6236c09a3 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/external/SchemaAwareUserDefinedTableMacro.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/external/SchemaAwareUserDefinedTableMacro.java @@ -48,6 +48,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; /** * Table macro designed for use with the Druid EXTEND operator. Example: @@ -172,10 +173,11 @@ public Set computeResources(final SqlCall call, final boolean in { Set resourceActions = new HashSet<>(); if (table instanceof ExternalTable && inputSourceTypeSecurityEnabled) { - resourceActions.add(new ResourceAction(new Resource( - ResourceType.EXTERNAL, - ((ExternalTable) table).getInputSourceType() - ), Action.READ)); + resourceActions.addAll(((ExternalTable) table) + .getInputSourceTypeSupplier().get().stream() + .map(inputSourceType -> + new ResourceAction(new Resource(ResourceType.EXTERNAL, inputSourceType), Action.READ)) + .collect(Collectors.toSet())); } else { resourceActions.addAll(base.computeResources(call, inputSourceTypeSecurityEnabled)); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/table/ExternalTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/table/ExternalTable.java index 3c6fbdd0acfe..bda5b4362596 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/table/ExternalTable.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/table/ExternalTable.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelOptTable.ToRelContext; import org.apache.calcite.rel.RelNode; @@ -30,6 +31,8 @@ import org.apache.druid.segment.column.RowSignature; import org.apache.druid.sql.calcite.external.ExternalTableScan; +import java.util.Set; + /** * Represents an source of data external to Druid: a CSV file, an HTTP request, etc. * Each such table represents one of Druid's {@link DataSource} types. Since SQL @@ -42,7 +45,7 @@ public class ExternalTable extends DruidTable private final DataSource dataSource; private final ObjectMapper objectMapper; - private final String inputSourceType; + private final Supplier> inputSourceTypeSupplier; /** * Cached row type, to avoid recreating types multiple times. @@ -53,13 +56,13 @@ public ExternalTable( final DataSource dataSource, final RowSignature rowSignature, final ObjectMapper objectMapper, - final String inputSourceType + final Supplier> inputSourceTypesSupplier ) { super(rowSignature); this.dataSource = Preconditions.checkNotNull(dataSource, "dataSource"); this.objectMapper = objectMapper; - this.inputSourceType = inputSourceType; + this.inputSourceTypeSupplier = inputSourceTypesSupplier; } @Override @@ -93,9 +96,9 @@ public RelDataType getRowType(final RelDataTypeFactory typeFactory) return rowType; } - public String getInputSourceType() + public Supplier> getInputSourceTypeSupplier() { - return inputSourceType; + return inputSourceTypeSupplier; } @Override @@ -110,7 +113,6 @@ public String toString() return "ExternalTable{" + "dataSource=" + dataSource + ", rowSignature=" + getRowSignature() + - ", inputSourceType=" + getInputSourceType() + '}'; } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteIngestionDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteIngestionDmlTest.java index 50d0bc445ba1..4830e352a47a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteIngestionDmlTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteIngestionDmlTest.java @@ -62,6 +62,7 @@ import org.junit.Assert; import org.junit.internal.matchers.ThrowableMessageMatcher; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java index d9b6b6e32f4f..afaefc0d8363 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java @@ -23,6 +23,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.apache.druid.data.input.InputFormat; +import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.InputSource; +import org.apache.druid.data.input.InputSourceReader; import org.apache.druid.data.input.impl.CsvInputFormat; import org.apache.druid.data.input.impl.InlineInputSource; import org.apache.druid.java.util.common.StringUtils; @@ -55,6 +59,8 @@ import org.junit.Test; import org.junit.internal.matchers.ThrowableMessageMatcher; +import javax.annotation.Nullable; +import java.io.File; import java.io.IOException; import java.util.Collections; import java.util.HashMap; @@ -473,6 +479,72 @@ public void testInsertFromExternalFunctionalStyleWithSchemaWithInputsourceSecuri .verify(); } + @Test + public void testInsertFromExternalAnonymousInputSourceFunctionalStyleWithSchemaWithInputsourceSecurity() + { + String extern; + ObjectMapper queryJsonMapper = queryFramework().queryJsonMapper(); + try { + extern = StringUtils.format( + "TABLE(extern(" + + "inputSource => '%s'," + + "inputFormat => '%s'))", + queryJsonMapper.writeValueAsString( + new InputSource() + { + @Override + public boolean isSplittable() + { + return false; + } + + @Override + public boolean needsFormat() + { + return false; + } + + @Override + public InputSourceReader reader( + InputRowSchema inputRowSchema, + @Nullable InputFormat inputFormat, + File temporaryDirectory + ) + { + return null; + } + } + ), + queryJsonMapper.writeValueAsString( + new CsvInputFormat(ImmutableList.of("x", "y", "z"), null, false, false, 0) + ) + ); + } + catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + testIngestionQuery() + .sql("INSERT INTO dst SELECT * FROM %s\n" + + " (x VARCHAR, y VARCHAR, z BIGINT)\n" + + "PARTITIONED BY ALL TIME", + extern + ) + .authentication(CalciteTests.SUPER_USER_AUTH_RESULT) + .authConfig(AuthConfig.newBuilder().setEnableInputSourceSecurity(true).build()) + .expectTarget("dst", externalDataSource.getSignature()) + .expectResources(dataSourceWrite("dst"), externalRead("inline")) + .expectQuery( + newScanQueryBuilder() + .dataSource(externalDataSource) + .intervals(querySegmentSpec(Filtration.eternity())) + .columns("x", "y", "z") + .context(PARTITIONED_BY_ALL_TIME_QUERY_CONTEXT) + .build() + ) + .expectLogicalPlanFrom("insertFromExternal") + .verify(); + } + @Test public void testInsertWithPartitionedBy() { From fad918b3eba284fdd3839ab6fdafef734c2ef8dc Mon Sep 17 00:00:00 2001 From: zachjsh Date: Fri, 7 Apr 2023 14:30:53 -0400 Subject: [PATCH 4/6] * add more tests --- .../sql/calcite/BaseCalciteQueryTest.java | 2 +- .../sql/calcite/CalciteIngestionDmlTest.java | 77 ++++++++++++++++- .../sql/calcite/CalciteInsertDmlTest.java | 86 ++++++++++++------- ...rnalWithoutSecuritySupport-logicalPlan.txt | 3 + 4 files changed, 132 insertions(+), 36 deletions(-) create mode 100644 sql/src/test/resources/calcite/expected/ingest/InsertFromExternalWithoutSecuritySupport-logicalPlan.txt diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index ea85780bd7b0..43eb443045ad 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -992,7 +992,7 @@ public SqlStatementFactory getSqlStatementFactory( * factory is specific to one test and one planner config. This method can be * overridden to control the objects passed to the factory. */ - private SqlStatementFactory getSqlStatementFactory( + SqlStatementFactory getSqlStatementFactory( PlannerConfig plannerConfig, AuthConfig authConfig ) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteIngestionDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteIngestionDmlTest.java index 4830e352a47a..75d3fa24322a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteIngestionDmlTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteIngestionDmlTest.java @@ -19,6 +19,8 @@ package org.apache.druid.sql.calcite; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; @@ -27,8 +29,13 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.inject.Binder; +import org.apache.druid.data.input.AbstractInputSource; +import org.apache.druid.data.input.InputFormat; +import org.apache.druid.data.input.InputSplit; +import org.apache.druid.data.input.SplitHintSpec; import org.apache.druid.data.input.impl.CsvInputFormat; import org.apache.druid.data.input.impl.InlineInputSource; +import org.apache.druid.data.input.impl.SplittableInputSource; import org.apache.druid.guice.DruidInjectorBuilder; import org.apache.druid.initialization.DruidModule; import org.apache.druid.java.util.common.ISE; @@ -62,11 +69,15 @@ import org.junit.Assert; import org.junit.internal.matchers.ThrowableMessageMatcher; +import javax.annotation.Nullable; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.stream.Stream; public class CalciteIngestionDmlTest extends BaseCalciteQueryTest { @@ -144,7 +155,9 @@ public void configure(Binder binder) public List getJacksonModules() { // We want this module to bring input sources along for the ride. - return new InputSourceModule().getJacksonModules(); + List modules = new ArrayList<>(new InputSourceModule().getJacksonModules()); + modules.add(new SimpleModule("test-module").registerSubtypes(TestFileInputSource.class)); + return modules; } @Override @@ -366,7 +379,7 @@ private void verifyValidationError() final Throwable e = Assert.assertThrows( Throwable.class, () -> { - getSqlStatementFactory(plannerConfig).directStatement(sqlQuery()).execute(); + getSqlStatementFactory(plannerConfig, authConfig).directStatement(sqlQuery()).execute(); } ); @@ -424,4 +437,64 @@ private SqlQueryPlus sqlQuery() .build(); } } + + static class TestFileInputSource extends AbstractInputSource implements SplittableInputSource + { + private final List files; + + @JsonCreator + TestFileInputSource(@JsonProperty("files") List fileList) + { + files = fileList; + } + + @JsonProperty + public List getFiles() + { + return files; + } + + @Override + public Stream> createSplits(InputFormat inputFormat, @Nullable SplitHintSpec splitHintSpec) + { + return files.stream().map(InputSplit::new); + } + + @Override + public int estimateNumSplits(InputFormat inputFormat, @Nullable SplitHintSpec splitHintSpec) + { + return files.size(); + } + + @Override + public SplittableInputSource withSplit(InputSplit split) + { + return new TestFileInputSource(ImmutableList.of(split.get())); + } + + @Override + public boolean needsFormat() + { + return true; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TestFileInputSource that = (TestFileInputSource) o; + return Objects.equals(files, that.files); + } + + @Override + public int hashCode() + { + return Objects.hash(files); + } + } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java index afaefc0d8363..8e2338ecb7b2 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java @@ -23,10 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import org.apache.druid.data.input.InputFormat; -import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.InputSource; -import org.apache.druid.data.input.InputSourceReader; import org.apache.druid.data.input.impl.CsvInputFormat; import org.apache.druid.data.input.impl.InlineInputSource; import org.apache.druid.java.util.common.StringUtils; @@ -59,7 +56,6 @@ import org.junit.Test; import org.junit.internal.matchers.ThrowableMessageMatcher; -import javax.annotation.Nullable; import java.io.File; import java.io.IOException; import java.util.Collections; @@ -480,8 +476,19 @@ public void testInsertFromExternalFunctionalStyleWithSchemaWithInputsourceSecuri } @Test - public void testInsertFromExternalAnonymousInputSourceFunctionalStyleWithSchemaWithInputsourceSecurity() + public void testInsertFromExternalWithoutSecuritySupport() { + InputSource inputSource = + new TestFileInputSource(ImmutableList.of(new File("testFile.csv").getAbsoluteFile())); + final ExternalDataSource externalDataSource = new ExternalDataSource( + inputSource, + new CsvInputFormat(ImmutableList.of("x", "y", "z"), null, false, false, 0), + RowSignature.builder() + .add("x", ColumnType.STRING) + .add("y", ColumnType.STRING) + .add("z", ColumnType.LONG) + .build() + ); String extern; ObjectMapper queryJsonMapper = queryFramework().queryJsonMapper(); try { @@ -489,32 +496,7 @@ public void testInsertFromExternalAnonymousInputSourceFunctionalStyleWithSchemaW "TABLE(extern(" + "inputSource => '%s'," + "inputFormat => '%s'))", - queryJsonMapper.writeValueAsString( - new InputSource() - { - @Override - public boolean isSplittable() - { - return false; - } - - @Override - public boolean needsFormat() - { - return false; - } - - @Override - public InputSourceReader reader( - InputRowSchema inputRowSchema, - @Nullable InputFormat inputFormat, - File temporaryDirectory - ) - { - return null; - } - } - ), + queryJsonMapper.writeValueAsString(inputSource), queryJsonMapper.writeValueAsString( new CsvInputFormat(ImmutableList.of("x", "y", "z"), null, false, false, 0) ) @@ -530,9 +512,9 @@ public InputSourceReader reader( extern ) .authentication(CalciteTests.SUPER_USER_AUTH_RESULT) - .authConfig(AuthConfig.newBuilder().setEnableInputSourceSecurity(true).build()) + .authConfig(AuthConfig.newBuilder().setEnableInputSourceSecurity(false).build()) .expectTarget("dst", externalDataSource.getSignature()) - .expectResources(dataSourceWrite("dst"), externalRead("inline")) + .expectResources(dataSourceWrite("dst"), Externals.EXTERNAL_RESOURCE_ACTION) .expectQuery( newScanQueryBuilder() .dataSource(externalDataSource) @@ -541,7 +523,45 @@ public InputSourceReader reader( .context(PARTITIONED_BY_ALL_TIME_QUERY_CONTEXT) .build() ) + .expectLogicalPlanFrom("InsertFromExternalWithoutSecuritySupport") + .verify(); + } + + @Test + public void testInsertFromExternalWithoutSecuritySupportWithInputsourceSecurityEnabled() + { + String extern; + ObjectMapper queryJsonMapper = queryFramework().queryJsonMapper(); + try { + extern = StringUtils.format( + "TABLE(extern(" + + "inputSource => '%s'," + + "inputFormat => '%s'))", + queryJsonMapper.writeValueAsString(new TestFileInputSource(ImmutableList.of(new File("testFile.csv").getAbsoluteFile()))), + queryJsonMapper.writeValueAsString( + new CsvInputFormat(ImmutableList.of("x", "y", "z"), null, false, false, 0) + ) + ); + } + catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + testIngestionQuery() + .sql("INSERT INTO dst SELECT * FROM %s\n" + + " (x VARCHAR, y VARCHAR, z BIGINT)\n" + + "PARTITIONED BY ALL TIME", + extern + ) + .authentication(CalciteTests.SUPER_USER_AUTH_RESULT) + .authConfig(AuthConfig.newBuilder().setEnableInputSourceSecurity(true).build()) .expectLogicalPlanFrom("insertFromExternal") + .expectValidationError( + CoreMatchers.allOf( + CoreMatchers.instanceOf(SqlPlanningException.class), + ThrowableMessageMatcher.hasMessage(CoreMatchers.equalTo( + "org.apache.druid.java.util.common.UOE: This inputSource does not support input source based security")) + ) + ) .verify(); } diff --git a/sql/src/test/resources/calcite/expected/ingest/InsertFromExternalWithoutSecuritySupport-logicalPlan.txt b/sql/src/test/resources/calcite/expected/ingest/InsertFromExternalWithoutSecuritySupport-logicalPlan.txt new file mode 100644 index 000000000000..a5f87a12a866 --- /dev/null +++ b/sql/src/test/resources/calcite/expected/ingest/InsertFromExternalWithoutSecuritySupport-logicalPlan.txt @@ -0,0 +1,3 @@ +LogicalInsert(target=[dst], partitionedBy=[AllGranularity], clusteredBy=[]) + LogicalProject(x=[$0], y=[$1], z=[$2]) + ExternalTableScan(dataSource=[{"type":"external","inputSource":{"type":"CalciteIngestionDmlTest$TestFileInputSource","files":["/Users/zachsherman/projects/opensrc-druid/sql/testFile.csv"]},"inputFormat":{"type":"csv","columns":["x","y","z"]},"signature":[{"name":"x","type":"STRING"},{"name":"y","type":"STRING"},{"name":"z","type":"LONG"}]}]) From a43eff008494f783a1660eaf57de37b856271ab7 Mon Sep 17 00:00:00 2001 From: zachjsh Date: Fri, 7 Apr 2023 15:16:09 -0400 Subject: [PATCH 5/6] * fix failing test --- .../org/apache/druid/sql/calcite/CalciteInsertDmlTest.java | 5 +++-- .../InsertFromExternalWithoutSecuritySupport-logicalPlan.txt | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java index 8e2338ecb7b2..fb184e89c880 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteInsertDmlTest.java @@ -479,7 +479,7 @@ public void testInsertFromExternalFunctionalStyleWithSchemaWithInputsourceSecuri public void testInsertFromExternalWithoutSecuritySupport() { InputSource inputSource = - new TestFileInputSource(ImmutableList.of(new File("testFile.csv").getAbsoluteFile())); + new TestFileInputSource(ImmutableList.of(new File("/tmp/foo.csv").getAbsoluteFile())); final ExternalDataSource externalDataSource = new ExternalDataSource( inputSource, new CsvInputFormat(ImmutableList.of("x", "y", "z"), null, false, false, 0), @@ -537,7 +537,8 @@ public void testInsertFromExternalWithoutSecuritySupportWithInputsourceSecurityE "TABLE(extern(" + "inputSource => '%s'," + "inputFormat => '%s'))", - queryJsonMapper.writeValueAsString(new TestFileInputSource(ImmutableList.of(new File("testFile.csv").getAbsoluteFile()))), + queryJsonMapper.writeValueAsString( + new TestFileInputSource(ImmutableList.of(new File("/tmp/foo.csv").getAbsoluteFile()))), queryJsonMapper.writeValueAsString( new CsvInputFormat(ImmutableList.of("x", "y", "z"), null, false, false, 0) ) diff --git a/sql/src/test/resources/calcite/expected/ingest/InsertFromExternalWithoutSecuritySupport-logicalPlan.txt b/sql/src/test/resources/calcite/expected/ingest/InsertFromExternalWithoutSecuritySupport-logicalPlan.txt index a5f87a12a866..272f660d21af 100644 --- a/sql/src/test/resources/calcite/expected/ingest/InsertFromExternalWithoutSecuritySupport-logicalPlan.txt +++ b/sql/src/test/resources/calcite/expected/ingest/InsertFromExternalWithoutSecuritySupport-logicalPlan.txt @@ -1,3 +1,3 @@ LogicalInsert(target=[dst], partitionedBy=[AllGranularity], clusteredBy=[]) LogicalProject(x=[$0], y=[$1], z=[$2]) - ExternalTableScan(dataSource=[{"type":"external","inputSource":{"type":"CalciteIngestionDmlTest$TestFileInputSource","files":["/Users/zachsherman/projects/opensrc-druid/sql/testFile.csv"]},"inputFormat":{"type":"csv","columns":["x","y","z"]},"signature":[{"name":"x","type":"STRING"},{"name":"y","type":"STRING"},{"name":"z","type":"LONG"}]}]) + ExternalTableScan(dataSource=[{"type":"external","inputSource":{"type":"CalciteIngestionDmlTest$TestFileInputSource","files":["/tmp/foo.csv"]},"inputFormat":{"type":"csv","columns":["x","y","z"]},"signature":[{"name":"x","type":"STRING"},{"name":"y","type":"STRING"},{"name":"z","type":"LONG"}]}]) From 92cd38c37e92a737ce818018548376504ba586e2 Mon Sep 17 00:00:00 2001 From: zachjsh Date: Fri, 7 Apr 2023 17:48:20 -0400 Subject: [PATCH 6/6] * fix checkstyle issues --- .../org/apache/druid/catalog/model/table/ExternalTableSpec.java | 1 - .../apache/druid/sql/calcite/external/DruidExternTableMacro.java | 1 - .../druid/sql/calcite/external/ExternalOperatorConversion.java | 1 - 3 files changed, 3 deletions(-) diff --git a/server/src/main/java/org/apache/druid/catalog/model/table/ExternalTableSpec.java b/server/src/main/java/org/apache/druid/catalog/model/table/ExternalTableSpec.java index 92d782f8cf6c..7ea408004d0d 100644 --- a/server/src/main/java/org/apache/druid/catalog/model/table/ExternalTableSpec.java +++ b/server/src/main/java/org/apache/druid/catalog/model/table/ExternalTableSpec.java @@ -19,7 +19,6 @@ package org.apache.druid.catalog.model.table; -import com.google.common.base.Function; import com.google.common.base.Supplier; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.InputSource; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/external/DruidExternTableMacro.java b/sql/src/main/java/org/apache/druid/sql/calcite/external/DruidExternTableMacro.java index 83521198f98d..e74a0fdd8696 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/external/DruidExternTableMacro.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/external/DruidExternTableMacro.java @@ -20,7 +20,6 @@ package org.apache.druid.sql.calcite.external; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlCharStringLiteral; import org.apache.calcite.sql.SqlNode; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/external/ExternalOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/external/ExternalOperatorConversion.java index 3dacbea66511..f2f667af4e31 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/external/ExternalOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/external/ExternalOperatorConversion.java @@ -34,7 +34,6 @@ import org.apache.druid.segment.column.RowSignature; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Map;