diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 57b88b55fc21..5fd31366ad0f 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -616,9 +616,10 @@ the [HDFS input source](../ingestion/input-sources.md#hdfs-input-source). You can set the following property to specify permissible protocols for the [HTTP input source](../ingestion/input-sources.md#http-input-source). -|Property|Possible values|Description|Default| -|--------|---------------|-----------|-------| -|`druid.ingestion.http.allowedProtocols`|List of protocols|Allowed protocols for the HTTP input source.|`["http", "https"]`| +|Property| Possible values | Description |Default| +|--------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|-------| +|`druid.ingestion.http.allowedProtocols`| List of protocols | Allowed protocols for the HTTP input source. |`["http", "https"]`| +|`druid.ingestion.http.allowedHeaders`| A list of permitted request headers for the HTTP input source. By default, the list is empty, which means no headers are allowed in the ingestion specification. |`[]`| ### External data access security configuration diff --git a/processing/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java b/processing/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java index 12f2316fb67d..0c4c9197e5fd 100644 --- a/processing/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java +++ b/processing/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java @@ -100,13 +100,11 @@ public static void throwIfInvalidProtocols(HttpInputSourceConfig config, List requestHeaders) { - if (config.getAllowedHeaders().size() > 0) { - for (Map.Entry entry : requestHeaders.entrySet()) { - if (!config.getAllowedHeaders().contains(StringUtils.toLowerCase(entry.getKey()))) { - throw InvalidInput.exception("Got forbidden header %s, allowed headers are only %s ", - entry.getKey(), config.getAllowedHeaders() - ); - } + for (Map.Entry entry : requestHeaders.entrySet()) { + if (!config.getAllowedHeaders().contains(StringUtils.toLowerCase(entry.getKey()))) { + throw InvalidInput.exception("Got forbidden header [%s], allowed headers are only [%s]. You can control the allowed headers by updating druid.ingestion.http.allowedHeaders", + entry.getKey(), config.getAllowedHeaders() + ); } } } diff --git a/processing/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java b/processing/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java index 118b56838b6c..c11cb96f34d4 100644 --- a/processing/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java +++ b/processing/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java @@ -30,7 +30,6 @@ import org.apache.druid.data.input.impl.systemfield.SystemField; import org.apache.druid.data.input.impl.systemfield.SystemFields; import org.apache.druid.error.DruidException; -import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.metadata.DefaultPasswordProvider; import org.junit.Assert; import org.junit.Rule; @@ -41,8 +40,7 @@ import java.net.URI; import java.util.Collections; import java.util.EnumSet; -import java.util.Set; -import java.util.stream.Collectors; +import java.util.HashSet; public class HttpInputSourceTest { @@ -152,12 +150,17 @@ public void testSystemFields() } @Test - public void testAllowedHeaders() + public void testEmptyAllowedHeaders() { HttpInputSourceConfig httpInputSourceConfig = new HttpInputSourceConfig( null, - Sets.newHashSet("R-cookie", "Content-type") + new HashSet<>() ); + expectedException.expect(DruidException.class); + expectedException.expectMessage( + "Got forbidden header [r-Cookie], allowed headers are only [[]]. " + + "You can control the allowed headers by updating druid.ingestion.http.allowedHeaders"); + final HttpInputSource inputSource = new HttpInputSource( ImmutableList.of(URI.create("http://test.com/http-test")), "myName", @@ -166,12 +169,6 @@ public void testAllowedHeaders() ImmutableMap.of("r-Cookie", "test", "Content-Type", "application/json"), httpInputSourceConfig ); - Set expectedSet = inputSource.getRequestHeaders() - .keySet() - .stream() - .map(StringUtils::toLowerCase) - .collect(Collectors.toSet()); - Assert.assertEquals(expectedSet, httpInputSourceConfig.getAllowedHeaders()); } @Test @@ -183,7 +180,7 @@ public void shouldFailOnForbiddenHeaders() ); expectedException.expect(DruidException.class); expectedException.expectMessage( - "Got forbidden header G-Cookie, allowed headers are only [r-cookie, content-type]"); + "Got forbidden header [G-Cookie], allowed headers are only [[r-cookie, content-type]]"); new HttpInputSource( ImmutableList.of(URI.create("http://test.com/http-test")), "myName", diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/IngestTableFunctionTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/IngestTableFunctionTest.java index 7401477e6d79..9a1a79dea4d6 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/IngestTableFunctionTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/IngestTableFunctionTest.java @@ -20,9 +20,14 @@ package org.apache.druid.sql.calcite; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.google.inject.Binder; import org.apache.calcite.avatica.SqlType; import org.apache.druid.catalog.model.Columns; import org.apache.druid.data.input.impl.CsvInputFormat; @@ -31,20 +36,28 @@ import org.apache.druid.data.input.impl.JsonInputFormat; import org.apache.druid.data.input.impl.LocalInputSource; import org.apache.druid.data.input.impl.systemfield.SystemFields; +import org.apache.druid.guice.DruidInjectorBuilder; +import org.apache.druid.initialization.DruidModule; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.UOE; import org.apache.druid.metadata.DefaultPasswordProvider; +import org.apache.druid.metadata.input.InputSourceModule; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.ForbiddenException; import org.apache.druid.sql.calcite.external.ExternalDataSource; +import org.apache.druid.sql.calcite.external.ExternalOperatorConversion; import org.apache.druid.sql.calcite.external.Externals; +import org.apache.druid.sql.calcite.external.HttpOperatorConversion; +import org.apache.druid.sql.calcite.external.InlineOperatorConversion; +import org.apache.druid.sql.calcite.external.LocalOperatorConversion; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.planner.Calcites; import org.apache.druid.sql.calcite.util.CalciteTests; +import org.apache.druid.sql.guice.SqlBindings; import org.apache.druid.sql.http.SqlParameter; import org.hamcrest.CoreMatchers; import org.junit.internal.matchers.ThrowableMessageMatcher; @@ -53,8 +66,10 @@ import java.io.File; import java.net.URI; import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.List; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -69,6 +84,7 @@ * query ensure that the resulting MSQ task is identical regardless of the path * taken. */ +@SqlTestFrameworkConfig.ComponentSupplier(IngestTableFunctionTest.ExportComponentSupplier.class) public class IngestTableFunctionTest extends CalciteIngestionDmlTest { protected static URI toURI(String uri) @@ -97,6 +113,20 @@ protected static URI toURI(String uri) .add("z", ColumnType.LONG) .build() ); + protected final ExternalDataSource localDataSource = new ExternalDataSource( + new LocalInputSource( + null, + null, + Arrays.asList(new File("/tmp/foo.csv"), new File("/tmp/bar.csv")), + SystemFields.none() + ), + new CsvInputFormat(ImmutableList.of("x", "y", "z"), null, false, false, 0), + RowSignature.builder() + .add("x", ColumnType.STRING) + .add("y", ColumnType.STRING) + .add("z", ColumnType.LONG) + .build() + ); /** * Basic use of EXTERN @@ -262,7 +292,7 @@ public void testHttpFn2() new DefaultPasswordProvider("secret"), SystemFields.none(), ImmutableMap.of("Accept", "application/ndjson", "a", "b"), - new HttpInputSourceConfig(null, null) + new HttpInputSourceConfig(null, Sets.newHashSet("Accept", "a")) ), new CsvInputFormat(ImmutableList.of("timestamp", "isRobot"), null, false, false, 0), RowSignature.builder() @@ -549,21 +579,6 @@ public void testInlineFn() .verify(); } - protected final ExternalDataSource localDataSource = new ExternalDataSource( - new LocalInputSource( - null, - null, - Arrays.asList(new File("/tmp/foo.csv"), new File("/tmp/bar.csv")), - SystemFields.none() - ), - new CsvInputFormat(ImmutableList.of("x", "y", "z"), null, false, false, 0), - RowSignature.builder() - .add("x", ColumnType.STRING) - .add("y", ColumnType.STRING) - .add("z", ColumnType.LONG) - .build() - ); - /** * Basic use of LOCALFILES */ @@ -702,4 +717,66 @@ public void testLocalFnNotNull() .expectLogicalPlanFrom("localExtern") .verify(); } + + protected static class ExportComponentSupplier extends IngestionDmlComponentSupplier + { + public ExportComponentSupplier(TempDirProducer tempFolderProducer) + { + super(tempFolderProducer); + } + + @Override + public void configureGuice(DruidInjectorBuilder builder) + { + builder.addModule(new DruidModule() + { + + // Clone of MSQExternalDataSourceModule since it is not + // visible here. + @Override + public List getJacksonModules() + { + return Collections.singletonList( + new SimpleModule(getClass().getSimpleName()) + .registerSubtypes(ExternalDataSource.class) + ); + } + + @Override + public void configure(Binder binder) + { + // Adding the config to allow following 2 headers. + binder.bind(HttpInputSourceConfig.class) + .toInstance(new HttpInputSourceConfig(null, ImmutableSet.of("Accept", "a"))); + + } + }); + + builder.addModule(new DruidModule() + { + + @Override + public List getJacksonModules() + { + // We want this module to bring input sources along for the ride. + List modules = new ArrayList<>(new InputSourceModule().getJacksonModules()); + modules.add(new SimpleModule("test-module").registerSubtypes(TestFileInputSource.class)); + return modules; + } + + @Override + public void configure(Binder binder) + { + // Set up the EXTERN macro. + SqlBindings.addOperatorConversion(binder, ExternalOperatorConversion.class); + + // Enable the extended table functions for testing even though these + // are not enabled in production in Druid 26. + SqlBindings.addOperatorConversion(binder, HttpOperatorConversion.class); + SqlBindings.addOperatorConversion(binder, InlineOperatorConversion.class); + SqlBindings.addOperatorConversion(binder, LocalOperatorConversion.class); + } + }); + } + } }