Update doc for allowedHeaders

docs/configuration/index.md

@@ -616,9 +616,10 @@ the [HDFS input source](../ingestion/input-sources.md#hdfs-input-source).
 You can set the following property to specify permissible protocols for
 the [HTTP input source](../ingestion/input-sources.md#http-input-source).
 
-|Property|Possible values|Description|Default|
-|--------|---------------|-----------|-------|
-|`druid.ingestion.http.allowedProtocols`|List of protocols|Allowed protocols for the HTTP input source.|`["http", "https"]`|
+|Property| Possible values                                                                                                                                                  | Description                                                                                                                               |Default|
+|--------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|-------|
+|`druid.ingestion.http.allowedProtocols`| List of protocols                                                                                                                                                | Allowed protocols for the HTTP input source.                                                                                              |`["http", "https"]`|
+|`druid.ingestion.http.allowedHeaders`| A list of permitted request headers for the HTTP input source. By default, the list is empty, which means no headers are allowed in the ingestion specification. |`[]`|
 
 ### External data access security configuration
 

processing/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java

@@ -100,13 +100,11 @@ public static void throwIfInvalidProtocols(HttpInputSourceConfig config, List<UR
 
   public static void throwIfForbiddenHeaders(HttpInputSourceConfig config, Map<String, String> requestHeaders)
   {
-    if (config.getAllowedHeaders().size() > 0) {
-      for (Map.Entry<String, String> entry : requestHeaders.entrySet()) {
-        if (!config.getAllowedHeaders().contains(StringUtils.toLowerCase(entry.getKey()))) {
-          throw InvalidInput.exception("Got forbidden header %s, allowed headers are only %s ",
-                                       entry.getKey(), config.getAllowedHeaders()
-          );
-        }
+    for (Map.Entry<String, String> entry : requestHeaders.entrySet()) {
+      if (!config.getAllowedHeaders().contains(StringUtils.toLowerCase(entry.getKey()))) {
+        throw InvalidInput.exception("Got forbidden header [%s], allowed headers are only [%s]. You can control the allowed headers by updating druid.ingestion.http.allowedHeaders",
+                                     entry.getKey(), config.getAllowedHeaders()
+        );
       }
     }
   }

processing/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java

@@ -30,7 +30,6 @@
 import org.apache.druid.data.input.impl.systemfield.SystemField;
 import org.apache.druid.data.input.impl.systemfield.SystemFields;
 import org.apache.druid.error.DruidException;
-import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.metadata.DefaultPasswordProvider;
 import org.junit.Assert;
 import org.junit.Rule;
@@ -41,8 +40,7 @@
 import java.net.URI;
 import java.util.Collections;
 import java.util.EnumSet;
-import java.util.Set;
-import java.util.stream.Collectors;
+import java.util.HashSet;
 
 public class HttpInputSourceTest
 {
@@ -152,12 +150,17 @@ public void testSystemFields()
   }
 
   @Test
-  public void testAllowedHeaders()
+  public void testEmptyAllowedHeaders()
   {
     HttpInputSourceConfig httpInputSourceConfig = new HttpInputSourceConfig(
         null,
-        Sets.newHashSet("R-cookie", "Content-type")
+        new HashSet<>()
     );
+    expectedException.expect(DruidException.class);
+    expectedException.expectMessage(
+        "Got forbidden header [r-Cookie], allowed headers are only [[]]. "
+        + "You can control the allowed headers by updating druid.ingestion.http.allowedHeaders");
+
     final HttpInputSource inputSource = new HttpInputSource(
         ImmutableList.of(URI.create("http://test.com/http-test")),
         "myName",
@@ -166,12 +169,6 @@ public void testAllowedHeaders()
         ImmutableMap.of("r-Cookie", "test", "Content-Type", "application/json"),
         httpInputSourceConfig
     );
-    Set<String> expectedSet = inputSource.getRequestHeaders()
-                                         .keySet()
-                                         .stream()
-                                         .map(StringUtils::toLowerCase)
-                                         .collect(Collectors.toSet());
-    Assert.assertEquals(expectedSet, httpInputSourceConfig.getAllowedHeaders());
   }
 
   @Test
@@ -183,7 +180,7 @@ public void shouldFailOnForbiddenHeaders()
     );
     expectedException.expect(DruidException.class);
     expectedException.expectMessage(
-        "Got forbidden header G-Cookie, allowed headers are only [r-cookie, content-type]");
+        "Got forbidden header [G-Cookie], allowed headers are only [[r-cookie, content-type]]");
     new HttpInputSource(
         ImmutableList.of(URI.create("http://test.com/http-test")),
         "myName",

sql/src/test/java/org/apache/druid/sql/calcite/IngestTableFunctionTest.java

@@ -20,9 +20,14 @@
 package org.apache.druid.sql.calcite;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.Module;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import com.google.inject.Binder;
 import org.apache.calcite.avatica.SqlType;
 import org.apache.druid.catalog.model.Columns;
 import org.apache.druid.data.input.impl.CsvInputFormat;
@@ -31,20 +36,28 @@
 import org.apache.druid.data.input.impl.JsonInputFormat;
 import org.apache.druid.data.input.impl.LocalInputSource;
 import org.apache.druid.data.input.impl.systemfield.SystemFields;
+import org.apache.druid.guice.DruidInjectorBuilder;
+import org.apache.druid.initialization.DruidModule;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.UOE;
 import org.apache.druid.metadata.DefaultPasswordProvider;
+import org.apache.druid.metadata.input.InputSourceModule;
 import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.RowSignature;
 import org.apache.druid.server.security.Access;
 import org.apache.druid.server.security.AuthConfig;
 import org.apache.druid.server.security.ForbiddenException;
 import org.apache.druid.sql.calcite.external.ExternalDataSource;
+import org.apache.druid.sql.calcite.external.ExternalOperatorConversion;
 import org.apache.druid.sql.calcite.external.Externals;
+import org.apache.druid.sql.calcite.external.HttpOperatorConversion;
+import org.apache.druid.sql.calcite.external.InlineOperatorConversion;
+import org.apache.druid.sql.calcite.external.LocalOperatorConversion;
 import org.apache.druid.sql.calcite.filtration.Filtration;
 import org.apache.druid.sql.calcite.planner.Calcites;
 import org.apache.druid.sql.calcite.util.CalciteTests;
+import org.apache.druid.sql.guice.SqlBindings;
 import org.apache.druid.sql.http.SqlParameter;
 import org.hamcrest.CoreMatchers;
 import org.junit.internal.matchers.ThrowableMessageMatcher;
@@ -53,8 +66,10 @@
 import java.io.File;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -69,6 +84,7 @@
  * query ensure that the resulting MSQ task is identical regardless of the path
  * taken.
  */
+@SqlTestFrameworkConfig.ComponentSupplier(IngestTableFunctionTest.ExportComponentSupplier.class)
 public class IngestTableFunctionTest extends CalciteIngestionDmlTest
 {
   protected static URI toURI(String uri)
@@ -97,6 +113,20 @@ protected static URI toURI(String uri)
                   .add("z", ColumnType.LONG)
                   .build()
   );
+  protected final ExternalDataSource localDataSource = new ExternalDataSource(
+      new LocalInputSource(
+          null,
+          null,
+          Arrays.asList(new File("/tmp/foo.csv"), new File("/tmp/bar.csv")),
+          SystemFields.none()
+      ),
+      new CsvInputFormat(ImmutableList.of("x", "y", "z"), null, false, false, 0),
+      RowSignature.builder()
+                  .add("x", ColumnType.STRING)
+                  .add("y", ColumnType.STRING)
+                  .add("z", ColumnType.LONG)
+                  .build()
+  );
 
   /**
    * Basic use of EXTERN
@@ -262,7 +292,7 @@ public void testHttpFn2()
             new DefaultPasswordProvider("secret"),
             SystemFields.none(),
             ImmutableMap.of("Accept", "application/ndjson", "a", "b"),
-            new HttpInputSourceConfig(null, null)
+            new HttpInputSourceConfig(null, Sets.newHashSet("Accept", "a"))
         ),
         new CsvInputFormat(ImmutableList.of("timestamp", "isRobot"), null, false, false, 0),
         RowSignature.builder()
@@ -549,21 +579,6 @@ public void testInlineFn()
         .verify();
   }
 
-  protected final ExternalDataSource localDataSource = new ExternalDataSource(
-      new LocalInputSource(
-          null,
-          null,
-          Arrays.asList(new File("/tmp/foo.csv"), new File("/tmp/bar.csv")),
-          SystemFields.none()
-      ),
-      new CsvInputFormat(ImmutableList.of("x", "y", "z"), null, false, false, 0),
-      RowSignature.builder()
-                  .add("x", ColumnType.STRING)
-                  .add("y", ColumnType.STRING)
-                  .add("z", ColumnType.LONG)
-                  .build()
-  );
-
   /**
    * Basic use of LOCALFILES
    */
@@ -702,4 +717,66 @@ public void testLocalFnNotNull()
         .expectLogicalPlanFrom("localExtern")
         .verify();
   }
+
+  protected static class ExportComponentSupplier extends IngestionDmlComponentSupplier
+  {
+    public ExportComponentSupplier(TempDirProducer tempFolderProducer)
+    {
+      super(tempFolderProducer);
+    }
+
+    @Override
+    public void configureGuice(DruidInjectorBuilder builder)
+    {
+      builder.addModule(new DruidModule()
+      {
+
+        // Clone of MSQExternalDataSourceModule since it is not
+        // visible here.
+        @Override
+        public List<? extends Module> getJacksonModules()
+        {
+          return Collections.singletonList(
+              new SimpleModule(getClass().getSimpleName())
+                  .registerSubtypes(ExternalDataSource.class)
+          );
+        }
+
+        @Override
+        public void configure(Binder binder)
+        {
+          // Adding the config to allow following 2 headers.
+          binder.bind(HttpInputSourceConfig.class)
+                .toInstance(new HttpInputSourceConfig(null, ImmutableSet.of("Accept", "a")));
+
+        }
+      });
+
+      builder.addModule(new DruidModule()
+      {
+
+        @Override
+        public List<? extends Module> getJacksonModules()
+        {
+          // We want this module to bring input sources along for the ride.
+          List<Module> modules = new ArrayList<>(new InputSourceModule().getJacksonModules());
+          modules.add(new SimpleModule("test-module").registerSubtypes(TestFileInputSource.class));
+          return modules;
+        }
+
+        @Override
+        public void configure(Binder binder)
+        {
+          // Set up the EXTERN macro.
+          SqlBindings.addOperatorConversion(binder, ExternalOperatorConversion.class);
+
+          // Enable the extended table functions for testing even though these
+          // are not enabled in production in Druid 26.
+          SqlBindings.addOperatorConversion(binder, HttpOperatorConversion.class);
+          SqlBindings.addOperatorConversion(binder, InlineOperatorConversion.class);
+          SqlBindings.addOperatorConversion(binder, LocalOperatorConversion.class);
+        }
+      });
+    }
+  }
 }