support for array expressions in TransformSpec with ExpressionTransform

core/src/main/java/org/apache/druid/data/input/impl/CSVParseSpec.java

@@ -57,7 +57,6 @@ public CSVParseSpec(
       for (String column : columns) {
         Preconditions.checkArgument(!column.contains(","), "Column[%s] has a comma, it cannot", column);
       }
-      verify(dimensionsSpec.getDimensionNames());
     } else {
       Preconditions.checkArgument(
           hasHeaderRow,
@@ -102,14 +101,6 @@ public int getSkipHeaderRows()
     return skipHeaderRows;
   }
 
-  @Override
-  public void verify(List<String> usedCols)
-  {
-    for (String columnName : usedCols) {
-      Preconditions.checkArgument(columns.contains(columnName), "column[%s] not in columns.", columnName);
-    }
-  }
-
   @Override
   public Parser<String, Object> makeParser()
   {

core/src/main/java/org/apache/druid/data/input/impl/DelimitedParseSpec.java

@@ -60,7 +60,6 @@ public DelimitedParseSpec(
       for (String column : this.columns) {
         Preconditions.checkArgument(!column.contains(","), "Column[%s] has a comma, it cannot", column);
       }
-      verify(dimensionsSpec.getDimensionNames());
     } else {
       Preconditions.checkArgument(
           hasHeaderRow,
@@ -112,14 +111,6 @@ public int getSkipHeaderRows()
     return skipHeaderRows;
   }
 
-  @Override
-  public void verify(List<String> usedCols)
-  {
-    for (String columnName : usedCols) {
-      Preconditions.checkArgument(columns.contains(columnName), "column[%s] not in columns.", columnName);
-    }
-  }
-
   @Override
   public Parser<String, Object> makeParser()
   {

core/src/main/java/org/apache/druid/data/input/impl/JSONLowercaseParseSpec.java

@@ -25,8 +25,6 @@
 import org.apache.druid.java.util.common.parsers.JSONToLowerParser;
 import org.apache.druid.java.util.common.parsers.Parser;
 
-import java.util.List;
-
 /**
  * This class is only here for backwards compatibility
  */
@@ -45,11 +43,6 @@ public JSONLowercaseParseSpec(
     this.objectMapper = new ObjectMapper();
   }
 
-  @Override
-  public void verify(List<String> usedCols)
-  {
-  }
-
   @Override
   public Parser<String, Object> makeParser()
   {

core/src/main/java/org/apache/druid/data/input/impl/JSONParseSpec.java

@@ -28,7 +28,6 @@
 import org.apache.druid.java.util.common.parsers.Parser;
 
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
@@ -62,11 +61,6 @@ public JSONParseSpec(TimestampSpec ts, DimensionsSpec dims)
     this(ts, dims, null, null);
   }
 
-  @Override
-  public void verify(List<String> usedCols)
-  {
-  }
-
   @Override
   public Parser<String, Object> makeParser()
   {

core/src/main/java/org/apache/druid/data/input/impl/JavaScriptParseSpec.java

@@ -27,8 +27,6 @@
 import org.apache.druid.java.util.common.parsers.Parser;
 import org.apache.druid.js.JavaScriptConfig;
 
-import java.util.List;
-
 /**
  */
 public class JavaScriptParseSpec extends ParseSpec
@@ -59,11 +57,6 @@ public String getFunction()
     return function;
   }
 
-  @Override
-  public void verify(List<String> usedCols)
-  {
-  }
-
   @Override
   public Parser<String, Object> makeParser()
   {

core/src/main/java/org/apache/druid/data/input/impl/ParseSpec.java

@@ -27,8 +27,6 @@
 import org.apache.druid.guice.annotations.PublicApi;
 import org.apache.druid.java.util.common.parsers.Parser;
 
-import java.util.List;
-
 @ExtensionPoint
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "format")
 @JsonSubTypes(value = {
@@ -64,12 +62,6 @@ public DimensionsSpec getDimensionsSpec()
     return dimensionsSpec;
   }
 
-  @PublicApi
-  public void verify(List<String> usedCols)
-  {
-    // do nothing
-  }
-
   public Parser<String, Object> makeParser()
   {
     return null;

core/src/main/java/org/apache/druid/data/input/impl/RegexParseSpec.java

@@ -22,7 +22,6 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
 import org.apache.druid.java.util.common.parsers.Parser;
 import org.apache.druid.java.util.common.parsers.RegexParser;
 
@@ -50,8 +49,6 @@ public RegexParseSpec(
     this.listDelimiter = listDelimiter;
     this.columns = columns;
     this.pattern = pattern;
-
-    verify(dimensionsSpec.getDimensionNames());
   }
 
   @JsonProperty
@@ -72,16 +69,6 @@ public List<String> getColumns()
     return columns;
   }
 
-  @Override
-  public void verify(List<String> usedCols)
-  {
-    if (columns != null) {
-      for (String columnName : usedCols) {
-        Preconditions.checkArgument(columns.contains(columnName), "column[%s] not in columns.", columnName);
-      }
-    }
-  }
-
   @Override
   public Parser<String, Object> makeParser()
   {

core/src/main/java/org/apache/druid/java/util/common/StringUtils.java

@@ -112,7 +112,7 @@ public static byte[] toUtf8(final String string)
 
   /**
    * Encodes "string" into the buffer "byteBuffer", using no more than the number of bytes remaining in the buffer.
-   * Will only encode whole characters. The byteBuffer's position and limit be changed during operation, but will
+   * Will only encode whole characters. The byteBuffer's position and limit may be changed during operation, but will
    * be reset before this method call ends.
    *
    * @return the number of bytes written, which may be shorter than the full encoded string length if there

core/src/test/java/org/apache/druid/data/input/impl/CSVParseSpecTest.java

@@ -27,28 +27,6 @@
 
 public class CSVParseSpecTest
 {
-  @Test(expected = IllegalArgumentException.class)
-  public void testColumnMissing()
-  {
-    @SuppressWarnings("unused") // expected exception
-    final ParseSpec spec = new CSVParseSpec(
-        new TimestampSpec(
-            "timestamp",
-            "auto",
-            null
-        ),
-        new DimensionsSpec(
-            DimensionsSpec.getDefaultSchemas(Arrays.asList("a", "b")),
-            new ArrayList<>(),
-            new ArrayList<>()
-        ),
-        ",",
-        Collections.singletonList("a"),
-        false,
-        0
-    );
-  }
-
   @Test(expected = IllegalArgumentException.class)
   public void testComma()
   {
@@ -65,7 +43,7 @@ public void testComma()
             new ArrayList<>()
         ),
         ",",
-        Collections.singletonList("a"),
+        Collections.singletonList("a,"),
         false,
         0
     );

core/src/test/java/org/apache/druid/data/input/impl/DelimitedParseSpecTest.java

@@ -58,29 +58,6 @@ public void testSerde() throws IOException
     Assert.assertEquals(Collections.singletonList("abc"), serde.getDimensionsSpec().getDimensionNames());
   }
 
-  @Test(expected = IllegalArgumentException.class)
-  public void testColumnMissing()
-  {
-    @SuppressWarnings("unused") // expected exception
-    final ParseSpec spec = new DelimitedParseSpec(
-        new TimestampSpec(
-            "timestamp",
-            "auto",
-            null
-        ),
-        new DimensionsSpec(
-            DimensionsSpec.getDefaultSchemas(Arrays.asList("a", "b")),
-            new ArrayList<>(),
-            new ArrayList<>()
-        ),
-        ",",
-        " ",
-        Collections.singletonList("a"),
-        false,
-        0
-    );
-  }
-
   @Test(expected = IllegalArgumentException.class)
   public void testComma()
   {
@@ -98,7 +75,7 @@ public void testComma()
         ),
         ",",
         null,
-        Collections.singletonList("a"),
+        Collections.singletonList("a,"),
         false,
         0
     );

docs/misc/math-expr.md

@@ -60,9 +60,6 @@ dialects. However, by using the `array_to_string` function, aggregations may be
 complete array, allowing the complete row to be preserved. Using `string_to_array` in an expression post-aggregator,
 allows transforming the stringified dimension back into the true native array type.
 
-> Note that array functions are not currently supported at ingestion time with
-> [`transformSpec`](../ingestion/index.md#transformspec).
-
 
 The following built-in functions are available.
 

indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java

@@ -101,6 +101,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -237,9 +238,9 @@ public void testTransformSpec() throws Exception
     File tmpFile = File.createTempFile("druid", "index", tmpDir);
 
     try (BufferedWriter writer = Files.newWriter(tmpFile, StandardCharsets.UTF_8)) {
-      writer.write("2014-01-01T00:00:10Z,a,1\n");
-      writer.write("2014-01-01T01:00:20Z,b,1\n");
-      writer.write("2014-01-01T02:00:30Z,c,1\n");
+      writer.write("2014-01-01T00:00:10Z,a,an|array,1|2|3,1\n");
+      writer.write("2014-01-01T01:00:20Z,b,another|array,3|4,1\n");
+      writer.write("2014-01-01T02:00:30Z,c,and|another,0|1,1\n");
     }
 
     IndexTask indexTask = new IndexTask(
@@ -248,11 +249,40 @@ public void testTransformSpec() throws Exception
         createIngestionSpec(
             jsonMapper,
             tmpDir,
-            null,
+            new CSVParseSpec(
+                new TimestampSpec(
+                    "ts",
+                    "auto",
+                    null
+                ),
+                new DimensionsSpec(
+                    DimensionsSpec.getDefaultSchemas(
+                        Arrays.asList(
+                            "ts",
+                            "dim",
+                            "dim_array",
+                            "dim_num_array",
+                            "dimt",
+                            "dimtarray1",
+                            "dimtarray2",
+                            "dimtnum_array"
+                        )
+                    ),
+                    new ArrayList<>(),
+                    new ArrayList<>()
+                ),
+                "|",
+                Arrays.asList("ts", "dim", "dim_array", "dim_num_array", "val"),
+                false,
+                0
+            ),
             new TransformSpec(
                 new SelectorDimFilter("dim", "b", null),
                 ImmutableList.of(
-                    new ExpressionTransform("dimt", "concat(dim,dim)", ExprMacroTable.nil())
+                    new ExpressionTransform("dimt", "concat(dim,dim)", ExprMacroTable.nil()),
+                    new ExpressionTransform("dimtarray1", "array(dim, dim)", ExprMacroTable.nil()),
+                    new ExpressionTransform("dimtarray2", "map(d -> concat(d, 'foo'), dim_array)", ExprMacroTable.nil()),
+                    new ExpressionTransform("dimtnum_array", "map(d -> d + 3, dim_num_array)", ExprMacroTable.nil())
                 )
             ),
             null,
@@ -271,6 +301,47 @@ public void testTransformSpec() throws Exception
     final List<DataSegment> segments = runTask(indexTask).rhs;
 
     Assert.assertEquals(1, segments.size());
+    DataSegment segment = segments.get(0);
+    final File segmentFile = segmentLoader.getSegmentFiles(segment);
+
+    final WindowedStorageAdapter adapter = new WindowedStorageAdapter(
+        new QueryableIndexStorageAdapter(indexIO.loadIndex(segmentFile)),
+        segment.getInterval()
+    );
+    final Sequence<Cursor> cursorSequence = adapter.getAdapter().makeCursors(
+        null,
+        segment.getInterval(),
+        VirtualColumns.EMPTY,
+        Granularities.ALL,
+        false,
+        null
+    );
+    final List<Map<String, Object>> transforms = cursorSequence
+        .map(cursor -> {
+          final DimensionSelector selector1 = cursor.getColumnSelectorFactory()
+                                                   .makeDimensionSelector(new DefaultDimensionSpec("dimt", "dimt"));
+          final DimensionSelector selector2 = cursor.getColumnSelectorFactory()
+                                                    .makeDimensionSelector(new DefaultDimensionSpec("dimtarray1", "dimtarray1"));
+          final DimensionSelector selector3 = cursor.getColumnSelectorFactory()
+                                                   .makeDimensionSelector(new DefaultDimensionSpec("dimtarray2", "dimtarray2"));
+          final DimensionSelector selector4 = cursor.getColumnSelectorFactory()
+                                                    .makeDimensionSelector(new DefaultDimensionSpec("dimtnum_array", "dimtnum_array"));
+
+
+          Map<String, Object> row = new HashMap<>();
+          row.put("dimt", selector1.defaultGetObject());
+          row.put("dimtarray1", selector2.defaultGetObject());
+          row.put("dimtarray2", selector3.defaultGetObject());
+          row.put("dimtnum_array", selector4.defaultGetObject());
+          cursor.advance();
+          return row;
+        })
+        .toList();
+    Assert.assertEquals(1, transforms.size());
+    Assert.assertEquals("bb", transforms.get(0).get("dimt"));
+    Assert.assertEquals(ImmutableList.of("b", "b"), transforms.get(0).get("dimtarray1"));
+    Assert.assertEquals(ImmutableList.of("anotherfoo", "arrayfoo"), transforms.get(0).get("dimtarray2"));
+    Assert.assertEquals(ImmutableList.of("6.0", "7.0"), transforms.get(0).get("dimtnum_array"));
 
     Assert.assertEquals("test", segments.get(0).getDataSource());
     Assert.assertEquals(Intervals.of("2014/P1D"), segments.get(0).getInterval());