From 269b9f0367fcc55d2bfb84b1ceeaa4ab8b7ba719 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Wed, 5 Feb 2020 22:34:38 -0800 Subject: [PATCH 1/3] Make Transform an ExtensionPoint --- .../main/java/org/apache/druid/segment/transform/Transform.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/processing/src/main/java/org/apache/druid/segment/transform/Transform.java b/processing/src/main/java/org/apache/druid/segment/transform/Transform.java index a12313c14d31..a481a4c08a11 100644 --- a/processing/src/main/java/org/apache/druid/segment/transform/Transform.java +++ b/processing/src/main/java/org/apache/druid/segment/transform/Transform.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.apache.druid.guice.annotations.ExtensionPoint; /** * A row transform that is part of a {@link TransformSpec}. Transforms allow adding new fields to input rows. Each @@ -34,6 +35,7 @@ * they cannot refer to other transforms. And they cannot remove fields, only add them. However, they can shadow a * field with another field containing all nulls, which will act similarly to removing the field. */ +@ExtensionPoint @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes(value = { @JsonSubTypes.Type(name = "expression", value = ExpressionTransform.class) From 118e89c1e2ae9ea0e1c923c355bff6f1ccacf756 Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Wed, 5 Feb 2020 22:53:13 -0800 Subject: [PATCH 2/3] Add transform to the list of documented extensions --- docs/development/modules.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/development/modules.md b/docs/development/modules.md index 7ef5c971920a..71068d4cf40c 100644 --- a/docs/development/modules.md +++ b/docs/development/modules.md @@ -46,6 +46,7 @@ Druid's extensions leverage Guice in order to add things at runtime. Basically, 1. Add new Jersey resources by calling `Jerseys.addResource(binder, clazz)`. 1. Add new Jetty filters by extending `org.apache.druid.server.initialization.jetty.ServletFilterHolder`. 1. Add new secret providers by extending `org.apache.druid.metadata.PasswordProvider`. +1. Add new ingest transform by implementing the `org.apache.druid.segment.transform.Transform` interface from the `druid-processing` package. 1. Bundle your extension with all the other Druid extensions Extensions are added to the system via an implementation of `org.apache.druid.initialization.DruidModule`. From b365d28f92ad9083bd72bf5724fd88561901390a Mon Sep 17 00:00:00 2001 From: Mitch Lloyd Date: Tue, 18 Feb 2020 21:28:15 -0800 Subject: [PATCH 3/3] Add example transform implementation --- docs/development/modules.md | 66 +++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/docs/development/modules.md b/docs/development/modules.md index 71068d4cf40c..e6245a20e26c 100644 --- a/docs/development/modules.md +++ b/docs/development/modules.md @@ -240,6 +240,72 @@ In your implementation of `org.apache.druid.initialization.DruidModule`, `getJac where `SomePasswordProvider` is the implementation of `PasswordProvider` interface, you can have a look at `org.apache.druid.metadata.EnvironmentVariablePasswordProvider` for example. +### Adding a Transform Extension + +To create a transform extension implement the `org.apache.druid.segment.transform.Transform` interface. You'll need to install the `druid-processing` package to import `org.apache.druid.segment.transform`. + +```java +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.segment.transform.RowFunction; +import org.apache.druid.segment.transform.Transform; + +public class MyTransform implements Transform { + private final String name; + + @JsonCreator + public MyTransform( + @JsonProperty("name") final String name + ) { + this.name = name; + } + + @JsonProperty + @Override + public String getName() { + return name; + } + + @Override + public RowFunction getRowFunction() { + return new MyRowFunction(); + } + + static class MyRowFunction implements RowFunction { + @Override + public Object eval(Row row) { + return "transformed-value"; + } + } +} +``` + +Then register your transform as a Jackson module. + +```java +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.jsontype.NamedModule; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import com.google.common.collect.ImmutableList; +import org.apache.druid.initialization.DruidModule; + +public class MyTransformModule implements DruidModule { + @Override + public List getJacksonModules() { + return return ImmutableList.of( + new SimpleModule("MyTransformModule").registerSubtypes( + new NamedType(MyTransform.class, "my-transform") + ) + ): + } + + @Override + public void configure(Binder binder) { + } +} +``` + ### Bundle your extension with all the other Druid extensions When you do `mvn install`, Druid extensions will be packaged within the Druid tarball and `extensions` directory, which are both underneath `distribution/target/`.