feat: add udfs to pinot image

Dockerfile

@@ -3,17 +3,18 @@ FROM alpine:latest AS builder
 ENV PINOT_VERSION=0.5.0
 
 RUN wget -qO- https://downloads.apache.org/incubator/pinot/apache-pinot-incubating-$PINOT_VERSION/apache-pinot-incubating-$PINOT_VERSION-bin.tar.gz | tar -xzf- && \
-    mv apache-pinot-incubating-0.5.0-bin /pinot && \
+    mv apache-pinot-incubating-$PINOT_VERSION-bin /pinot && \
     rm -rf /pinot/examples
 
-FROM openjdk:8-jdk-slim
+FROM openjdk:11-jdk-slim
 LABEL maintainer="Hypertrace https://www.hypertrace.org/"
 
 ENV PINOT_HOME=/opt/pinot
 
 VOLUME ["${PINOT_HOME}/configs", "${PINOT_HOME}/data"]
 
 COPY --from=builder /pinot ${PINOT_HOME}
+COPY build/plugins "${PINOT_HOME}/plugins"
 
 # expose ports for controller/broker/server/admin
 EXPOSE 9000 8099 8098 8097 8096 9514

build.gradle.kts

@@ -6,6 +6,28 @@ plugins {
 
 hypertraceDocker {
   defaultImage {
-    imageName.set("pinot")
+    tasks.named(buildTaskName) {
+      dependsOn("copyPlugins")
+    }
   }
-}
+}
+
+val plugins by configurations.creating
+
+dependencies {
+  plugins(project(":pinot-udf"))
+}
+
+tasks.register<Sync>("copyPlugins") {
+  from(plugins)
+  into("${buildDir}/plugins")
+}
+
+subprojects {
+  pluginManager.withPlugin("java") {
+    configure<JavaPluginExtension> {
+      sourceCompatibility = JavaVersion.VERSION_11
+      targetCompatibility = JavaVersion.VERSION_11
+    }
+  }
+}

pinot-avro-serde/build.gradle.kts

@@ -0,0 +1,14 @@
+plugins {
+  `java-library`
+}
+
+dependencies {
+  compileOnly("org.apache.pinot:pinot-spi:0.5.0")
+  compileOnly("org.apache.pinot:pinot-avro-base:0.5.0")
+  compileOnly("org.apache.kafka:kafka-streams:5.5.1-ccs")
+  compileOnly("org.apache.kafka:kafka-clients:5.5.1-ccs")
+  implementation("org.hypertrace.core.kafkastreams.framework:kafka-streams-serdes:0.1.11") {
+    // disable the transitive dependencies and use them from pinot itself.
+    isTransitive = false
+  }
+}

pinot-servicemanager/Dockerfile

@@ -7,7 +7,8 @@ FROM cimg/openjdk:14.0.2 AS install
 ARG JITPACK_USER=apache
 # Override to use a different expression, such as 'master-SNAPSHOT'
 #   Note: 0.5.0 does not have parallel start (PR 5917): 21a372b2f58f9c6e27fe9710d6952ed519342061
-ARG JITPACK_TAG=23797914c6fea24e34f3ebfca9a73e0fff72c2b7
+#   Note: See "Warning on versions" in README before upgrading version
+ARG JITPACK_TAG=1c4fc13169ff3ed8cab3a2ddcdd15270e07d74fc
 
 USER root
 WORKDIR /install

pinot-servicemanager/build.gradle.kts

@@ -1,41 +1,23 @@
 plugins {
-  `java-library`
   id("org.hypertrace.docker-publish-plugin")
 }
 
 hypertraceDocker {
   defaultImage {
-    imageName.set("pinot-servicemanager")
     tasks.named(buildTaskName) {
       dependsOn("copyPlugins")
     }
   }
 }
 
-configure<JavaPluginExtension> {
-  sourceCompatibility = JavaVersion.VERSION_11
-  targetCompatibility = JavaVersion.VERSION_11
-}
-
 val plugins by configurations.creating
 
 dependencies {
-  compileOnly("org.apache.pinot:pinot-spi:0.5.0")
-  compileOnly("org.apache.pinot:pinot-avro-base:0.5.0")
-  compileOnly("org.apache.kafka:kafka-streams:5.5.1-ccs")
-  compileOnly("org.apache.kafka:kafka-clients:5.5.1-ccs")
-  implementation("org.hypertrace.core.kafkastreams.framework:kafka-streams-serdes:0.1.11") {
-    // disable the transitive dependencies and use them from pinot itself.
-    isTransitive = false
-  }
-}
-
-dependencies {
-  plugins(project(":${project.name}"))
+  plugins(project(":pinot-udf"))
+  plugins(project(":pinot-avro-serde"))
 }
 
 tasks.register<Sync>("copyPlugins") {
   from(plugins)
   into("${buildDir}/plugins")
 }
-

pinot-servicemanager/install.sh

@@ -15,16 +15,17 @@ do
   rm -rf temp.zip classes/META-INF/license
 done
 
+# TODO: try maven-dependency-plugin:unpack instead of wget
+#       https://maven.apache.org/plugins/maven-dependency-plugin/examples/unpacking-artifacts.html
+#       https://github.com/hypertrace/pinot/issues/16
+
 # copy hypertrace plugins
 for JAR in plugins/*
 do
   unzip -qo $JAR -d classes
 done
 
 rm -rf plugins
-# TODO: try maven-dependency-plugin:unpack instead of wget
-#       https://maven.apache.org/plugins/maven-dependency-plugin/examples/unpacking-artifacts.html
-#       https://github.com/hypertrace/pinot/issues/16
 
 mkdir etc
 

pinot-udf/README.md

@@ -0,0 +1,20 @@
+# Pinot UDFs
+
+## Requirements
+These UDFs should have a minimal dependency tree and be compiled to support a java target no
+higher than our pinot images use as a runtime. Currently, that means java 11. Any common dependencies
+in the dependency tree (which can be seen via `gradlew dependencies --configuration runtimeClasspath`)
+must be shaded, so should be avoided.
+
+## Warning on versions
+As of [2379791](https://github.com/apache/incubator-pinot/commit/23797914c6fea24e34f3ebfca9a73e0fff72c2b7)
+(between 0.5.0 and 0.6.0) pinot introduces a breaking change moving the ScalarFunction annotation to
+a new packacge. Because these UDFs are used by both the main pinot image and the
+pinot-servicemanager image, both images must use either an earlier version or a later version
+(although they should be in sync regardless), and the compile dependency on pinot in this module
+should reflect that same version.
+
+## Rationale
+This UDF jar is added as a plugin to the pinot images built in this repository. The UDFs are used to
+support projected attributes, where we support creating new attributes as read time-only projections,
+using a combination of custom UDFs and those built in to pinot.

pinot-udf/build.gradle.kts

@@ -0,0 +1,8 @@
+plugins {
+  `java-library`
+}
+
+dependencies {
+  implementation("org.hypertrace.core.attribute.service:attribute-projection-functions:0.4.1")
+  compileOnly("org.apache.pinot:pinot-common:0.5.0")
+}

pinot-udf/src/main/java/org/apache/pinot/scalar/function/hypertrace/HypertraceScalarFunctions.java

@@ -0,0 +1,34 @@
+package org.apache.pinot.scalar.function.hypertrace;
+
+import static java.util.Objects.isNull;
+
+import org.apache.pinot.common.function.annotations.ScalarFunction;
+import org.hypertrace.core.attribute.service.projection.functions.DefaultValue;
+import org.hypertrace.core.attribute.service.projection.functions.Hash;
+
+public class HypertraceScalarFunctions {
+
+  private static final String NULL_STRING = "null";
+  public static final String HASH_FUNCTION_NAME = "hash";
+  public static final String DEFAULT_STRING_FUNCTION_NAME = "defaultString";
+
+  @ScalarFunction(name = HASH_FUNCTION_NAME)
+  public static String hash(String value) {
+    return replaceNullWithNullString(Hash.hash(replaceNullStringWithNull(value)));
+  }
+
+  @ScalarFunction(name = DEFAULT_STRING_FUNCTION_NAME)
+  public static String defaultString(String value, String defaultValue) {
+    return replaceNullWithNullString(
+        DefaultValue.defaultString(
+            replaceNullStringWithNull(value), replaceNullStringWithNull(defaultValue)));
+  }
+
+  private static String replaceNullStringWithNull(String value) {
+    return NULL_STRING.equals(value) ? null : value;
+  }
+
+  private static String replaceNullWithNullString(String value) {
+    return isNull(value) ? NULL_STRING : value;
+  }
+}

settings.gradle.kts

@@ -11,3 +11,5 @@ plugins {
 }
 
 include(":pinot-servicemanager")
+include(":pinot-udf")
+include(":pinot-avro-serde")