From 3526556ddb278c2537a5b5f6e3e735cd04ebc3bf Mon Sep 17 00:00:00 2001 From: Aaron Steinfeld Date: Fri, 11 Sep 2020 18:00:13 -0400 Subject: [PATCH 1/4] feat: add udfs to pinot image --- build.gradle.kts | 1 + pinot-servicemanager/Dockerfile | 2 ++ pinot-servicemanager/build.gradle.kts | 13 +++++++ .../docker-bin/start-servicemanager | 2 +- pinot-udf/build.gradle.kts | 8 +++++ .../hypertrace/HypertraceScalarFunctions.java | 34 +++++++++++++++++++ settings.gradle.kts | 1 + 7 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 pinot-udf/build.gradle.kts create mode 100644 pinot-udf/src/main/java/org/apache/pinot/scalar/function/hypertrace/HypertraceScalarFunctions.java diff --git a/build.gradle.kts b/build.gradle.kts index 4abf39e..6808f22 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,6 +1,7 @@ plugins { id("org.hypertrace.docker-plugin") version "0.7.1" id("org.hypertrace.docker-publish-plugin") version "0.7.1" + id("org.hypertrace.repository-plugin") version "0.2.3" } hypertraceDocker { diff --git a/pinot-servicemanager/Dockerfile b/pinot-servicemanager/Dockerfile index a17e810..93f7b96 100644 --- a/pinot-servicemanager/Dockerfile +++ b/pinot-servicemanager/Dockerfile @@ -41,6 +41,8 @@ USER ${USER} # Copy binaries and config we installed earlier COPY --from=install --chown=${USER} /install . +COPY build/plugins plugins/ + # expose ports for controller/broker/server/admin/servicemanager EXPOSE 9000 8099 8098 8097 8096 9514 7098 diff --git a/pinot-servicemanager/build.gradle.kts b/pinot-servicemanager/build.gradle.kts index 238701f..26002e7 100644 --- a/pinot-servicemanager/build.gradle.kts +++ b/pinot-servicemanager/build.gradle.kts @@ -5,5 +5,18 @@ plugins { hypertraceDocker { defaultImage { imageName.set("pinot-servicemanager") + tasks.named(buildTaskName) { + dependsOn("copyPlugins") + } } } +val plugins by configurations.creating + +dependencies { + plugins(project(":pinot-udf")) +} + +tasks.register("copyPlugins") { + from(plugins) + into("${buildDir}/plugins") +} diff --git a/pinot-servicemanager/docker-bin/start-servicemanager b/pinot-servicemanager/docker-bin/start-servicemanager index d70b765..1fce97e 100755 --- a/pinot-servicemanager/docker-bin/start-servicemanager +++ b/pinot-servicemanager/docker-bin/start-servicemanager @@ -36,7 +36,7 @@ echo Starting Pinot Service Manager # TODO: link to pinot issue why we need internal access exec java \ --add-opens java.base/jdk.internal.ref=ALL-UNNAMED \ - -classpath 'classes:libs/*' \ + -classpath 'classes:plugins/*' \ $JAVA_OPTS \ -Dlog4j.configurationFile=etc/log4j2.properties \ -Dapp.name=pinot-admin \ diff --git a/pinot-udf/build.gradle.kts b/pinot-udf/build.gradle.kts new file mode 100644 index 0000000..9a3da7b --- /dev/null +++ b/pinot-udf/build.gradle.kts @@ -0,0 +1,8 @@ +plugins { + `java-library` +} + +dependencies { + implementation("org.hypertrace.core.attribute.service:attribute-projection-functions:0.3.4-SNAPSHOT") + compileOnly("org.apache.pinot:pinot-common:0.5.0") +} diff --git a/pinot-udf/src/main/java/org/apache/pinot/scalar/function/hypertrace/HypertraceScalarFunctions.java b/pinot-udf/src/main/java/org/apache/pinot/scalar/function/hypertrace/HypertraceScalarFunctions.java new file mode 100644 index 0000000..05498b6 --- /dev/null +++ b/pinot-udf/src/main/java/org/apache/pinot/scalar/function/hypertrace/HypertraceScalarFunctions.java @@ -0,0 +1,34 @@ +package org.apache.pinot.scalar.function.hypertrace; + +import static java.util.Objects.isNull; + +import org.apache.pinot.common.function.annotations.ScalarFunction; +import org.hypertrace.core.attribute.service.projection.functions.DefaultValue; +import org.hypertrace.core.attribute.service.projection.functions.Hash; + +public class HypertraceScalarFunctions { + + private static final String NULL_STRING = "null"; + public static final String HASH_FUNCTION_NAME = "hash"; + public static final String DEFAULT_STRING_FUNCTION_NAME = "defaultString"; + + @ScalarFunction(name = HASH_FUNCTION_NAME) + public static String hash(String value) { + return replaceNullWithNullString(Hash.hash(replaceNullStringWithNull(value))); + } + + @ScalarFunction(name = DEFAULT_STRING_FUNCTION_NAME) + public static String defaultString(String value, String defaultValue) { + return replaceNullWithNullString( + DefaultValue.defaultString( + replaceNullStringWithNull(value), replaceNullStringWithNull(defaultValue))); + } + + private static String replaceNullStringWithNull(String value) { + return NULL_STRING.equals(value) ? null : value; + } + + private static String replaceNullWithNullString(String value) { + return isNull(value) ? NULL_STRING : value; + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index cc4f819..289d20c 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -11,3 +11,4 @@ plugins { } include(":pinot-servicemanager") +include(":pinot-udf") From 291ec0721960fa2c88db05f44d718132c3483e0f Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Mon, 14 Sep 2020 12:16:41 +0800 Subject: [PATCH 2/4] Fixes broken project setup --- build.gradle.kts | 1 + pinot-servicemanager/Dockerfile | 5 +++-- pinot-servicemanager/docker-bin/start-servicemanager | 3 ++- pinot-udf/build.gradle.kts | 8 ++++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 6808f22..4b210e2 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,6 +1,7 @@ plugins { id("org.hypertrace.docker-plugin") version "0.7.1" id("org.hypertrace.docker-publish-plugin") version "0.7.1" + id("org.hypertrace.docker-java-application-plugin") version "0.7.1" apply false id("org.hypertrace.repository-plugin") version "0.2.3" } diff --git a/pinot-servicemanager/Dockerfile b/pinot-servicemanager/Dockerfile index 93f7b96..8309097 100644 --- a/pinot-servicemanager/Dockerfile +++ b/pinot-servicemanager/Dockerfile @@ -12,6 +12,9 @@ RUN mkdir libs && cp \ /opt/pinot/plugins/pinot-stream-ingestion/pinot-kafka-2.0/* \ libs; +# Add custom Hypertrace UDF plugin jars +COPY build/plugins libs/ + # Pinot starts faster when classes are extracted # TODO: When we switch to Maven for pinot, use maven-dependency-plugin:unpack RUN mkdir classes && cd classes && \ @@ -41,8 +44,6 @@ USER ${USER} # Copy binaries and config we installed earlier COPY --from=install --chown=${USER} /install . -COPY build/plugins plugins/ - # expose ports for controller/broker/server/admin/servicemanager EXPOSE 9000 8099 8098 8097 8096 9514 7098 diff --git a/pinot-servicemanager/docker-bin/start-servicemanager b/pinot-servicemanager/docker-bin/start-servicemanager index 1fce97e..bb83aff 100755 --- a/pinot-servicemanager/docker-bin/start-servicemanager +++ b/pinot-servicemanager/docker-bin/start-servicemanager @@ -34,9 +34,10 @@ test -d schemas && install-schema & echo Starting Pinot Service Manager # TODO: link to pinot issue why we need internal access +# NOTE: '-classpath classes' because it is faster to start up with a flat classpath exec java \ --add-opens java.base/jdk.internal.ref=ALL-UNNAMED \ - -classpath 'classes:plugins/*' \ + -classpath classes \ $JAVA_OPTS \ -Dlog4j.configurationFile=etc/log4j2.properties \ -Dapp.name=pinot-admin \ diff --git a/pinot-udf/build.gradle.kts b/pinot-udf/build.gradle.kts index 9a3da7b..ab2ee2b 100644 --- a/pinot-udf/build.gradle.kts +++ b/pinot-udf/build.gradle.kts @@ -2,6 +2,14 @@ plugins { `java-library` } +// Pinot images run with Java 11, not JDK 14 +// +// If any other dependencies are added, make sure they also are limited accordingly +configure { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} + dependencies { implementation("org.hypertrace.core.attribute.service:attribute-projection-functions:0.3.4-SNAPSHOT") compileOnly("org.apache.pinot:pinot-common:0.5.0") From 569b3eeec6ac6bc9c2571f1020f7c294e56be1da Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Mon, 21 Sep 2020 10:12:05 +0800 Subject: [PATCH 3/4] remove copy/pasta --- build.gradle.kts | 1 - 1 file changed, 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 4b210e2..6808f22 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,7 +1,6 @@ plugins { id("org.hypertrace.docker-plugin") version "0.7.1" id("org.hypertrace.docker-publish-plugin") version "0.7.1" - id("org.hypertrace.docker-java-application-plugin") version "0.7.1" apply false id("org.hypertrace.repository-plugin") version "0.2.3" } From 808ed96af5c801e260b4601937482b94b1d85aaf Mon Sep 17 00:00:00 2001 From: Aaron Steinfeld Date: Fri, 25 Sep 2020 18:52:18 -0400 Subject: [PATCH 4/4] feat: add udfs and upgrade to java 11 non-bundled image --- Dockerfile | 5 +++-- build.gradle.kts | 14 +++++++++++++- pinot-servicemanager/build.gradle.kts | 1 - pinot-udf/build.gradle.kts | 2 +- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 08a1c13..d71f6da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,10 @@ FROM alpine:latest AS builder ENV PINOT_VERSION=0.5.0 RUN wget -qO- https://downloads.apache.org/incubator/pinot/apache-pinot-incubating-$PINOT_VERSION/apache-pinot-incubating-$PINOT_VERSION-bin.tar.gz | tar -xzf- && \ - mv apache-pinot-incubating-0.5.0-bin /pinot && \ + mv apache-pinot-incubating-$PINOT_VERSION-bin /pinot && \ rm -rf /pinot/examples -FROM openjdk:8-jdk-slim +FROM openjdk:11-jdk-slim LABEL maintainer="Hypertrace https://www.hypertrace.org/" ENV PINOT_HOME=/opt/pinot @@ -14,6 +14,7 @@ ENV PINOT_HOME=/opt/pinot VOLUME ["${PINOT_HOME}/configs", "${PINOT_HOME}/data"] COPY --from=builder /pinot ${PINOT_HOME} +COPY build/plugins "${PINOT_HOME}/plugins" # expose ports for controller/broker/server/admin EXPOSE 9000 8099 8098 8097 8096 9514 diff --git a/build.gradle.kts b/build.gradle.kts index 6808f22..f7e3189 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -6,6 +6,18 @@ plugins { hypertraceDocker { defaultImage { - imageName.set("pinot") + tasks.named(buildTaskName) { + dependsOn("copyPlugins") + } } } +val plugins by configurations.creating + +dependencies { + plugins(project(":pinot-udf")) +} + +tasks.register("copyPlugins") { + from(plugins) + into("${buildDir}/plugins") +} diff --git a/pinot-servicemanager/build.gradle.kts b/pinot-servicemanager/build.gradle.kts index 26002e7..a78231f 100644 --- a/pinot-servicemanager/build.gradle.kts +++ b/pinot-servicemanager/build.gradle.kts @@ -4,7 +4,6 @@ plugins { hypertraceDocker { defaultImage { - imageName.set("pinot-servicemanager") tasks.named(buildTaskName) { dependsOn("copyPlugins") } diff --git a/pinot-udf/build.gradle.kts b/pinot-udf/build.gradle.kts index ab2ee2b..f416b17 100644 --- a/pinot-udf/build.gradle.kts +++ b/pinot-udf/build.gradle.kts @@ -11,6 +11,6 @@ configure { } dependencies { - implementation("org.hypertrace.core.attribute.service:attribute-projection-functions:0.3.4-SNAPSHOT") + implementation("org.hypertrace.core.attribute.service:attribute-projection-functions:0.4.1") compileOnly("org.apache.pinot:pinot-common:0.5.0") }