diff --git a/Dockerfile b/Dockerfile index 08a1c13..d71f6da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,10 @@ FROM alpine:latest AS builder ENV PINOT_VERSION=0.5.0 RUN wget -qO- https://downloads.apache.org/incubator/pinot/apache-pinot-incubating-$PINOT_VERSION/apache-pinot-incubating-$PINOT_VERSION-bin.tar.gz | tar -xzf- && \ - mv apache-pinot-incubating-0.5.0-bin /pinot && \ + mv apache-pinot-incubating-$PINOT_VERSION-bin /pinot && \ rm -rf /pinot/examples -FROM openjdk:8-jdk-slim +FROM openjdk:11-jdk-slim LABEL maintainer="Hypertrace https://www.hypertrace.org/" ENV PINOT_HOME=/opt/pinot @@ -14,6 +14,7 @@ ENV PINOT_HOME=/opt/pinot VOLUME ["${PINOT_HOME}/configs", "${PINOT_HOME}/data"] COPY --from=builder /pinot ${PINOT_HOME} +COPY build/plugins "${PINOT_HOME}/plugins" # expose ports for controller/broker/server/admin EXPOSE 9000 8099 8098 8097 8096 9514 diff --git a/build.gradle.kts b/build.gradle.kts index e69af34..fbd50ae 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -6,6 +6,28 @@ plugins { hypertraceDocker { defaultImage { - imageName.set("pinot") + tasks.named(buildTaskName) { + dependsOn("copyPlugins") + } } -} \ No newline at end of file +} + +val plugins by configurations.creating + +dependencies { + plugins(project(":pinot-udf")) +} + +tasks.register("copyPlugins") { + from(plugins) + into("${buildDir}/plugins") +} + +subprojects { + pluginManager.withPlugin("java") { + configure { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 + } + } +} diff --git a/pinot-avro-serde/build.gradle.kts b/pinot-avro-serde/build.gradle.kts new file mode 100644 index 0000000..1587707 --- /dev/null +++ b/pinot-avro-serde/build.gradle.kts @@ -0,0 +1,14 @@ +plugins { + `java-library` +} + +dependencies { + compileOnly("org.apache.pinot:pinot-spi:0.5.0") + compileOnly("org.apache.pinot:pinot-avro-base:0.5.0") + compileOnly("org.apache.kafka:kafka-streams:5.5.1-ccs") + compileOnly("org.apache.kafka:kafka-clients:5.5.1-ccs") + implementation("org.hypertrace.core.kafkastreams.framework:kafka-streams-serdes:0.1.11") { + // disable the transitive dependencies and use them from pinot itself. + isTransitive = false + } +} diff --git a/pinot-servicemanager/src/main/java/org/hypertrace/pinot/plugins/GenericAvroMessageDecoder.java b/pinot-avro-serde/src/main/java/org/hypertrace/pinot/plugins/GenericAvroMessageDecoder.java similarity index 100% rename from pinot-servicemanager/src/main/java/org/hypertrace/pinot/plugins/GenericAvroMessageDecoder.java rename to pinot-avro-serde/src/main/java/org/hypertrace/pinot/plugins/GenericAvroMessageDecoder.java diff --git a/pinot-servicemanager/Dockerfile b/pinot-servicemanager/Dockerfile index d602ba5..d38428c 100644 --- a/pinot-servicemanager/Dockerfile +++ b/pinot-servicemanager/Dockerfile @@ -7,7 +7,8 @@ FROM cimg/openjdk:14.0.2 AS install ARG JITPACK_USER=apache # Override to use a different expression, such as 'master-SNAPSHOT' # Note: 0.5.0 does not have parallel start (PR 5917): 21a372b2f58f9c6e27fe9710d6952ed519342061 -ARG JITPACK_TAG=23797914c6fea24e34f3ebfca9a73e0fff72c2b7 +# Note: See "Warning on versions" in README before upgrading version +ARG JITPACK_TAG=1c4fc13169ff3ed8cab3a2ddcdd15270e07d74fc USER root WORKDIR /install diff --git a/pinot-servicemanager/build.gradle.kts b/pinot-servicemanager/build.gradle.kts index db2d70d..943b633 100644 --- a/pinot-servicemanager/build.gradle.kts +++ b/pinot-servicemanager/build.gradle.kts @@ -1,41 +1,23 @@ plugins { - `java-library` id("org.hypertrace.docker-publish-plugin") } hypertraceDocker { defaultImage { - imageName.set("pinot-servicemanager") tasks.named(buildTaskName) { dependsOn("copyPlugins") } } } -configure { - sourceCompatibility = JavaVersion.VERSION_11 - targetCompatibility = JavaVersion.VERSION_11 -} - val plugins by configurations.creating dependencies { - compileOnly("org.apache.pinot:pinot-spi:0.5.0") - compileOnly("org.apache.pinot:pinot-avro-base:0.5.0") - compileOnly("org.apache.kafka:kafka-streams:5.5.1-ccs") - compileOnly("org.apache.kafka:kafka-clients:5.5.1-ccs") - implementation("org.hypertrace.core.kafkastreams.framework:kafka-streams-serdes:0.1.11") { - // disable the transitive dependencies and use them from pinot itself. - isTransitive = false - } -} - -dependencies { - plugins(project(":${project.name}")) + plugins(project(":pinot-udf")) + plugins(project(":pinot-avro-serde")) } tasks.register("copyPlugins") { from(plugins) into("${buildDir}/plugins") } - diff --git a/pinot-servicemanager/install.sh b/pinot-servicemanager/install.sh index 7d45af0..5ad17e7 100755 --- a/pinot-servicemanager/install.sh +++ b/pinot-servicemanager/install.sh @@ -15,6 +15,10 @@ do rm -rf temp.zip classes/META-INF/license done +# TODO: try maven-dependency-plugin:unpack instead of wget +# https://maven.apache.org/plugins/maven-dependency-plugin/examples/unpacking-artifacts.html +# https://github.com/hypertrace/pinot/issues/16 + # copy hypertrace plugins for JAR in plugins/* do @@ -22,9 +26,6 @@ do done rm -rf plugins -# TODO: try maven-dependency-plugin:unpack instead of wget -# https://maven.apache.org/plugins/maven-dependency-plugin/examples/unpacking-artifacts.html -# https://github.com/hypertrace/pinot/issues/16 mkdir etc diff --git a/pinot-udf/README.md b/pinot-udf/README.md new file mode 100644 index 0000000..05279fa --- /dev/null +++ b/pinot-udf/README.md @@ -0,0 +1,20 @@ +# Pinot UDFs + +## Requirements +These UDFs should have a minimal dependency tree and be compiled to support a java target no +higher than our pinot images use as a runtime. Currently, that means java 11. Any common dependencies +in the dependency tree (which can be seen via `gradlew dependencies --configuration runtimeClasspath`) +must be shaded, so should be avoided. + +## Warning on versions +As of [2379791](https://github.com/apache/incubator-pinot/commit/23797914c6fea24e34f3ebfca9a73e0fff72c2b7) +(between 0.5.0 and 0.6.0) pinot introduces a breaking change moving the ScalarFunction annotation to +a new packacge. Because these UDFs are used by both the main pinot image and the +pinot-servicemanager image, both images must use either an earlier version or a later version +(although they should be in sync regardless), and the compile dependency on pinot in this module +should reflect that same version. + +## Rationale +This UDF jar is added as a plugin to the pinot images built in this repository. The UDFs are used to +support projected attributes, where we support creating new attributes as read time-only projections, +using a combination of custom UDFs and those built in to pinot. \ No newline at end of file diff --git a/pinot-udf/build.gradle.kts b/pinot-udf/build.gradle.kts new file mode 100644 index 0000000..a5e9736 --- /dev/null +++ b/pinot-udf/build.gradle.kts @@ -0,0 +1,8 @@ +plugins { + `java-library` +} + +dependencies { + implementation("org.hypertrace.core.attribute.service:attribute-projection-functions:0.4.1") + compileOnly("org.apache.pinot:pinot-common:0.5.0") +} diff --git a/pinot-udf/src/main/java/org/apache/pinot/scalar/function/hypertrace/HypertraceScalarFunctions.java b/pinot-udf/src/main/java/org/apache/pinot/scalar/function/hypertrace/HypertraceScalarFunctions.java new file mode 100644 index 0000000..05498b6 --- /dev/null +++ b/pinot-udf/src/main/java/org/apache/pinot/scalar/function/hypertrace/HypertraceScalarFunctions.java @@ -0,0 +1,34 @@ +package org.apache.pinot.scalar.function.hypertrace; + +import static java.util.Objects.isNull; + +import org.apache.pinot.common.function.annotations.ScalarFunction; +import org.hypertrace.core.attribute.service.projection.functions.DefaultValue; +import org.hypertrace.core.attribute.service.projection.functions.Hash; + +public class HypertraceScalarFunctions { + + private static final String NULL_STRING = "null"; + public static final String HASH_FUNCTION_NAME = "hash"; + public static final String DEFAULT_STRING_FUNCTION_NAME = "defaultString"; + + @ScalarFunction(name = HASH_FUNCTION_NAME) + public static String hash(String value) { + return replaceNullWithNullString(Hash.hash(replaceNullStringWithNull(value))); + } + + @ScalarFunction(name = DEFAULT_STRING_FUNCTION_NAME) + public static String defaultString(String value, String defaultValue) { + return replaceNullWithNullString( + DefaultValue.defaultString( + replaceNullStringWithNull(value), replaceNullStringWithNull(defaultValue))); + } + + private static String replaceNullStringWithNull(String value) { + return NULL_STRING.equals(value) ? null : value; + } + + private static String replaceNullWithNullString(String value) { + return isNull(value) ? NULL_STRING : value; + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index cc4f819..4ba0578 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -11,3 +11,5 @@ plugins { } include(":pinot-servicemanager") +include(":pinot-udf") +include(":pinot-avro-serde")