[Backport 27.0.0] Supervisor API documentation refactor

.github/scripts/setup_test_profiling_env.sh

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+set -e
+
+JAR_INPUT_FILE="jfr-profiler-1.0.0.jar"
+JAR_OUTPUT_FILE="jfr-profiler.jar"
+ENV_VAR="JFR_PROFILER_ARG_LINE"
+
+if [ "$#" -ne 5 ]; then
+    echo "usage: $0 <jdk_version> <run_id> <run_number> <run_attempt> <module>"
+fi
+
+if [[ "$1" == "17" ]];
+then
+  curl https://static.imply.io/cp/$JAR_INPUT_FILE -s -o $JAR_OUTPUT_FILE
+
+  echo $ENV_VAR=-javaagent:"$PWD"/$JAR_OUTPUT_FILE \
+  -Djfr.profiler.http.username=druid-ci \
+  -Djfr.profiler.http.password=w3Fb6PW8LIo849mViEkbgA== \
+  -Djfr.profiler.tags.project=druid \
+  -Djfr.profiler.tags.run_id=$2 \
+  -Djfr.profiler.tags.run_number=$3 \
+  -Djfr.profiler.tags.run_attempt=$4 \
+  -Djfr.profiler.tags.module=$5
+else
+  echo $ENV_VAR=\"\"
+fi
+
+

.github/scripts/unit_tests_script.sh

@@ -21,7 +21,8 @@ unset _JAVA_OPTIONS
 
 # Set MAVEN_OPTS for Surefire launcher.
 MAVEN_OPTS='-Xmx2500m' ${MVN} test -pl ${MAVEN_PROJECTS} \
-${MAVEN_SKIP} -Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL}
+${MAVEN_SKIP} -Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL} \
+-DjfrProfilerArgLine="${JFR_PROFILER_ARG_LINE}"
 sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 || exit 0"
 free -m
 ${MVN} -pl ${MAVEN_PROJECTS} jacoco:report || { echo "coverage_failure=false" >> "$GITHUB_ENV" && false; }

.github/workflows/cron-job-its.yml

@@ -27,7 +27,7 @@ on:
 
 jobs:
   build:
-    if: github.event_name == 'schedule'
+    if: (github.event_name == 'schedule' && github.repository == 'apache/druid')
     name: build (jdk8)
     runs-on: ubuntu-latest
     steps:
@@ -107,10 +107,12 @@ jobs:
       group: other
 
   security_vulnerabilities:
+    if: github.repository == 'apache/druid'
     name: security vulnerabilities
     strategy:
+      fail-fast: false
       matrix:
-        HADOOP_PROFILE: [ '', '-Phadoop3' ]
+        HADOOP_PROFILE: [ '', '-Phadoop2' ]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout branch
@@ -123,9 +125,11 @@ jobs:
           distribution: 'zulu'
           cache: maven
 
+      - name: maven build # needed to rebuild incase of maven snapshot resolution fails
+        run: mvn clean install dependency:go-offline -P dist -P skip-static-checks,skip-tests -Dmaven.javadoc.skip=true -Dcyclonedx.skip=true -Dweb.console.skip=true
+
       - name: security vulnerabilities check
         env:
-          MVN: mvn --no-snapshot-updates
           HADOOP_PROFILE: ${{ matrix.HADOOP_PROFILE }}
         run: |
           mvn dependency-check:purge dependency-check:check ${HADOOP_PROFILE} || { echo "

.github/workflows/reusable-unit-tests.yml

@@ -86,6 +86,11 @@ jobs:
             echo "DRUID_USE_DEFAULT_VALUE_FOR_NULL=true" >> $GITHUB_ENV
           fi
 
+      - name: test profiling
+        run: | 
+          ./.github/scripts/setup_test_profiling_env.sh ${{ inputs.jdk }} ${{ github.run_id }} \
+          ${{ github.run_number }} ${{ github.run_attempt }} ${{ inputs.module }} >> $GITHUB_ENV
+
       - name: fetch base branch for test coverage
         if: ${{ github.base_ref != '' }}
         run: |

README.md

@@ -17,8 +17,8 @@
   ~ under the License.
   -->
 
-[![Coverage Status](https://img.shields.io/codecov/c/gh/apache/druid)](https://codecov.io/gh/apache/druid)
-[![Docker](https://img.shields.io/badge/container-docker-blue.svg)](https://hub.docker.com/r/apache/druid)
+[![Coverage Status](https://img.shields.io/codecov/c/gh/apache/druid?logo=codecov)](https://codecov.io/gh/apache/druid)
+[![Docker](https://img.shields.io/badge/container-docker-blue.svg?logo=docker)](https://hub.docker.com/r/apache/druid)
 [![Helm](https://img.shields.io/badge/helm-druid-5F90AB?logo=helm)](https://github.com/apache/druid/blob/master/helm/druid/README.md)
 <!--- Following badges are disabled until they can be fixed: -->
 <!--- [![Inspections Status](https://img.shields.io/teamcity/http/teamcity.jetbrains.com/s/OpenSourceProjects_Druid_Inspections.svg?label=TeamCity%20inspections)](https://teamcity.jetbrains.com/viewType.html?buildTypeId=OpenSourceProjects_Druid_Inspections) -->

benchmarks/pom.xml

@@ -27,7 +27,7 @@
   <parent>
     <groupId>org.apache.druid</groupId>
     <artifactId>druid</artifactId>
-    <version>27.0.0-SNAPSHOT</version>
+    <version>28.0.0-SNAPSHOT</version>
   </parent>
 
   <dependencies>

benchmarks/src/test/java/org/apache/druid/benchmark/DictionaryEncodedStringIndexSupplierBenchmark.java

@@ -26,12 +26,12 @@
 import org.apache.druid.collections.bitmap.RoaringBitmapFactory;
 import org.apache.druid.common.config.NullHandling;
 import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.segment.column.BitmapColumnIndex;
-import org.apache.druid.segment.column.IndexedUtf8ValueSetIndex;
-import org.apache.druid.segment.column.StringValueSetIndex;
 import org.apache.druid.segment.data.BitmapSerdeFactory;
 import org.apache.druid.segment.data.GenericIndexed;
 import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
+import org.apache.druid.segment.index.BitmapColumnIndex;
+import org.apache.druid.segment.index.IndexedUtf8ValueIndexes;
+import org.apache.druid.segment.index.semantic.StringValueSetIndexes;
 import org.apache.druid.segment.serde.StringUtf8ColumnIndexSupplier;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -72,7 +72,7 @@ public class DictionaryEncodedStringIndexSupplierBenchmark
   public static class BenchmarkState
   {
     @Nullable
-    private IndexedUtf8ValueSetIndex<?> stringValueSetIndex;
+    private IndexedUtf8ValueIndexes<?> stringValueSetIndex;
     private final TreeSet<ByteBuffer> values = new TreeSet<>();
     private static final int START_INT = 10_000_000;
 
@@ -112,7 +112,7 @@ public void setup()
       );
       StringUtf8ColumnIndexSupplier<?> indexSupplier =
           new StringUtf8ColumnIndexSupplier<>(bitmapFactory, dictionaryUtf8::singleThreaded, bitmaps, null);
-      stringValueSetIndex = (IndexedUtf8ValueSetIndex<?>) indexSupplier.as(StringValueSetIndex.class);
+      stringValueSetIndex = (IndexedUtf8ValueIndexes<?>) indexSupplier.as(StringValueSetIndexes.class);
       List<Integer> filterValues = new ArrayList<>();
       List<Integer> nonFilterValues = new ArrayList<>();
       for (int i = 0; i < dictionarySize; i++) {

cloud/aws-common/pom.xml

@@ -28,7 +28,7 @@
     <parent>
         <groupId>org.apache.druid</groupId>
         <artifactId>druid</artifactId>
-        <version>27.0.0-SNAPSHOT</version>
+        <version>28.0.0-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

cloud/gcp-common/pom.xml

@@ -28,7 +28,7 @@
     <parent>
         <groupId>org.apache.druid</groupId>
         <artifactId>druid</artifactId>
-        <version>27.0.0-SNAPSHOT</version>
+        <version>28.0.0-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 

distribution/docker/docker-compose.yml

@@ -51,7 +51,7 @@ services:
       - ZOO_MY_ID=1
 
   coordinator:
-    image: apache/druid:27.0.0
+    image: apache/druid:28.0.0
     container_name: coordinator
     volumes:
       - druid_shared:/opt/shared
@@ -67,7 +67,7 @@ services:
       - environment
 
   broker:
-    image: apache/druid:27.0.0
+    image: apache/druid:28.0.0
     container_name: broker
     volumes:
       - broker_var:/opt/druid/var
@@ -83,7 +83,7 @@ services:
       - environment
 
   historical:
-    image: apache/druid:27.0.0
+    image: apache/druid:28.0.0
     container_name: historical
     volumes:
       - druid_shared:/opt/shared
@@ -100,7 +100,7 @@ services:
       - environment
 
   middlemanager:
-    image: apache/druid:27.0.0
+    image: apache/druid:28.0.0
     container_name: middlemanager
     volumes:
       - druid_shared:/opt/shared
@@ -118,7 +118,7 @@ services:
       - environment
 
   router:
-    image: apache/druid:27.0.0
+    image: apache/druid:28.0.0
     container_name: router
     volumes:
       - router_var:/opt/druid/var

distribution/pom.xml

@@ -30,7 +30,7 @@
     <parent>
         <artifactId>druid</artifactId>
         <groupId>org.apache.druid</groupId>
-        <version>27.0.0-SNAPSHOT</version>
+        <version>28.0.0-SNAPSHOT</version>
     </parent>
 
     <dependencies>
@@ -633,6 +633,8 @@
                                         <argument>org.apache.druid.extensions.contrib:aliyun-oss-extensions</argument>
                                         <argument>-c</argument>
                                         <argument>org.apache.druid.extensions.contrib:opentelemetry-emitter</argument>
+                                        <argument>-c</argument>
+                                        <argument>org.apache.druid.extensions:druid-iceberg-extensions</argument>
                                     </arguments>
                                 </configuration>
                             </execution>