[Backport] Druid automated quickstart

.github/scripts/display_dependency_analysis_error_message.sh → .github/scripts/analyze_dependencies_script.sh

@@ -13,6 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+#!bin/bash
+
+${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true ${HADOOP_PROFILE} ||
 { echo "
     The dependency analysis has found a dependency that is either:
 

.github/scripts/license_checks_script.sh

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+
+set -e
+
+./.github/scripts/setup_generate_license.sh
+${MVN} apache-rat:check -Prat --fail-at-end \
+-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
+-Drat.consoleOutput=true ${HADOOP_PROFILE}
+# Generate dependency reports and checks they are valid.
+mkdir -p target
+distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2
+distribution/bin/check-licenses.py licenses.yaml target/license-reports

.github/scripts/setup_generate_license.sh

@@ -13,6 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+#!/bin/bash
+
+set -e
+
 sudo apt-get update && sudo apt-get install python3 -y
 curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3
 pip3 install wheel  # install wheel first explicitly

.github/workflows/static-checks.yml

@@ -35,160 +35,139 @@ env:
   MAVEN_OPTS: -Xmx3000m
 
 jobs:
-  build:
+  static-checks:
+    strategy:
+      matrix:
+        java: [ 'jdk8', 'jdk11', 'jdk17' ]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - name: setup java 8
+      - name: checkout branch
+        uses: actions/checkout@v3
+
+      - name: set java version
+        run: |
+          export jdk=${{ matrix.java }}
+          echo "java_version=${jdk:3}" >> $GITHUB_ENV
+
+      - name: setup ${{ matrix.java }}
         uses: actions/setup-java@v3
         with:
           distribution: 'zulu'
-          java-version: '8'
+          java-version: ${{ env.java_version }}
           cache: 'maven'
-      - run: |
+
+      - name: packaging check
+        run: |
+          ./.github/scripts/setup_generate_license.sh
+          ${MVN} clean install -Prat -Pdist -Pbundle-contrib-exts --fail-at-end \
+          -pl '!benchmarks' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -Dweb.console.skip=false -T1C
+
+      - name: script checks
+        # who watches the watchers?
+        if: ${{ matrix.java == 'jdk8' }}
+        run: ./check_test_suite_test.py
+
+      - name: (openjdk11) strict compilation
+        if: ${{ matrix.java == 'jdk11' }}
+        # errorprone requires JDK 11
+        # Strict compilation requires more than 2 GB
+        run: ${MVN} clean -DstrictCompile compile test-compile --fail-at-end ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
+
+      - name: maven install
+        if: ${{ matrix.java == 'jdk8' }}
+        run: |
           echo 'Running Maven install...' &&
           ${MVN} clean install -q -ff -pl '!distribution,!:druid-it-image,!:druid-it-cases' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C &&
           ${MVN} install -q -ff -pl 'distribution' ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
 
-  animal_sniffer_checks:
-    runs-on: ubuntu-latest
-    needs: [build]
-    steps:
-      - uses: actions/checkout@v3
-      - name: setup java 8
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'zulu'
-          java-version: '8'
-          cache: 'maven'
-      - run: ${MVN} animal-sniffer:check --fail-at-end
+      - name: license checks
+        if: ${{ matrix.java == 'jdk8' }}
+        run: ./.github/scripts/license_checks_script.sh
 
-  checkstyle:
-    runs-on: ubuntu-latest
-    needs: [build]
-    steps:
-      - uses: actions/checkout@v3
-      - name: setup java 8
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'zulu'
-          java-version: '8'
-          cache: 'maven'
-      - run: ${MVN} checkstyle:checkstyle --fail-at-end
+      - name: license checks for hadoop3
+        if: ${{ matrix.java == 'jdk8' }}
+        env:
+          HADOOP_PROFILE: -Phadoop3
+        run: ./.github/scripts/license_checks_script.sh
 
-  enforcer_checks:
-    runs-on: ubuntu-latest
-    needs: [build]
-    steps:
-      - uses: actions/checkout@v3
-      - name: setup java 8
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'zulu'
-          java-version: '8'
-          cache: 'maven'
-      - run: ${MVN} enforcer:enforce --fail-at-end
+      - name: analyze dependencies
+        if: ${{ matrix.java == 'jdk8' }}
+        run: |
+          ./.github/scripts/analyze_dependencies_script.sh
 
-  forbidden_api_checks:
-    runs-on: ubuntu-latest
-    needs: [build]
-    steps:
-      - uses: actions/checkout@v3
-      - name: setup java 8
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'zulu'
-          java-version: '8'
-          cache: 'maven'
-      - run: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end
+      - name: analyze dependencies for hadoop3
+        if: ${{ matrix.java == 'jdk8' }}
+        env:
+          HADOOP_PROFILE: -Phadoop3
+        run: |
+          ./.github/scripts/analyze_dependencies_script.sh
 
-  pmd_checks:
-    runs-on: ubuntu-latest
-    needs: [build]
-    steps:
-      - uses: actions/checkout@v3
-      - name: setup java 8
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'zulu'
-          java-version: '8'
-          cache: 'maven'
-      - run: ${MVN} pmd:check --fail-at-end  # TODO: consider adding pmd:cpd-check
+      - name: animal sniffer checks
+        if: ${{ matrix.java == 'jdk8' }}
+        run: ${MVN} animal-sniffer:check --fail-at-end
 
-  spotbugs_checks:
-    runs-on: ubuntu-latest
-    needs: [build]
-    steps:
-      - uses: actions/checkout@v3
-      - name: setup java 8
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'zulu'
-          java-version: '8'
-          cache: 'maven'
-      - run: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks'
+      - name: checkstyle
+        if: ${{ matrix.java == 'jdk8' }}
+        run: ${MVN} checkstyle:checkstyle --fail-at-end
 
-  license_checks:
-    runs-on: ubuntu-latest
-    needs: [build]
-    strategy:
-      matrix:
-        HADOOP_PROFILE: ['', '-Phadoop3']
-    steps:
-      - uses: actions/checkout@v3
-      - name: setup java 8
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'zulu'
-          java-version: '8'
-          cache: 'maven'
-      - run: |
-          ./.github/scripts/setup_generate_license.sh
-          ${MVN} apache-rat:check -Prat --fail-at-end \
-          -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
-          -Drat.consoleOutput=true ${{ matrix.HADOOP_PROFILE }}
-          # Generate dependency reports and checks they are valid.
-          mkdir -p target
-          distribution/bin/generate-license-dependency-reports.py . target --clean-maven-artifact-transfer --parallel 2
-          distribution/bin/check-licenses.py licenses.yaml target/license-reports
-
-  script_checks:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        # who watches the watchers?
-      - run: ./check_test_suite_test.py
+      - name: enforcer checks
+        if: ${{ matrix.java == 'jdk8' }}
+        run: ${MVN} enforcer:enforce --fail-at-end
 
-  analyze_dependencies:
-    runs-on: ubuntu-latest
-    needs: [build]
-    strategy:
-      matrix:
-        HADOOP_PROFILE: [ '', '-Phadoop3' ]
-    steps:
-      - uses: actions/checkout@v3
-      - name: setup java 8
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'zulu'
-          java-version: '8'
-          cache: 'maven'
-      - run: |-
-          ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true ${{ matrix.HADOOP_PROFILE }} ||
-          ./.github/scripts/display_dependency_analysis_error_message.sh
+      - name: forbidden api checks
+        if: ${{ matrix.java == 'jdk8' }}
+        run: ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end
 
-  openjdk11_strict_compilation:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: setup java 11
-        uses: actions/setup-java@v3
+      - name: pmd checks
+        if: ${{ matrix.java == 'jdk8' }}
+        run: ${MVN} pmd:check --fail-at-end  # TODO: consider adding pmd:cpd-check
+
+      - name: spotbugs checks
+        if: ${{ matrix.java == 'jdk8' }}
+        run: ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks'
+
+      - name: intellij inspections
+        if: ${{ matrix.java == 'jdk8' }}
+        run: |
+          docker run --rm \
+          -v $(pwd):/project \
+          -v ~/.m2:/home/inspect/.m2 \
+          ccaominh/intellij-inspect:1.0.0 \
+          /project/pom.xml \
+          /project/.idea/inspectionProfiles/Druid.xml \
+          --levels ERROR \
+          --scope JavaInspectionsScope
+
+      - name: setup node
+        if: ${{ matrix.java == 'jdk8' }}
+        uses: actions/setup-node@v3
         with:
-          distribution: 'zulu'
-          java-version: '11'
-          cache: 'maven'
+          node-version: 16.17.0
 
-        # errorprone requires JDK 11
-        # Strict compilation requires more than 2 GB
-      - run: |
-          ${MVN} clean -DstrictCompile compile test-compile --fail-at-end ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS}
+      - name: docs
+        if: ${{ matrix.java == 'jdk8' }}
+        run: |
+          (cd website && npm install)
+          cd website
+          npm run link-lint
+          npm run spellcheck
+
+      - name: web console
+        if: ${{ matrix.java == 'jdk8' }}
+        run: |
+          ${MVN} test -pl 'web-console'
+          cd web-console
+          { for i in 1 2 3; do npm run codecov && break || sleep 15; done }
+
+      - name: web console end-to-end test
+        if: ${{ matrix.java == 'jdk8' }}
+        run: |
+          ./.github/scripts/setup_generate_license.sh
+          sudo apt-get update && sudo apt-get install python3 -y
+          curl https://bootstrap.pypa.io/pip/3.5/get-pip.py | sudo -H python3
+          pip3 install wheel  # install wheel first explicitly
+          pip3 install --upgrade pyyaml
+          web-console/script/druid build
+          web-console/script/druid start
+          (cd web-console && npm run test-e2e)
+          web-console/script/druid stop