From b0758ce87b5d1d80b74cc5ed02928002e26e55e7 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Thu, 11 Mar 2021 22:05:45 +0100 Subject: [PATCH 1/4] HDDS-4889. Add simple CI check for docs --- .github/workflows/post-commit.yml | 3 +- hadoop-ozone/dev-support/checks/_lib.sh | 28 +++++++++++++++++ hadoop-ozone/dev-support/checks/docs.sh | 40 +++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100755 hadoop-ozone/dev-support/checks/docs.sh diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml index f94f3a2fc9f7..e12c9c74a262 100644 --- a/.github/workflows/post-commit.yml +++ b/.github/workflows/post-commit.yml @@ -77,6 +77,7 @@ jobs: - author - bats - checkstyle + - docs - findbugs - rat - unit @@ -93,7 +94,7 @@ jobs: maven-repo-${{ hashFiles('**/pom.xml') }}-8 maven-repo-${{ hashFiles('**/pom.xml') }} maven-repo- - if: ${{ !contains('author,bats', matrix.check) }} + if: ${{ !contains('author,bats,docs', matrix.check) }} - name: Execute tests run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh - name: Summary of failures diff --git a/hadoop-ozone/dev-support/checks/_lib.sh b/hadoop-ozone/dev-support/checks/_lib.sh index ddc2866cd3fe..56b15da03088 100644 --- a/hadoop-ozone/dev-support/checks/_lib.sh +++ b/hadoop-ozone/dev-support/checks/_lib.sh @@ -87,6 +87,34 @@ _install_flekszible() { chmod +x bin/flekszible } +install_hugo() { + _install_tool hugo bin +} + +_install_hugo() { + : ${HUGO_VERSION:=0.81.0} + + local os=$(uname -s) + local arch=$(uname -m) + + mkdir bin + + case "${os}" in + Darwin) + os=macOS + ;; + esac + + case "${arch}" in + x86_64) + arch=64bit + ;; + esac + + curl -LSs "https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_${HUGO_VERSION}_${os}-${arch}.tar.gz" | tar -xz -f - -C bin hugo + chmod +x bin/hugo +} + install_virtualenv() { _install_tool virtualenv } diff --git a/hadoop-ozone/dev-support/checks/docs.sh b/hadoop-ozone/dev-support/checks/docs.sh new file mode 100755 index 000000000000..11cb21859a4e --- /dev/null +++ b/hadoop-ozone/dev-support/checks/docs.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -u -o pipefail + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "${DIR}/../../.." || exit 1 + +source "${DIR}/_lib.sh" +install_hugo + +REPORT_DIR=${OUTPUT_DIR:-"${DIR}/../../../target/docs"} +mkdir -p "${REPORT_DIR}" +REPORT_FILE="${REPORT_DIR}/summary.txt" + +hadoop-hdds/docs/dev-support/bin/generate-site.sh | tee "${REPORT_DIR}/output.log" +rc=$? + +grep -o 'ERROR.*' "${REPORT_DIR}/output.log" > "${REPORT_FILE}" + +wc -l "${REPORT_FILE}" | awk '{ print $1 }' > "${REPORT_DIR}/failures" + +if [[ -s "${REPORT_FILE}" ]]; then + exit 1 +fi + +exit ${rc} From 982541d5df723f4d6c5186e86acde04a02d321c5 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 14 Apr 2021 20:19:51 +0200 Subject: [PATCH 2/4] HDDS-5110. DEBUG --- .../scm/TestStorageContainerManagerHttpServer.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHttpServer.java index 20d5485d8aca..2b8b14d27b45 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHttpServer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHttpServer.java @@ -45,12 +45,17 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Test http server os SCM with various HTTP option. */ @RunWith(value = Parameterized.class) public class TestStorageContainerManagerHttpServer { + + private static final Logger LOG = + LoggerFactory.getLogger(TestStorageContainerManagerHttpServer.class); private static final String BASEDIR = GenericTestUtils .getTempPath(TestStorageContainerManagerHttpServer.class.getSimpleName()); private static String keystoresDir; @@ -130,13 +135,13 @@ private static boolean canAccess(String scheme, InetSocketAddress addr) { if (addr == null) { return false; } + String url = scheme + "://" + NetUtils.getHostPortString(addr) + "/jmx"; try { - URL url = - new URL(scheme + "://" + NetUtils.getHostPortString(addr) + "/jmx"); - URLConnection conn = connectionFactory.openConnection(url); + URLConnection conn = connectionFactory.openConnection(new URL(url)); conn.connect(); conn.getContent(); } catch (IOException e) { + LOG.info("Cannot access {}", url, e); return false; } return true; From 0e0283f3071e6601471ff2dac1375bf299026b85 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 19 Apr 2021 10:19:03 +0200 Subject: [PATCH 3/4] Revert "HDDS-5110. DEBUG" This reverts commit 982541d5df723f4d6c5186e86acde04a02d321c5. --- .../scm/TestStorageContainerManagerHttpServer.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHttpServer.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHttpServer.java index 079df784a5d7..e5727f825d00 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHttpServer.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHttpServer.java @@ -45,17 +45,12 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Test http server os SCM with various HTTP option. */ @RunWith(value = Parameterized.class) public class TestStorageContainerManagerHttpServer { - - private static final Logger LOG = - LoggerFactory.getLogger(TestStorageContainerManagerHttpServer.class); private static final String BASEDIR = GenericTestUtils .getTempPath(TestStorageContainerManagerHttpServer.class.getSimpleName()); private static String keystoresDir; @@ -137,13 +132,13 @@ private static boolean canAccess(String scheme, InetSocketAddress addr) { if (addr == null) { return false; } - String url = scheme + "://" + NetUtils.getHostPortString(addr) + "/jmx"; try { - URLConnection conn = connectionFactory.openConnection(new URL(url)); + URL url = + new URL(scheme + "://" + NetUtils.getHostPortString(addr) + "/jmx"); + URLConnection conn = connectionFactory.openConnection(url); conn.connect(); conn.getContent(); } catch (IOException e) { - LOG.info("Cannot access {}", url, e); return false; } return true; From 9e6312c30c1b31c64988f6e2450acb26ae9f80e7 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 19 Apr 2021 10:48:34 +0200 Subject: [PATCH 4/4] trigger new CI check