diff --git a/.gitignore b/.gitignore index 794447c..93c6522 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,11 @@ *.swp +# build directories +build*/* +cmake-build-debug/* +dependencies/* + # Thirdparty dirs third-party/* /gcc-debug/ @@ -25,6 +30,7 @@ src/hbase/utils/version.h # Tests *-test +src/test/target/* # Executables load-client diff --git a/bin/hbase-native-client-personality.sh b/bin/hbase-native-client-personality.sh new file mode 100755 index 0000000..5324703 --- /dev/null +++ b/bin/hbase-native-client-personality.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# You'll need a local installation of +# [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.1.0/#yetus-precommit) +# to use this personality. +# +# Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact and +# build from it, or use the convenience binaries provided on that download page. +# +# To run against, e.g. HBASE-15074 you'd then do +# ```bash +# test-patch --personality=bin/hbase-native-client-personality.sh HBASE-15074 +# ``` +# +# If you want to skip the ~1 hour it'll take to do all the hadoop API checks, use +# ```bash +# test-patch --plugins=all,-hadoopcheck --personality=dev-support/hbase-personality.sh HBASE-15074 +# ```` +# +# pass the `--sentinel` flag if you want to allow test-patch to destructively alter local working +# directory / branch in order to have things match what the issue patch requests. + +# Plugins enabled. +personality_plugins "compile,unit,cmake,asflicense,github,jira,ctest,htmlout" + +if ! declare -f "yetus_info" >/dev/null; then + function yetus_info + { + echo "[$(date) INFO]: $*" 1>&2 + } +fi + +# work around yetus overwriting JAVA_HOME from our docker image +function docker_do_env_adds +{ + declare k + for k in "${DOCKER_EXTRAENVS[@]}"; do + if [[ "JAVA_HOME" == "${k}" ]]; then + if [ -n "${JAVA_HOME}" ]; then + DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}") + fi + else + DOCKER_EXTRAARGS+=("--env=${k}=${!k}") + fi + done + DOCKER_EXTRAARGS+=("-h=securecluster") +} + +## @description Globals specific to this personality +## @audience private +## @stability evolving +function personality_globals +{ + # shellcheck disable=SC2034 + SUDO_USER=root + # shellcheck disable=SC2034 + BUILD_NATIVE=true + # shellcheck disable=SC2034 + BUILDTOOL=cmake + # shellcheck disable=SC2034 + # Passed to cmake command using a custom personality. + CMAKE_ARGS="-DDOWNLOAD_DEPENDENCIES=ON" + # shellcheck disable=SC2034 + # Expected by Yetus for compiling non-jvm projects. + JVM_REQUIRED=false + #shellcheck disable=SC2034 + PROJECT_NAME=hbase-native-client + #shellcheck disable=SC2034 + PATCH_BRANCH_DEFAULT=master + #shellcheck disable=SC2034 + JIRA_ISSUE_RE='^HBASE-[0-9]+$' + #shellcheck disable=SC2034 + GITHUB_REPO="apache/hbase-native-client" + # Yetus 0.7.0 enforces limits. Default proclimit is 1000. + # Up it. See HBASE-19902 for how we arrived at this number. + #shellcheck disable=SC2034 + PROCLIMIT=10000 + # Override if you want to bump up the memlimit for docker. + # shellcheck disable=SC2034 + DOCKERMEMLIMIT=4g +} + +## @description Queue up modules for this personality +## @audience private +## @stability evolving +## @param repostatus +## @param testtype +function personality_modules +{ + local repostatus=$1 + local testtype=$2 + local args + yetus_info "Personality: ${repostatus} ${testtype}" + clear_personality_queue + if [[ "${testtype}" =~ CMakeLists.txt ]]; then + args=${CMAKE_ARGS} + yetus_debug "Appending CMake args ${args}" + fi + personality_enqueue_module . ${args} +} diff --git a/bin/jenkins/Jenkinsfile_GitHub b/bin/jenkins/Jenkinsfile_GitHub new file mode 100644 index 0000000..60c4cb0 --- /dev/null +++ b/bin/jenkins/Jenkinsfile_GitHub @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pipeline { + + agent { + label 'Hadoop' + } + + options { + // N.B. this is per-branch, which means per PR + disableConcurrentBuilds() + buildDiscarder(logRotator(numToKeepStr: '15')) + timeout (time: 7, unit: 'HOURS') + timestamps() + skipDefaultCheckout() + } + + environment { + SRC_REL = 'src' + PATCH_REL = 'output' + YETUS_REL = 'yetus' + // Branch or tag name. Yetus release tags are 'rel/X.Y.Z' + YETUS_VERSION = 'rel/0.12.0' + DOCKERFILE_REL = "${SRC_REL}/docker-files/Dockerfile" + YETUS_DRIVER_REL = "${SRC_REL}/bin/run_tests_with_yetus.sh" + ARCHIVE_PATTERN_LIST = 'org.apache.h*.txt,*.dumpstream,*.dump,*.jstack,*.pstack,*.core,hs_err*.log' + BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}" + } + + parameters { + booleanParam(name: 'DEBUG', + defaultValue: false, + description: 'Print extra outputs for debugging the jenkins job and yetus') + } + + stages { + stage ('precommit checks') { + parallel { + stage ('yetus patch ctests') { + agent { + node { + label 'Hadoop' + } + } + environment { + WORKDIR_REL = "hbase-native-client-precommit" + WORKDIR = "${WORKSPACE}/${WORKDIR_REL}" + SOURCEDIR = "${WORKDIR}/${SRC_REL}" + PATCHDIR = "${WORKDIR}/${PATCH_REL}" + BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}" + DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}" + YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}" + YETUSDIR = "${WORKDIR}/${YETUS_REL}" + } + steps { + dir("${SOURCEDIR}") { + checkout scm + } + dir("${YETUSDIR}") { + checkout([ + $class : 'GitSCM', + branches : [[name: "${YETUS_VERSION}"]], + userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]] + ) + } + dir("${WORKDIR}") { + withCredentials([ + usernamePassword( + credentialsId: 'apache-hbase-at-github.com', + passwordVariable: 'GITHUB_PASSWORD', + usernameVariable: 'GITHUB_USER' + )]) { + sh label: 'test-patch', script: ''' + hostname -a ; pwd ; ls -la + printenv 2>&1 | sort + echo "[INFO] Launching Yetus via ${YETUS_DRIVER}" + "${YETUS_DRIVER}" + ''' + } + } + } + post { + always { + sh label: 'zip surefire reports', script: ''' + if [ -d "${PATCHDIR}/archiver" ]; then + count=$(find "${PATCHDIR}/archiver" -type f | wc -l) + if [[ 0 -ne ${count} ]]; then + echo "zipping ${count} archived files" + zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver" + else + echo "No archived files, skipping compressing." + fi + else + echo "No archiver directory, skipping compressing." + fi + ''' + // Has to be relative to WORKSPACE. + archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit" + archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*" + publishHTML target: [ + allowMissing: true, + keepAll: true, + alwaysLinkToLastBuild: true, + // Has to be relative to WORKSPACE + reportDir: "${WORKDIR_REL}/${PATCH_REL}", + reportFiles: 'report.html', + reportName: 'PR ctests report' + ] + } + // Jenkins pipeline jobs fill slaves on PRs without this :( + cleanup() { + script { + sh label: 'Cleanup workspace', script: ''' + # See YETUS-764 + if [ -f "${PATCHDIR}/pidfile.txt" ]; then + echo "test-patch process appears to still be running: killing" + kill `cat "${PATCHDIR}/pidfile.txt"` || true + sleep 10 + fi + if [ -f "${PATCHDIR}/cidfile.txt" ]; then + echo "test-patch container appears to still be running: killing" + docker kill `cat "${PATCHDIR}/cidfile.txt"` || true + fi + # See HADOOP-13951 + chmod -R u+rxw "${WORKSPACE}" + ''' + dir ("${WORKDIR}") { + deleteDir() + } + } + } + } + } + } + } + } + + post { + // Jenkins pipeline jobs fill hosts on PRs without this :( + cleanup() { + script { + sh label: 'Cleanup workspace', script: ''' + # See HADOOP-13951 + chmod -R u+rxw "${WORKSPACE}" + ''' + deleteDir() + } + } + } +} diff --git a/bin/jenkins/gather_machine_environment.sh b/bin/jenkins/gather_machine_environment.sh new file mode 100755 index 0000000..fa99d3f --- /dev/null +++ b/bin/jenkins/gather_machine_environment.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +function usage { + echo "Usage: ${0} /path/for/output/dir" + echo "" + echo " Gather info about a build machine that test harnesses should poll before running." + echo " presumes you'll then archive the passed output dir." + + exit 1 +} + +if [ "$#" -lt 1 ]; then + usage +fi + + +declare output=$1 + +if [ ! -d "${output}" ] || [ ! -w "${output}" ]; then + echo "Specified output directory must exist and be writable." >&2 + exit 1 +fi + +echo "getting machine specs, find in ${BUILD_URL}/artifact/${output}/" +echo "JAVA_HOME: ${JAVA_HOME}" >"${output}/java_home" 2>&1 || true +ls -l "${JAVA_HOME}" >"${output}/java_home_ls" 2>&1 || true +echo "MAVEN_HOME: ${MAVEN_HOME}" >"${output}/mvn_home" 2>&1 || true +mvn --offline --version >"${output}/mvn_version" 2>&1 || true +cat /proc/cpuinfo >"${output}/cpuinfo" 2>&1 || true +cat /proc/meminfo >"${output}/meminfo" 2>&1 || true +cat /proc/diskstats >"${output}/diskstats" 2>&1 || true +cat /sys/block/sda/stat >"${output}/sys-block-sda-stat" 2>&1 || true +df -h >"${output}/df-h" 2>&1 || true +ps -Aww >"${output}/ps-Aww" 2>&1 || true +ifconfig -a >"${output}/ifconfig-a" 2>&1 || true +lsblk -ta >"${output}/lsblk-ta" 2>&1 || true +lsblk -fa >"${output}/lsblk-fa" 2>&1 || true +ulimit -l >"${output}/ulimit-l" 2>&1 || true +uptime >"${output}/uptime" 2>&1 || true diff --git a/bin/run_tests_with_yetus.sh b/bin/run_tests_with_yetus.sh new file mode 100755 index 0000000..633a234 --- /dev/null +++ b/bin/run_tests_with_yetus.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eo pipefail + +if [[ "true" = "${DEBUG}" ]]; then + set -x + printenv +fi + +declare -i missing_env=0 +declare -a required_envs=( + # these ENV variables define the required API with Jenkinsfile_GitHub + "ARCHIVE_PATTERN_LIST" + "BUILD_URL_ARTIFACTS" + "DOCKERFILE" + "GITHUB_PASSWORD" + "GITHUB_USER" + "PATCHDIR" + "SOURCEDIR" + "YETUSDIR" +) +# Validate params +for required_env in "${required_envs[@]}"; do + if [ -z "${!required_env}" ]; then + echo "[ERROR] Required environment variable '${required_env}' is not set." + missing_env=${missing_env}+1 + fi +done + +if [ ${missing_env} -gt 0 ]; then + echo "[ERROR] Please set the required environment variables before invoking. If this error is " \ + "on Jenkins, then please file a JIRA about the error." + exit 1 +fi + +# this must be clean for every run +rm -rf "${PATCHDIR}" +mkdir -p "${PATCHDIR}" + +# Gather machine information +mkdir "${PATCHDIR}/machine" +"${SOURCEDIR}/bin/jenkins/gather_machine_environment.sh" "${PATCHDIR}/machine" + +YETUS_ARGS+=("--archive-list=${ARCHIVE_PATTERN_LIST}") +YETUS_ARGS+=("--ignore-unknown-options=true") +YETUS_ARGS+=("--patch-dir=${PATCHDIR}") +# where the source is located +YETUS_ARGS+=("--basedir=${SOURCEDIR}") +# lots of different output formats +YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt") +YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html") +# enable writing back to Github +YETUS_ARGS+=("--github-password=${GITHUB_PASSWORD}") +YETUS_ARGS+=("--github-user=${GITHUB_USER}") +# rsync these files back into the archive dir +YETUS_ARGS+=("--archive-list=${ARCHIVE_PATTERN_LIST}") +# URL for user-side presentation in reports and such to our artifacts +YETUS_ARGS+=("--build-url-artifacts=${BUILD_URL_ARTIFACTS}") +# run in docker mode and specifically point to our +YETUS_ARGS+=("--docker") +YETUS_ARGS+=("--dockerfile=${DOCKERFILE}") + +# TODO (HBASE-23900): cannot assume test-patch runs directly from sources +TESTPATCHBIN="${YETUSDIR}/precommit/src/main/shell/test-patch.sh" + +if [[ "true" = "${DEBUG}" ]]; then + YETUS_ARGS=(--debug "${YETUS_ARGS[@]}") +fi + +# Sanity checks for downloaded Yetus binaries. +if [ ! -x "${TESTPATCHBIN}" ]; then + echo "Something is amiss with Yetus download." + exit 1 +fi + +# Runs in docker/non-docker mode. In docker mode, we pass the appropriate docker file with +# all the dependencies installed as a part of init. +if [[ "true" = "${RUN_IN_DOCKER}" ]]; then + YETUS_ARGS=( + --docker \ + "--dockerfile=${SOURCEDIR}/docker-files/Dockerfile" \ + "${YETUS_ARGS[@]}" + ) +fi + +echo "Using YETUS_ARGS: ${YETUS_ARGS[*]}" + +# shellcheck disable=SC2068 +/bin/bash "${TESTPATCHBIN}" \ + --personality="${SOURCEDIR}/bin/hbase-native-client-personality.sh" \ + --run-tests \ + ${YETUS_ARGS[@]} diff --git a/bin/start-docker.sh b/bin/start-docker.sh index 20db789..7c3c808 100755 --- a/bin/start-docker.sh +++ b/bin/start-docker.sh @@ -39,7 +39,7 @@ fi; # Build the image # # This shouldn't be needed after the development environment is a little more stable. -docker build -t hbase_native -f docker-files/Dockerfile . +docker build -t hbase_native -f docker-files/Dockerfile docker-files # After the image is built run the thing docker run --privileged=true -h="securecluster" -p 16050:16050/tcp \ diff --git a/docker-files/Dockerfile b/docker-files/Dockerfile index 0a735b4..3cad6d2 100644 --- a/docker-files/Dockerfile +++ b/docker-files/Dockerfile @@ -15,6 +15,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This Dockerfile is shared for both personal use as well as precommit testing. Certain changes might break Yetus +# precommit test-patch integration, so be mindful of that when making changes. + FROM ubuntu:16.04 ARG CC=/usr/bin/gcc-5 @@ -71,10 +74,17 @@ RUN apt-get update && \ echo 'xst -k hbase-host.keytab hbase/securecluster@EXAMPLE.COM' >> /tmp/krb-princ.pass ; \ kadmin.local < /tmp/krb-princ.pass ; -COPY docker-files/krb5.conf /etc +COPY krb5.conf /etc RUN echo "enabled=1" >> /etc/default/apport -ENTRYPOINT /usr/sbin/krb5kdc -P /var/run/krb5kdc.pid && echo "/tmp/core.%h.%e.%t" >> /proc/sys/kernel/core_pattern && sysctl -p && ulimit -c unlimited && /bin/bash - WORKDIR /usr/src/hbase/hbase-native-client + +CMD ["/usr/sbin/krb5kdc -P /var/run/krb5kdc.pid && sysctl -p && ulimit -c unlimited && /bin/bash"] + +### +# Everything past this point is either not needed for testing or breaks Yetus. +# So tell Yetus not to read the rest of the file: +# YETUS CUT HERE +### +