apache · bharathv · Jul 13, 2020 · Jun 19, 2020
diff --git a/.gitignore b/.gitignore
@@ -16,6 +16,11 @@
 
 *.swp
 
+# build directories
+build*/*
+cmake-build-debug/*
+dependencies/*
+
 # Thirdparty dirs
 third-party/*
 /gcc-debug/
@@ -25,6 +30,7 @@ src/hbase/utils/version.h
 
 # Tests
 *-test
+src/test/target/*
 
 # Executables
 load-client

diff --git a/bin/hbase-native-client-personality.sh b/bin/hbase-native-client-personality.sh
@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# You'll need a local installation of
+# [Apache Yetus' precommit checker](http://yetus.apache.org/documentation/0.1.0/#yetus-precommit)
+# to use this personality.
+#
+# Download from: http://yetus.apache.org/downloads/ . You can either grab the source artifact and
+# build from it, or use the convenience binaries provided on that download page.
+#
+# To run against, e.g. HBASE-15074 you'd then do
+# ```bash
+# test-patch --personality=bin/hbase-native-client-personality.sh HBASE-15074
+# ```
+#
+# If you want to skip the ~1 hour it'll take to do all the hadoop API checks, use
+# ```bash
+# test-patch  --plugins=all,-hadoopcheck --personality=dev-support/hbase-personality.sh HBASE-15074
+# ````
+#
+# pass the `--sentinel` flag if you want to allow test-patch to destructively alter local working
+# directory / branch in order to have things match what the issue patch requests.
+
+# Plugins enabled.
+personality_plugins "compile,unit,cmake,asflicense,github,jira,ctest,htmlout"
+
+if ! declare -f "yetus_info" >/dev/null; then
+  function yetus_info
+  {
+    echo "[$(date) INFO]: $*" 1>&2
+  }
+fi
+
+# work around yetus overwriting JAVA_HOME from our docker image
+function docker_do_env_adds
+{
+  declare k
+  for k in "${DOCKER_EXTRAENVS[@]}"; do
+    if [[ "JAVA_HOME" == "${k}" ]]; then
+      if [ -n "${JAVA_HOME}" ]; then
+        DOCKER_EXTRAARGS+=("--env=JAVA_HOME=${JAVA_HOME}")
+      fi
+    else
+      DOCKER_EXTRAARGS+=("--env=${k}=${!k}")
+    fi
+  done
+  DOCKER_EXTRAARGS+=("-h=securecluster")
+}
+
+## @description  Globals specific to this personality
+## @audience     private
+## @stability    evolving
+function personality_globals
+{
+  # shellcheck disable=SC2034
+  SUDO_USER=root
+  # shellcheck disable=SC2034
+  BUILD_NATIVE=true
+  # shellcheck disable=SC2034
+  BUILDTOOL=cmake
+  # shellcheck disable=SC2034
+  # Passed to cmake command using a custom personality.
+  CMAKE_ARGS="-DDOWNLOAD_DEPENDENCIES=ON"
+  # shellcheck disable=SC2034
+  # Expected by Yetus for compiling non-jvm projects.
+  JVM_REQUIRED=false
+  #shellcheck disable=SC2034
+  PROJECT_NAME=hbase-native-client
+  #shellcheck disable=SC2034
+  PATCH_BRANCH_DEFAULT=master
+  #shellcheck disable=SC2034
+  JIRA_ISSUE_RE='^HBASE-[0-9]+$'
+  #shellcheck disable=SC2034
+  GITHUB_REPO="apache/hbase-native-client"
+  # Yetus 0.7.0 enforces limits. Default proclimit is 1000.
+  # Up it. See HBASE-19902 for how we arrived at this number.
+  #shellcheck disable=SC2034
+  PROCLIMIT=10000
+  # Override if you want to bump up the memlimit for docker.
+  # shellcheck disable=SC2034
+  DOCKERMEMLIMIT=4g
+}
+
+## @description  Queue up modules for this personality
+## @audience     private
+## @stability    evolving
+## @param        repostatus
+## @param        testtype
+function personality_modules
+{
+  local repostatus=$1
+  local testtype=$2
+  local args
+  yetus_info "Personality: ${repostatus} ${testtype}"
+  clear_personality_queue
+  if [[ "${testtype}" =~ CMakeLists.txt ]]; then
+    args=${CMAKE_ARGS}
+    yetus_debug "Appending CMake args ${args}"
+  fi
+  personality_enqueue_module . ${args}
+}
diff --git a/bin/jenkins/Jenkinsfile_GitHub b/bin/jenkins/Jenkinsfile_GitHub
@@ -0,0 +1,165 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pipeline {
+
+    agent {
+        label 'Hadoop'
+    }
+
+    options {
+        // N.B. this is per-branch, which means per PR
+        disableConcurrentBuilds()
+        buildDiscarder(logRotator(numToKeepStr: '15'))
+        timeout (time: 7, unit: 'HOURS')
+        timestamps()
+        skipDefaultCheckout()
+    }
+
+    environment {
+        SRC_REL = 'src'
+        PATCH_REL = 'output'
+        YETUS_REL = 'yetus'
+        // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
+        YETUS_VERSION = 'rel/0.12.0'
+        DOCKERFILE_REL = "${SRC_REL}/docker-files/Dockerfile"
+        YETUS_DRIVER_REL = "${SRC_REL}/bin/run_tests_with_yetus.sh"
+        ARCHIVE_PATTERN_LIST = 'org.apache.h*.txt,*.dumpstream,*.dump,*.jstack,*.pstack,*.core,hs_err*.log'
+        BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
+    }
+
+    parameters {
+        booleanParam(name: 'DEBUG',
+               defaultValue: false,
+               description: 'Print extra outputs for debugging the jenkins job and yetus')
+    }
+
+    stages {
+        stage ('precommit checks') {
+            parallel {
+                stage ('yetus patch ctests') {
+                    agent {
+                        node {
+                            label 'Hadoop'
+                        }
+                    }
+                    environment {
+                        WORKDIR_REL = "hbase-native-client-precommit"
+                        WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
+                        SOURCEDIR = "${WORKDIR}/${SRC_REL}"
+                        PATCHDIR = "${WORKDIR}/${PATCH_REL}"
+                        BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
+                        DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
+                        YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
+                        YETUSDIR = "${WORKDIR}/${YETUS_REL}"
+                    }
+                    steps {
+                        dir("${SOURCEDIR}") {
+                            checkout scm
+                        }
+                        dir("${YETUSDIR}") {
+                            checkout([
+                              $class           : 'GitSCM',
+                              branches         : [[name: "${YETUS_VERSION}"]],
+                              userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
+                            )
+                        }
+                        dir("${WORKDIR}") {
+                            withCredentials([
+                              usernamePassword(
+                                credentialsId: 'apache-hbase-at-github.com',
+                                passwordVariable: 'GITHUB_PASSWORD',
+                                usernameVariable: 'GITHUB_USER'
+                              )]) {
+                                sh label: 'test-patch', script: '''
+                                    hostname -a ; pwd ; ls -la
+                                    printenv 2>&1 | sort
+                                    echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
+                                    "${YETUS_DRIVER}"
+                                '''
+                            }
+                        }
+                    }
+                    post {
+                        always {
+                            sh label: 'zip surefire reports', script: '''
+                                if [ -d "${PATCHDIR}/archiver" ]; then
+                                  count=$(find "${PATCHDIR}/archiver" -type f | wc -l)
+                                  if [[ 0 -ne ${count} ]]; then
+                                    echo "zipping ${count} archived files"
+                                    zip -q -m -r "${PATCHDIR}/test_logs.zip" "${PATCHDIR}/archiver"
+                                  else
+                                    echo "No archived files, skipping compressing."
+                                  fi
+                                else
+                                  echo "No archiver directory, skipping compressing."
+                                fi
+                            '''
+                            // Has to be relative to WORKSPACE.
+                            archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
+                            archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
+                            publishHTML target: [
+                              allowMissing: true,
+                              keepAll: true,
+                              alwaysLinkToLastBuild: true,
+                              // Has to be relative to WORKSPACE
+                              reportDir: "${WORKDIR_REL}/${PATCH_REL}",
+                              reportFiles: 'report.html',
+                              reportName: 'PR ctests report'
+                            ]
+                        }
+                        // Jenkins pipeline jobs fill slaves on PRs without this :(
+                        cleanup() {
+                            script {
+                                sh label: 'Cleanup workspace', script: '''
+                                    # See YETUS-764
+                                    if [ -f "${PATCHDIR}/pidfile.txt" ]; then
+                                      echo "test-patch process appears to still be running: killing"
+                                      kill `cat "${PATCHDIR}/pidfile.txt"` || true
+                                      sleep 10
+                                    fi
+                                    if [ -f "${PATCHDIR}/cidfile.txt" ]; then
+                                      echo "test-patch container appears to still be running: killing"
+                                      docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
+                                    fi
+                                    # See HADOOP-13951
+                                    chmod -R u+rxw "${WORKSPACE}"
+                                '''
+                                dir ("${WORKDIR}") {
+                                    deleteDir()
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    post {
+        // Jenkins pipeline jobs fill hosts on PRs without this :(
+        cleanup() {
+            script {
+                sh label: 'Cleanup workspace', script: '''
+                    # See HADOOP-13951
+                    chmod -R u+rxw "${WORKSPACE}"
+                    '''
+                deleteDir()
+            }
+        }
+    }
+}
diff --git a/bin/jenkins/gather_machine_environment.sh b/bin/jenkins/gather_machine_environment.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+function usage {
+  echo "Usage: ${0} /path/for/output/dir"
+  echo ""
+  echo "  Gather info about a build machine that test harnesses should poll before running."
+  echo "  presumes you'll then archive the passed output dir."
+
+  exit 1
+}
+
+if [ "$#" -lt 1 ]; then
+  usage
+fi
+
+
+declare output=$1
+
+if [ ! -d "${output}" ] || [ ! -w "${output}" ]; then
+  echo "Specified output directory must exist and be writable." >&2
+  exit 1
+fi
+
+echo "getting machine specs, find in ${BUILD_URL}/artifact/${output}/"
+echo "JAVA_HOME: ${JAVA_HOME}" >"${output}/java_home" 2>&1 || true
+ls -l "${JAVA_HOME}" >"${output}/java_home_ls" 2>&1 || true
+echo "MAVEN_HOME: ${MAVEN_HOME}" >"${output}/mvn_home" 2>&1 || true
+mvn --offline --version  >"${output}/mvn_version" 2>&1 || true
+cat /proc/cpuinfo >"${output}/cpuinfo" 2>&1 || true
+cat /proc/meminfo >"${output}/meminfo" 2>&1 || true
+cat /proc/diskstats >"${output}/diskstats" 2>&1 || true
+cat /sys/block/sda/stat >"${output}/sys-block-sda-stat" 2>&1 || true
+df -h >"${output}/df-h" 2>&1 || true
+ps -Aww >"${output}/ps-Aww" 2>&1 || true
+ifconfig -a >"${output}/ifconfig-a" 2>&1 || true
+lsblk -ta >"${output}/lsblk-ta" 2>&1 || true
+lsblk -fa >"${output}/lsblk-fa" 2>&1 || true
+ulimit -l >"${output}/ulimit-l" 2>&1 || true
+uptime >"${output}/uptime" 2>&1 || true