we have installed hadoop 3.2.X and hive 3.1.2 and tez installed 0.10

build-tools/install-protobuf.sh → .asf.yaml

@@ -1,5 +1,3 @@
-#!/bin/sh
-
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -16,7 +14,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
-wget https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz
-tar -xzvf protobuf-2.5.0.tar.gz
-cd protobuf-2.5.0 && ./configure --prefix=/usr && make && sudo make install
+github:
+  description: "Apache Tez"
+  homepage: https://tez.apache.org/
+  labels:
+    - tez
+    - java
+    - apache
+    - big-data
+    - hadoop
+  features:
+    wiki: false
+    issues: false
+    projects: false
+  enabled_merge_buttons:
+    squash:  true
+    merge:   false
+    rebase:  false
+notifications:
+  commits:      commits@tez.apache.org
+  issues:       issues@tez.apache.org
+  pullrequests: issues@tez.apache.org
+  jira_options: link label worklog

.gitignore

@@ -2,6 +2,7 @@
 *.ipr
 *.iws
 *.DS_Store
+*.pyc
 .idea
 .svn
 .classpath

.travis.yml

@@ -14,18 +14,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+dist: trusty
+
 language: java
 
 sudo: required
 
 env: MAVEN_OPTS="-Xmx2G -XX:MaxPermSize=512M"
 
-jdk:
-  - oraclejdk8
-
-before_install:
-  - ./build-tools/install-protobuf.sh
-
 script:
+  - jdk_switcher use oraclejdk8
   - mvn -B clean install package -DskipTests=true -Dmaven.javadoc.skip=true
 

BUILDING.txt

@@ -9,7 +9,7 @@ Requirements:
 * JDK 1.8+
 * Maven 3.1 or later
 * Findbugs 2.0.2 or later (if running findbugs)
-* ProtocolBuffer 2.5.0
+* ProtocolBuffer 3.21.1
 * Internet connection for first build (to fetch all dependencies)
 * Hadoop version should be 2.7.0 or higher.
 
@@ -104,16 +104,29 @@ Issue with PhantomJS on building in PowerPC.
  please try installing PhantomJS manually and rerun. Refer https://github.com/ibmsoe/phantomjs-1/blob/v2.1.1-ppc64/README.md
  and install it globally for the build to work.
 
+----------------------------------------------------------------------------------
+Skip UI build:
+
+In case you want to completely skip UI build, you can use 'noui' profile.
+For instance, a full build without tests and tez-ui looks like:
+
+ $ mvn clean install -DskipTests -Pnoui
+
+It's important to note that maven will still include tez-ui project, but all of the maven plugins are skipped.
+
 ----------------------------------------------------------------------------------
 Protocol Buffer compiler:
 
-The version of Protocol Buffer compiler, protoc, must be 2.5.0 and match the
-version of the protobuf JAR.
+The version of Protocol Buffer compiler, protoc, can be defined on-the-fly as:
+ $ mvn clean install -DskipTests -pl ./tez-api -Dprotobuf.version=3.7.1
+
+The default version is defined in the root pom.xml.
 
-If you have multiple versions of protoc in your system, you can set in your 
-build shell the PROTOC_PATH environment variable to point to the one you 
-want to use for the Tez build. If you don't define this environment variable,
-protoc is looked up in the PATH.
+If you have multiple versions of protoc in your system, you can set in your
+build shell the PROTOC_PATH environment variable to point to the one you
+want to use for the Tez build. If you don't define this environment variable then the
+embedded protoc compiler will be used with the version defined in ${protobuf.version}.
+It detects the platform and executes the corresponding protoc binary at build time.
 
 You can also specify the path to protoc while building using -Dprotoc.path
 

Jenkinsfile

@@ -0,0 +1,215 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pipeline {
+
+    agent {
+        label 'Hadoop'
+    }
+
+    options {
+        buildDiscarder(logRotator(numToKeepStr: '5'))
+        timeout (time: 20, unit: 'HOURS')
+        timestamps()
+        checkoutToSubdirectory('src')
+    }
+
+    environment {
+        SOURCEDIR = 'src'
+        // will also need to change notification section below
+        PATCHDIR = 'out'
+        DOCKERFILE = "${SOURCEDIR}/build-tools/docker/Dockerfile"
+        YETUS='yetus'
+        // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
+        YETUS_VERSION='rel/0.12.0'
+
+    }
+
+    parameters {
+        string(name: 'JIRA_ISSUE_KEY',
+               defaultValue: '',
+               description: 'The JIRA issue that has a patch needing pre-commit testing. Example: HADOOP-1234')
+    }
+
+    stages {
+        stage ('install yetus') {
+            steps {
+                dir("${WORKSPACE}/${YETUS}") {
+                    checkout([
+                        $class: 'GitSCM',
+                        branches: [[name: "${env.YETUS_VERSION}"]],
+                        userRemoteConfigs: [[ url: 'https://github.com/apache/yetus']]]
+                    )
+                }
+            }
+        }
+
+        stage ('precommit-run') {
+            steps {
+                withCredentials(
+                    [usernamePassword(credentialsId: 'apache-tez-at-github.com',
+                                        passwordVariable: 'GITHUB_TOKEN',
+                                        usernameVariable: 'GITHUB_USER'),
+                                        usernamePassword(credentialsId: 'tez-ci',
+                                        passwordVariable: 'JIRA_PASSWORD',
+                                        usernameVariable: 'JIRA_USER')]) {
+                        sh '''#!/usr/bin/env bash
+
+                        set -e
+
+                        TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
+
+                        # this must be clean for every run
+                        if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
+                          rm -rf "${WORKSPACE}/${PATCHDIR}"
+                        fi
+                        mkdir -p "${WORKSPACE}/${PATCHDIR}"
+
+                        # if given a JIRA issue, process it. If CHANGE_URL is set
+                        # (e.g., Github Branch Source plugin), process it.
+                        # otherwise exit, because we don't want Hadoop to do a
+                        # full build.  We wouldn't normally do this check for smaller
+                        # projects. :)
+                        if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
+                            YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
+                        elif [[ -z "${CHANGE_URL}" ]]; then
+                            echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html"
+                            exit 0
+                        fi
+
+                        YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
+
+                        # where the source is located
+                        YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
+
+                        # our project defaults come from a personality file
+                        YETUS_ARGS+=("--project=tez")
+
+                        # lots of different output formats
+                        YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
+                        YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
+                        YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
+
+                        # enable writing back to Github
+                        YETUS_ARGS+=(--github-user="${GITHUB_USER}")
+                        YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}")
+
+                        # auto-kill any surefire stragglers during unit test runs
+                        YETUS_ARGS+=("--reapermode=kill")
+
+                        # set relatively high limits for ASF machines
+                        # changing these to higher values may cause problems
+                        # with other jobs on systemd-enabled machines
+                        YETUS_ARGS+=("--proclimit=5500")
+                        YETUS_ARGS+=("--dockermemlimit=20g")
+
+                        # -1 findbugs issues that show up prior to the patch being applied
+                        # YETUS_ARGS+=("--findbugs-strict-precheck")
+
+                        # rsync these files back into the archive dir
+                        YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml")
+
+                        # URL for user-side presentation in reports and such to our artifacts
+                        # (needs to match the archive bits below)
+                        YETUS_ARGS+=("--build-url-artifacts=artifact/out")
+
+                        # plugins to enable
+                        YETUS_ARGS+=("--plugins=all")
+
+                        # use Hadoop's bundled shelldocs
+                        YETUS_ARGS+=("--shelldocs=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/shelldocs")
+
+                        # don't let these tests cause -1s because we aren't really paying that
+                        # much attention to them
+                        YETUS_ARGS+=("--tests-filter=checkstyle")
+
+                        # run in docker mode and specifically point to our
+                        # Dockerfile since we don't want to use the auto-pulled version.
+                        YETUS_ARGS+=("--docker")
+                        YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
+                        YETUS_ARGS+=("--mvn-custom-repos")
+
+                        # effectively treat dev-suport as a custom maven module
+                        YETUS_ARGS+=("--skip-dirs=dev-support")
+
+                        # help keep the ASF boxes clean
+                        YETUS_ARGS+=("--sentinel")
+
+                        # use emoji vote so it is easier to find the broken line
+                        YETUS_ARGS+=("--github-use-emoji-vote")
+
+                        # test with Java 8 and 11
+                        YETUS_ARGS+=("--java-home=/usr/lib/jvm/java-8-openjdk-amd64")
+                        YETUS_ARGS+=("--multijdkdirs=/usr/lib/jvm/java-11-openjdk-amd64")
+                        YETUS_ARGS+=("--multijdktests=compile")
+                        YETUS_ARGS+=("--debug")
+
+                        "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
+                        '''
+                }
+            }
+        }
+
+    }
+
+    post {
+        always {
+          script {
+            // Yetus output
+            archiveArtifacts "${env.PATCHDIR}/**"
+            // Publish the HTML report so that it can be looked at
+            // Has to be relative to WORKSPACE.
+            publishHTML (target: [
+                          allowMissing: true,
+                          keepAll: true,
+                          alwaysLinkToLastBuild: true,
+                          // Has to be relative to WORKSPACE
+                          reportDir: "${env.PATCHDIR}",
+                          reportFiles: 'report.html',
+                          reportName: 'Yetus Report'
+            ])
+            // Publish JUnit results
+            try {
+                junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml"
+            } catch(e) {
+                echo 'junit processing: ' + e.toString()
+            }
+          }
+        }
+
+        // Jenkins pipeline jobs fill slaves on PRs without this :(
+        cleanup() {
+            script {
+                sh '''
+                    # See YETUS-764
+                    if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
+                      echo "test-patch process appears to still be running: killing"
+                      kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
+                      sleep 10
+                    fi
+                    if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
+                      echo "test-patch container appears to still be running: killing"
+                      docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
+                    fi
+                    # See HADOOP-13951
+                    chmod -R u+rxw "${WORKSPACE}"
+                    '''
+                deleteDir()
+            }
+        }
+    }
+}

Tez_DOAP.rdf

@@ -34,6 +34,27 @@
     <download-page rdf:resource="http://tez.apache.org/releases/" />
     <programming-language>Java</programming-language>
     <category rdf:resource="http://projects.apache.org/category/big-data" />
+    <release>
+      <Version>
+        <name>Version 0.10.2</name>
+        <created>2022-07-30</created>
+        <revision>0.10.2</revision>
+      </Version>
+    </release>
+    <release>
+      <Version>
+        <name>Version 0.10.1</name>
+        <created>2021-07-01</created>
+        <revision>0.10.1</revision>
+      </Version>
+    </release>
+    <release>
+      <Version>
+        <name>Version 0.10.0</name>
+        <created>2020-10-15</created>
+        <revision>0.10.0</revision>
+      </Version>
+    </release>
     <release>
       <Version>
         <name>Version 0.9.2</name>

build-tools/.gitignore

@@ -0,0 +1,2 @@
+protobuf
+