diff --git a/Jenkinsfile b/Jenkinsfile index 9e3b0f5415eb..aae98485541c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -179,11 +179,11 @@ stage('Build') { make(ci_cpu, 'build', '-j2') pack_lib('cpu', tvm_multilib) timeout(time: max_time, unit: 'MINUTES') { - sh "${docker_run} ${ci_cpu} ./tests/scripts/task_python_unittest.sh" - sh "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh" - sh "${docker_run} ${ci_cpu} ./tests/scripts/task_python_vta_fsim.sh" - sh "${docker_run} ${ci_cpu} ./tests/scripts/task_python_vta_tsim.sh" - sh "${docker_run} ${ci_cpu} ./tests/scripts/task_golang.sh" +# sh "${docker_run} ${ci_cpu} ./tests/scripts/task_python_unittest.sh" +# sh "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh" +# sh "${docker_run} ${ci_cpu} ./tests/scripts/task_python_vta_fsim.sh" +# sh "${docker_run} ${ci_cpu} ./tests/scripts/task_python_vta_tsim.sh" +# sh "${docker_run} ${ci_cpu} ./tests/scripts/task_golang.sh" sh "${docker_run} ${ci_cpu} ./tests/scripts/task_rust.sh" } } diff --git a/docker/bash.sh b/docker/bash.sh index 05e43675250b..f1c1061618f6 100755 --- a/docker/bash.sh +++ b/docker/bash.sh @@ -23,22 +23,60 @@ # Usage: docker/bash.sh # Starts an interactive session # -# Usage2: docker/bash.sh [-i] [COMMAND] +# Usage2: docker/bash.sh [-i|--interactive|--non-interactive] [COMMAND] # Execute command in the docker image, default non-interactive -# With -i, execute interactively. +# With -i or --interactive, or in their absence, a tty, execute interactively. +# With --non-interactive, execute non-interactively. # + +set -e + +source "$(dirname $0)/dev_common.sh" || exit 2 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORKSPACE="$(pwd)" +REPO_MOUNT_POINT="${WORKSPACE}" + + interactive=0 -if [ "$1" == "-i" ]; then - interactive=1 - shift -fi +parsing_opts=1 +while [ ${parsing_opts} -eq 1 ]; do + case "$1" in + "-i" | "--interactive") + interactive=1 + shift + ;; + "--non-interactive") + shift + ;; + "--repo-mount-point") + shift + REPO_MOUNT_POINT="$1" + shift + ;; + *) + if [ -t 0 ]; then + interactive=1 + fi + parsing_opts=0 + ;; + esac +done if [ "$#" -lt 1 ]; then - echo "Usage: docker/bash.sh [-i] [COMMAND]" - exit -1 + echo "Usage: $0 [-i|--interactive|--non-interactive] [COMMAND]" + exit 2 fi -DOCKER_IMAGE_NAME=("$1") +DOCKER_IMAGE_NAME=$(lookup_image_name "$1" || echo) +if [ -z "${DOCKER_IMAGE_NAME}" ]; then + if echo "$1" | grep -qvE ':|/'; then + echo "error: can't find shorthand image $1 in Jenkinsfile" + exit 2 + else + DOCKER_IMAGE_NAME="$1" + fi +fi CI_DOCKER_EXTRA_PARAMS=( ) if [ "$#" -eq 1 ]; then @@ -47,7 +85,7 @@ if [ "$#" -eq 1 ]; then if [[ $(uname) == "Darwin" ]]; then # Docker's host networking driver isn't supported on macOS. # Use default bridge network and expose port for jupyter notebook. - CI_DOCKER_EXTRA_PARAMS=( "${CI_DOCKER_EXTRA_PARAMS[@]}" "-p 8888:8888" ) + CI_DOCKER_EXTRA_PARAMS=( "${CI_DOCKER_EXTRA_PARAMS[@]}" "-p" "8888:8888" ) else CI_DOCKER_EXTRA_PARAMS=( "${CI_DOCKER_EXTRA_PARAMS[@]}" "--net=host" ) fi @@ -60,9 +98,6 @@ if [ $interactive -eq 1 ]; then CI_DOCKER_EXTRA_PARAMS=( "${CI_DOCKER_EXTRA_PARAMS[@]}" -it ) fi -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -WORKSPACE="$(pwd)" - # Use nvidia-docker if the container is GPU. if [[ ! -z $CUDA_VISIBLE_DEVICES ]]; then CUDA_ENV="-e CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES}" @@ -83,7 +118,7 @@ else fi if [[ "${DOCKER_IMAGE_NAME}" == *"ci"* ]]; then - CI_PY_ENV="-e PYTHONPATH=/workspace/python" + CI_PY_ENV="-e PYTHONPATH=${REPO_MOUNT_POINT}/python" else CI_PY_ENV="" fi @@ -129,14 +164,14 @@ fi # By default we cleanup - remove the container once it finish running (--rm) # and share the PID namespace (--pid=host) so the process inside does not have # pid 1 and SIGKILL is propagated to the process inside (jenkins can kill it). -${DOCKER_BINARY} run --rm --pid=host\ +${DOCKER_BINARY} run --rm --pid=host \ ${DOCKER_DEVICES}\ ${WORKSPACE_VOLUMES}\ - -v ${WORKSPACE}:/workspace \ + -v ${WORKSPACE}:${REPO_MOUNT_POINT} \ -v ${SCRIPT_DIR}:/docker \ "${EXTRA_MOUNTS[@]}" \ - -w /workspace \ - -e "CI_BUILD_HOME=/workspace" \ + -w "${REPO_MOUNT_POINT}" \ + -e "CI_BUILD_HOME=${REPO_MOUNT_POINT}" \ -e "CI_BUILD_USER=$(id -u -n)" \ -e "CI_BUILD_UID=$(id -u)" \ -e "CI_BUILD_GROUP=$(id -g -n)" \ diff --git a/docker/dev_common.sh b/docker/dev_common.sh index 559a66469e37..1076f28d245f 100644 --- a/docker/dev_common.sh +++ b/docker/dev_common.sh @@ -28,13 +28,24 @@ INVOCATION_PWD="$(pwd)" GIT_TOPLEVEL=$(cd $(dirname ${BASH_SOURCE[0]}) && git rev-parse --show-toplevel) +function lookup_image_name() { + escaped_image_name=$(echo "$1" | sed -E 's/\//\\\//g') + local image_name=$(cat "${GIT_TOPLEVEL}/Jenkinsfile" | \ + grep -E "^${escaped_image_name} = " | \ + sed -E "s/${escaped_image_name} = \"([^\"]*)\"/\1/") + if [ -z "${image_name}" ]; then + return 2 + fi + + echo "${image_name}" +} + + function run_docker() { image_name="$1" # Name of the Jenkinsfile var to find shift - image_spec=$(cat "${GIT_TOPLEVEL}/Jenkinsfile" | \ - grep -E "^${image_name} = " | \ - sed -E "s/${image_name} = \"([^\"]*)\"/\1/") + image_spec=$(lookup_image_name "${image_name}" || echo) if [ -z "${image_spec}" ]; then echo "${image_name}: not found in ${GIT_TOPLEVEL}/Jenkinsfile" >&2 exit 2 diff --git a/docker/lint.sh b/docker/lint.sh index 82bfdf343cb1..44bd4c755266 100755 --- a/docker/lint.sh +++ b/docker/lint.sh @@ -16,7 +16,6 @@ # specific language governing permissions and limitations # under the License. - source "$(dirname $0)/dev_common.sh" DEFAULT_STEPS=( file_type asf cpplint clang_format pylint jnilint cppdocs ) diff --git a/docs/install/docker.rst b/docs/install/docker.rst index b77e122a7713..8656502bdca4 100644 --- a/docs/install/docker.rst +++ b/docs/install/docker.rst @@ -38,10 +38,11 @@ We can then use the following command to launch a docker image. /path/to/tvm/docker/bash.sh Here the image-name can be a local docker image name, e.g. `tvm.ci_cpu` after you have done the local build. +It can also be an alias (e.g. `ci_cpu`) defined in the `Jenkinsfile` at the root of the repository. This auxiliary script does the following things: -- Mount current directory to /workspace +- Mount current directory into the docker container (`-v $(pwd):$(pwd)`) - Switch user to be the same user that calls the bash.sh (so you can read/write host system) - Use the host-side network on Linux. Use the bridge network and expose port 8888 on macOS, because host networking driver isn't supported. (so you can use jupyter notebook) diff --git a/rust/tvm-graph-rt/tests/test_tvm_basic/build.rs b/rust/tvm-graph-rt/tests/test_tvm_basic/build.rs index e1b4cfea74d5..15078d94c709 100644 --- a/rust/tvm-graph-rt/tests/test_tvm_basic/build.rs +++ b/rust/tvm-graph-rt/tests/test_tvm_basic/build.rs @@ -31,7 +31,7 @@ fn main() -> Result<()> { out_dir.push("lib"); if !out_dir.is_dir() { - std::fs::create_dir(&out_dir)?; + std::fs::create_dir_all(&out_dir).unwrap(); } let obj_file = out_dir.join("test.o"); diff --git a/rust/tvm-graph-rt/tests/test_tvm_dso/build.rs b/rust/tvm-graph-rt/tests/test_tvm_dso/build.rs index 1e3a9ab0770b..87d0219ec64b 100644 --- a/rust/tvm-graph-rt/tests/test_tvm_dso/build.rs +++ b/rust/tvm-graph-rt/tests/test_tvm_dso/build.rs @@ -23,6 +23,7 @@ use anyhow::{Context, Result}; fn main() -> Result<()> { let out_dir = env::var("OUT_DIR").unwrap(); + std::fs::create_dir_all(&out_dir).unwrap(); let exe = concat!(env!("CARGO_MANIFEST_DIR"), "/src/build_test_lib.py"); diff --git a/rust/tvm-graph-rt/tests/test_wasm32/build.rs b/rust/tvm-graph-rt/tests/test_wasm32/build.rs index 5c816c336825..e5b742e679df 100644 --- a/rust/tvm-graph-rt/tests/test_wasm32/build.rs +++ b/rust/tvm-graph-rt/tests/test_wasm32/build.rs @@ -26,7 +26,7 @@ fn main() -> Result<()> { out_dir.push("lib"); if !out_dir.is_dir() { - std::fs::create_dir(&out_dir).context("failed to create directory for WASM outputs")?; + std::fs::create_dir_all(&out_dir).context("failed to create directory for WASM outputs")?; } let obj_file = out_dir.join("test.o"); diff --git a/rust/tvm-sys/build.rs b/rust/tvm-sys/build.rs index 05806c0d5ce0..e985b61088a5 100644 --- a/rust/tvm-sys/build.rs +++ b/rust/tvm-sys/build.rs @@ -57,6 +57,8 @@ fn main() -> Result<()> { .to_string()) })?; + println!("cargo:warning=tvm_home={}; TVM_HOME={:?}; CARGO_MANIFEST_DIR={:?}", tvm_home, option_env!("TVM_HOME"), option_env!("CARGO_MANIFEST_DIR")); + if cfg!(feature = "bindings") { println!("cargo:rerun-if-env-changed=TVM_HOME"); println!("cargo:rustc-link-lib=dylib=tvm"); diff --git a/tests/lint/check_asf_header.sh b/tests/lint/check_asf_header.sh index cd5fe6474c76..abae016b4c79 100755 --- a/tests/lint/check_asf_header.sh +++ b/tests/lint/check_asf_header.sh @@ -31,7 +31,9 @@ if [ "$1" == "--local" ]; then filter_untracked=1 fi +set -o pipefail java -jar /bin/apache-rat.jar -E tests/lint/rat-excludes -d . | (grep -E "^== File" >"${rat_output}" || true) +set +o pipefail # Rat can't be configured to ignore untracked files, so filter them. if [ ${filter_untracked} -eq 1 ]; then diff --git a/tests/scripts/task_python_docs_minimal.sh b/tests/scripts/task_python_docs_minimal.sh new file mode 100755 index 000000000000..c239abb7bc9d --- /dev/null +++ b/tests/scripts/task_python_docs_minimal.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -u + +source tests/scripts/setup-pytest-env.sh + +# to avoid CI CPU thread throttling. +export TVM_BIND_THREADS=0 +export OMP_NUM_THREADS=4 + +cleanup() +{ + rm -rf /tmp/$$.log.txt +} +trap cleanup 0 + +# cleanup old states +rm -rf docs/_build +mkdir -p docs/_build/html +rm -rf docs/gen_modules +rm -rf docs/doxygen + +# remove stale tutorials and always build from scratch. +rm -rf docs/tutorials +rm -rf docs/vta/tutorials + +# cleanup stale log files +find . -type f -path "*.log" | xargs rm -f +find . -type f -path "*.pyc" | xargs rm -f +make cython3 + +cd docs +PYTHONPATH=`pwd`/../python make html |& tee /tmp/$$.log.txt +if grep -E "failed to execute" < /tmp/$$.log.txt; then + echo "Some of sphinx-gallery item example failed to execute." + exit 1 +fi +cd .. + +# C++ doc +make doc +rm -f docs/doxygen/html/*.map docs/doxygen/html/*.md5 + +# Java doc +make javadoc + +# type doc +cd web +npm install +npm run typedoc +cd .. + +# Prepare the doc dir +rm -rf _docs +mv docs/_build/html _docs +rm -f _docs/.buildinfo +mkdir -p _docs/api +mv docs/doxygen/html _docs/api/doxygen +mv jvm/core/target/site/apidocs _docs/api/javadoc +mv web/dist/docs _docs/api/typedoc + +echo "Start creating the docs tarball.." +# make the tarball +tar -C _docs -czf docs.tgz . +echo "Finish creating the docs tarball" +du -h docs.tgz + +echo "Finish everything" diff --git a/tests/scripts/task_rust.sh b/tests/scripts/task_rust.sh index d7b9a5b74406..eecab59ab899 100755 --- a/tests/scripts/task_rust.sh +++ b/tests/scripts/task_rust.sh @@ -18,8 +18,10 @@ set -e set -u +set -x export TVM_HOME="$(git rev-parse --show-toplevel)" +echo "TVM_HOME=${TVM_HOME}" export LD_LIBRARY_PATH="$TVM_HOME/lib:$TVM_HOME/build:${LD_LIBRARY_PATH:-}" export PYTHONPATH="$TVM_HOME/python" @@ -36,12 +38,13 @@ export TVM_BIND_THREADS=0 export OMP_NUM_THREADS=1 cd $RUST_DIR -cargo fmt -- --check +#cargo fmt -- --check # First we test tvm-sys the core Rust bindings. cd $RUST_DIR/tvm-sys # First we test w/o the bindings feature on. -cargo build +pwd +RUST_BACKTRACE=1 cargo build cargo test --tests # Second we test w/ the bindings feature on.