merge

.bazelrc

@@ -95,6 +95,7 @@ test:asan --test_env=ASAN_OPTIONS="detect_leaks=0"
 test:asan --test_env=LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libasan.so.2 /usr/lib/gcc/x86_64-linux-gnu/7/libasan.so"
 # For example, for Ubuntu 18.04 libasan can be found here:
 # test:asan --test_env=LD_PRELOAD="/usr/lib/gcc/x86_64-linux-gnu/7/libasan.so"
+test:asan-buildkite --test_env=LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libasan.so.5"
 
 # CI configuration:
 aquery:ci --color=no

.buildkite/Dockerfile

@@ -2,18 +2,33 @@ FROM ubuntu:focal
 
 ARG REMOTE_CACHE_URL
 ARG BUILDKITE_PULL_REQUEST
+ARG BUILDKITE_COMMIT
+ARG BUILDKITE_PULL_REQUEST_BASE_BRANCH
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV TZ=America/Los_Angeles
+
 ENV BUILDKITE=true
 ENV CI=true
 ENV PYTHON=3.6
+ENV RAY_USE_RANDOM_PORTS=1
+ENV RAY_DEFAULT_BUILD=1
+ENV BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+ENV BUILDKITE_COMMIT=${BUILDKITE_COMMIT}
+ENV BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
 
 RUN apt-get update -qq
 RUN apt-get install -y -qq \
     curl python-is-python3 git build-essential \
-    sudo unzip apt-utils dialog tzdata wget
+    sudo unzip apt-utils dialog tzdata wget rsync \
+    language-pack-en tmux cmake gdb vim htop \
+    libgtk2.0-dev zlib1g-dev libgl1-mesa-dev
+
+# System conf for tests
 RUN locale -a
+ENV LC_ALL=en_US.utf8
+ENV LANG=en_US.utf8
+RUN echo "ulimit -c 0" >> /root/.bashrc
 
 # Setup Bazel caches
 RUN (echo "build --remote_cache=${REMOTE_CACHE_URL}" >> /root/.bazelrc); \
@@ -27,3 +42,7 @@ WORKDIR /ray
 COPY . .
 RUN ./ci/travis/ci.sh init
 RUN bash --login -i ./ci/travis/ci.sh build
+
+# Run determine test to run
+RUN bash --login -i -c "python ./ci/travis/determine_tests_to_run.py --output=json > affected_set.json"
+RUN cat affected_set.json

.buildkite/pipeline.yml

@@ -1,6 +1,184 @@
-- label: "Ray Core Tests (:buildkite: Experimental)"
+- label: ":book: Lint"
   commands:
-  - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only -- //:all -rllib/...
-- label: "Ray Dashboard Tests"
+    - export LINT=1
+    - ./ci/travis/install-dependencies.sh
+    - ./ci/travis/ci.sh lint
+    - ./ci/travis/ci.sh build
+
+- label: ":java: Java"
+  conditions: ["RAY_CI_JAVA_AFFECTED"]
   commands:
-  - bazel test --config=ci $(./scripts/bazel_export_options) python/ray/new_dashboard/...
+    - apt-get install -y openjdk-8-jdk maven clang-format
+    # Compile Java again so bazel will compile Java as a language.
+    - RAY_INSTALL_JAVA=1 ./ci/travis/ci.sh build
+    - ./java/test.sh
+
+- label: ":java: Streaming"
+  conditions:
+    ["RAY_CI_STREAMING_PYTHON_AFFECTED", "RAY_CI_STREAMING_JAVA_AFFECTED"]
+  commands:
+    - apt-get install -y openjdk-8-jdk maven
+    # Compile Java again so bazel will compile Java as a language.
+    - RAY_INSTALL_JAVA=1 ./ci/travis/ci.sh build
+    - bazel test --config=ci $(./scripts/bazel_export_options)
+      //streaming:all
+    - bash streaming/src/test/run_streaming_queue_test.sh
+
+- label: ":cpp: Worker"
+  commands:
+    - ./ci/travis/ci.sh test_cpp
+
+- label: ":cpp: Tests"
+  commands:
+    - bazel test --config=ci $(./scripts/bazel_export_options)
+      --build_tests_only
+      -- //:all -rllib/... -core_worker_test
+
+- label: ":cpp: Tests (ASAN)"
+  commands:
+    - bazel test --config=ci --config=asan $(./scripts/bazel_export_options)
+      --build_tests_only
+      --config=asan-buildkite
+      --jobs=2
+      -- //:all -//:core_worker_test
+
+- label: ":serverless: Dashboard + Serve Tests"
+  conditions:
+    [
+      "RAY_CI_SERVE_AFFECTED",
+      "RAY_CI_DASHBOARD_AFFECTED",
+      "RAY_CI_PYTHON_AFFECTED",
+    ]
+  commands:
+    - TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - bazel test --config=ci $(./scripts/bazel_export_options)
+      python/ray/new_dashboard/...
+    - bazel test --config=ci $(./scripts/bazel_export_options)
+      python/ray/serve/...
+
+- label: ":python: (Small & Large)"
+  conditions: ["RAY_CI_PYTHON_AFFECTED"]
+  commands:
+    - bazel test --config=ci $(./scripts/bazel_export_options)
+      --test_tag_filters=-kubernetes,-jenkins_only,-medium_size_python_tests_a_to_j,-medium_size_python_tests_k_to_z
+      python/ray/tests/...
+    - bazel test --config=ci $(./scripts/bazel_export_options)
+      --test_tag_filters=-kubernetes,-jenkins_only,client_tests
+      --test_env=RAY_CLIENT_MODE=1
+      python/ray/tests/...
+- label: ":python: (Medium A-J)"
+  conditions: ["RAY_CI_PYTHON_AFFECTED"]
+  commands:
+    - bazel test --config=ci $(./scripts/bazel_export_options)
+      --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_a_to_j
+      python/ray/tests/...
+- label: ":python: (Medium K-Z)"
+  conditions: ["RAY_CI_PYTHON_AFFECTED"]
+  commands:
+    - bazel test --config=ci $(./scripts/bazel_export_options)
+      --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_k_to_z
+      python/ray/tests/...
+
+- label: ":brain: RLlib: Learning tests (from rllib/tuned_examples/*.yaml)"
+  conditions: ["RAY_CI_RLLIB_AFFECTED"]
+  commands:
+    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - bazel test --config=ci $(./scripts/bazel_export_options)
+      --build_tests_only
+      --test_tag_filters=learning_tests_tf
+      rllib/...
+- label: ":brain: RLlib: Learning tests with tf=1.x (from rllib/tuned_examples/*.yaml)"
+  conditions: ["RAY_CI_RLLIB_AFFECTED"]
+  commands:
+    - RLLIB_TESTING=1 TF_VERSION=1.14.0 TFP_VERSION=0.7 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - bazel test --config=ci $(./scripts/bazel_export_options)
+      --build_tests_only
+      --test_tag_filters=learning_tests_tf
+      rllib/...
+- label: ":brain: RLlib: Learning tests with Torch (from rllib/tuned_examples/*.yaml)"
+  conditions: ["RAY_CI_RLLIB_AFFECTED"]
+  commands:
+    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - bazel test --config=ci $(./scripts/bazel_export_options)
+      --build_tests_only
+      --test_tag_filters=learning_tests_torch
+      rllib/...
+- label: ":brain: RLlib: Quick Agent train.py runs"
+  conditions: ["RAY_CI_RLLIB_AFFECTED"]
+  commands:
+    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - bazel test --config=ci $(./scripts/bazel_export_options)
+      --build_tests_only
+      --test_tag_filters=quick_train
+      --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
+      rllib/...
+    # Test everything that does not have any of the "main" labels:
+    # "learning_tests|quick_train|examples|tests_dir".
+    - bazel test --config=ci $(./scripts/bazel_export_options)
+      --build_tests_only
+      --test_tag_filters=-learning_tests_tf,-learning_tests_torch,-quick_train,-examples,-tests_dir
+      --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
+      rllib/...
+- label: ":brain: RLlib: rllib/examples/"
+  conditions: ["RAY_CI_RLLIB_AFFECTED"]
+  commands:
+    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
+      --test_tag_filters=examples_A,examples_B --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/...
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
+      --test_tag_filters=examples_C,examples_D --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/...
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
+      --test_tag_filters=examples_E,examples_F,examples_G,examples_H,examples_I,examples_J,examples_K,examples_L,examples_M,examples_N,examples_O,examples_P --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
+      rllib/...
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
+      --test_tag_filters=examples_Q,examples_R,examples_S,examples_T,examples_U,examples_V,examples_W,examples_X,examples_Y,examples_Z --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
+      rllib/...
+- label: ":brain: RLlib: rllib/tests/ (A-L)"
+  conditions: ["RAY_CI_RLLIB_AFFECTED"]
+  commands:
+    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
+      --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I,tests_dir_J,tests_dir_K,tests_dir_L --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
+      rllib/...
+- label: ":brain: RLlib: rllib/tests/ (M-Z)"
+  conditions: ["RAY_CI_RLLIB_AFFECTED"]
+  commands:
+    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
+      --test_tag_filters=tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
+      rllib/...
+
+- label: ":octopus: Tune tests and examples"
+  conditions: ["RAY_CI_TUNE_AFFECTED"]
+  commands:
+    - TUNE_TESTING=1 ./ci/travis/install-dependencies.sh
+    - bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-jenkins_only,-example python/ray/tune/...
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=example,-tf,-pytorch,-py37,-flaky python/ray/tune/...
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37,-flaky python/ray/tune/...
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37,-flaky python/ray/tune/...
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-py37,flaky python/ray/tune/...
+
+- label: ":octopus: SGD tests and examples"
+  conditions: ["RAY_CI_SGD_AFFECTED"]
+  commands:
+    - SGD_TESTING=1 ./ci/travis/install-dependencies.sh
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37 python/ray/util/sgd/...
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37 python/ray/util/sgd/...
+
+- label: ":octopus: Tune/SGD tests and examples. Python 3.7"
+  conditions: ["RAY_CI_TUNE_AFFECTED", "RAY_CI_SGD_AFFECTED"]
+  commands:
+    - TUNE_TESTING=1 PYTHON=3.7 INSTALL_HOROVOD=1 ./ci/travis/install-dependencies.sh
+    # Bcause Python version changed, we need to re-install Ray here
+    - rm -rf ./python/ray/thirdparty_files; ./ci/travis/ci.sh build
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=py37 python/ray/tune/...
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/util/xgboost/...
+
+- label: ":book: Doc tests and examples"
+  conditions:
+    ["RAY_CI_PYTHON_AFFECTED", "RAY_CI_TUNE_AFFECTED", "RAY_CI_DOC_AFFECTED"]
+  commands:
+    - DOC_TESTING=1 ./ci/travis/install-dependencies.sh
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,-pytorch,-py37 doc/...
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37 doc/...
+    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37 doc/...

.travis.yml

@@ -78,7 +78,9 @@ matrix:
         - . ./ci/travis/ci.sh build
       script:
         # Run all C++ unit tests with ASAN enabled. ASAN adds too much overhead to run Python tests.
-        - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only -- //:all
+        # NOTE: core_worker_test is out-of-date and should already covered by
+        # Python tests.
+        - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only -- //:all -core_worker_test
 
     - os: osx
       osx_image: xcode7
@@ -195,6 +197,7 @@ matrix:
       env:
         # - PYTHON=3.6
         - LINUX_WHEELS=1 LINUX_JARS=1
+        - DOCKER_BUILD_PY37=1
         - PYTHONWARNINGS=ignore
         - RAY_INSTALL_JAVA=1
       language: java
@@ -207,10 +210,32 @@ matrix:
         - . ./ci/travis/ci.sh test_wheels
         - export PATH="$HOME/miniconda3/bin:$PATH"
         - python -m pip install docker
-        - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py; fi
+        - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py PY37; fi
         - bash ./java/build-jar-multiplatform.sh linux
       cache: false
 
+
+    # Build Py36 & Py38 Docker Images
+    - os: linux
+      env:
+        - LINUX_WHEELS=1
+        - DOCKER_BUILD_PY36_38=1
+        - PYTHONWARNINGS=ignore
+      language: java
+      jdk: openjdk8
+      install:
+        - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED
+      before_script:
+        - . ./ci/travis/ci.sh build
+      script:
+        - wget --quiet "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" -O miniconda3.sh
+        - bash miniconda3.sh -b -p "$HOME/miniconda3"
+        - export PATH="$HOME/miniconda3/bin:$PATH"
+        - conda install -y python=3.7.6
+        - python -m pip install docker
+        - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py PY36_PY38; fi
+      cache: false
+
     # Build and deploy multi-platform jars.
     - os: linux
       env:
@@ -418,6 +443,7 @@ matrix:
       script:
         - ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=py37 python/ray/tune/...
         - ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/util/xgboost/...
+        - ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/util/lightning_accelerators/...
         # There are no python 3.7 tests for RaySGD at the moment
         # - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=py37 python/ray/util/sgd/...
         # - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=py37 doc/...
@@ -435,11 +461,10 @@ matrix:
       script:
         - . ./ci/travis/ci.sh test_cpp
 script:
-  # raylet integration tests (core_worker_tests included in bazel tests below)
-  - ./ci/suppress_output bash src/ray/test/run_object_manager_tests.sh
-
   # cc bazel tests (w/o RLlib)
-  - ./ci/suppress_output bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only -- //:all -rllib/...
+  # NOTE: core_worker_test is out-of-date and should already covered by Python
+  # tests.
+  - ./ci/suppress_output bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only -- //:all -rllib/... -core_worker_test
 
   # ray serve tests
   - if [ $RAY_CI_SERVE_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-jenkins_only python/ray/serve/...; fi
@@ -469,7 +494,7 @@ deploy:
     on:
       repo: ray-project/ray
       all_branches: true
-      condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1
+      condition: ($LINUX_WHEELS = 1 && $DOCKER_BUILD_PY37=1) || $MAC_WHEELS = 1
 
   - provider: s3
     edge: true # This supposedly opts in to deploy v2.
@@ -485,16 +510,16 @@ deploy:
     on:
       branch: master
       repo: ray-project/ray
-      condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1
+      condition: ($LINUX_WHEELS = 1 && $DOCKER_BUILD_PY37=1) || $MAC_WHEELS = 1
 
   - provider: script
     edge: true # This supposedly opts in to deploy v2.
-    script: export PATH="$HOME/miniconda3/bin:$PATH"; ./ci/keep_alive python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py
+    script: export PATH="$HOME/miniconda3/bin:$PATH"; ./ci/keep_alive python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py PY37
     skip_cleanup: true
     on:
       repo: ray-project/ray
       all_branches: true
-      condition: $LINUX_WHEELS = 1
+      condition: $LINUX_WHEELS = 1 && $DOCKER_BUILD_PY37 = 1
 
   # Upload jars so that we can debug locally for every commit
   - provider: s3
@@ -528,3 +553,12 @@ deploy:
       repo: ray-project/ray
       branch: master
       condition: $MULTIPLATFORM_JARS = 1 || $MAC_JARS = 1 || $LINUX_JARS = 1
+
+  - provider: script
+    edge: true # This supposedly opts in to deploy v2.
+    script: export PATH="$HOME/miniconda3/bin:$PATH"; ./ci/keep_alive python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py PY36_PY38
+    skip_cleanup: true
+    on:
+      repo: ray-project/ray
+      all_branches: true
+      condition: $LINUX_WHEELS = 1 && $DOCKER_BUILD_PY36_38 = 1