Revert "merge"

.bazelrc

@@ -95,7 +95,6 @@ test:asan --test_env=ASAN_OPTIONS="detect_leaks=0"
 test:asan --test_env=LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libasan.so.2 /usr/lib/gcc/x86_64-linux-gnu/7/libasan.so"
 # For example, for Ubuntu 18.04 libasan can be found here:
 # test:asan --test_env=LD_PRELOAD="/usr/lib/gcc/x86_64-linux-gnu/7/libasan.so"
-test:asan-buildkite --test_env=LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libasan.so.5"
 
 # CI configuration:
 aquery:ci --color=no

.buildkite/Dockerfile

@@ -2,33 +2,18 @@ FROM ubuntu:focal
 
 ARG REMOTE_CACHE_URL
 ARG BUILDKITE_PULL_REQUEST
-ARG BUILDKITE_COMMIT
-ARG BUILDKITE_PULL_REQUEST_BASE_BRANCH
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV TZ=America/Los_Angeles
-
 ENV BUILDKITE=true
 ENV CI=true
 ENV PYTHON=3.6
-ENV RAY_USE_RANDOM_PORTS=1
-ENV RAY_DEFAULT_BUILD=1
-ENV BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
-ENV BUILDKITE_COMMIT=${BUILDKITE_COMMIT}
-ENV BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
 
 RUN apt-get update -qq
 RUN apt-get install -y -qq \
     curl python-is-python3 git build-essential \
-    sudo unzip apt-utils dialog tzdata wget rsync \
-    language-pack-en tmux cmake gdb vim htop \
-    libgtk2.0-dev zlib1g-dev libgl1-mesa-dev
-
-# System conf for tests
+    sudo unzip apt-utils dialog tzdata wget
 RUN locale -a
-ENV LC_ALL=en_US.utf8
-ENV LANG=en_US.utf8
-RUN echo "ulimit -c 0" >> /root/.bashrc
 
 # Setup Bazel caches
 RUN (echo "build --remote_cache=${REMOTE_CACHE_URL}" >> /root/.bazelrc); \
@@ -42,7 +27,3 @@ WORKDIR /ray
 COPY . .
 RUN ./ci/travis/ci.sh init
 RUN bash --login -i ./ci/travis/ci.sh build
-
-# Run determine test to run
-RUN bash --login -i -c "python ./ci/travis/determine_tests_to_run.py --output=json > affected_set.json"
-RUN cat affected_set.json

.buildkite/pipeline.yml

@@ -1,184 +1,6 @@
-- label: ":book: Lint"
+- label: "Ray Core Tests (:buildkite: Experimental)"
   commands:
-    - export LINT=1
-    - ./ci/travis/install-dependencies.sh
-    - ./ci/travis/ci.sh lint
-    - ./ci/travis/ci.sh build
-
-- label: ":java: Java"
-  conditions: ["RAY_CI_JAVA_AFFECTED"]
+  - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only -- //:all -rllib/...
+- label: "Ray Dashboard Tests"
   commands:
-    - apt-get install -y openjdk-8-jdk maven clang-format
-    # Compile Java again so bazel will compile Java as a language.
-    - RAY_INSTALL_JAVA=1 ./ci/travis/ci.sh build
-    - ./java/test.sh
-
-- label: ":java: Streaming"
-  conditions:
-    ["RAY_CI_STREAMING_PYTHON_AFFECTED", "RAY_CI_STREAMING_JAVA_AFFECTED"]
-  commands:
-    - apt-get install -y openjdk-8-jdk maven
-    # Compile Java again so bazel will compile Java as a language.
-    - RAY_INSTALL_JAVA=1 ./ci/travis/ci.sh build
-    - bazel test --config=ci $(./scripts/bazel_export_options)
-      //streaming:all
-    - bash streaming/src/test/run_streaming_queue_test.sh
-
-- label: ":cpp: Worker"
-  commands:
-    - ./ci/travis/ci.sh test_cpp
-
-- label: ":cpp: Tests"
-  commands:
-    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --build_tests_only
-      -- //:all -rllib/... -core_worker_test
-
-- label: ":cpp: Tests (ASAN)"
-  commands:
-    - bazel test --config=ci --config=asan $(./scripts/bazel_export_options)
-      --build_tests_only
-      --config=asan-buildkite
-      --jobs=2
-      -- //:all -//:core_worker_test
-
-- label: ":serverless: Dashboard + Serve Tests"
-  conditions:
-    [
-      "RAY_CI_SERVE_AFFECTED",
-      "RAY_CI_DASHBOARD_AFFECTED",
-      "RAY_CI_PYTHON_AFFECTED",
-    ]
-  commands:
-    - TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
-    - bazel test --config=ci $(./scripts/bazel_export_options)
-      python/ray/new_dashboard/...
-    - bazel test --config=ci $(./scripts/bazel_export_options)
-      python/ray/serve/...
-
-- label: ":python: (Small & Large)"
-  conditions: ["RAY_CI_PYTHON_AFFECTED"]
-  commands:
-    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --test_tag_filters=-kubernetes,-jenkins_only,-medium_size_python_tests_a_to_j,-medium_size_python_tests_k_to_z
-      python/ray/tests/...
-    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --test_tag_filters=-kubernetes,-jenkins_only,client_tests
-      --test_env=RAY_CLIENT_MODE=1
-      python/ray/tests/...
-- label: ":python: (Medium A-J)"
-  conditions: ["RAY_CI_PYTHON_AFFECTED"]
-  commands:
-    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_a_to_j
-      python/ray/tests/...
-- label: ":python: (Medium K-Z)"
-  conditions: ["RAY_CI_PYTHON_AFFECTED"]
-  commands:
-    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --test_tag_filters=-kubernetes,-jenkins_only,medium_size_python_tests_k_to_z
-      python/ray/tests/...
-
-- label: ":brain: RLlib: Learning tests (from rllib/tuned_examples/*.yaml)"
-  conditions: ["RAY_CI_RLLIB_AFFECTED"]
-  commands:
-    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
-    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --build_tests_only
-      --test_tag_filters=learning_tests_tf
-      rllib/...
-- label: ":brain: RLlib: Learning tests with tf=1.x (from rllib/tuned_examples/*.yaml)"
-  conditions: ["RAY_CI_RLLIB_AFFECTED"]
-  commands:
-    - RLLIB_TESTING=1 TF_VERSION=1.14.0 TFP_VERSION=0.7 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
-    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --build_tests_only
-      --test_tag_filters=learning_tests_tf
-      rllib/...
-- label: ":brain: RLlib: Learning tests with Torch (from rllib/tuned_examples/*.yaml)"
-  conditions: ["RAY_CI_RLLIB_AFFECTED"]
-  commands:
-    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
-    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --build_tests_only
-      --test_tag_filters=learning_tests_torch
-      rllib/...
-- label: ":brain: RLlib: Quick Agent train.py runs"
-  conditions: ["RAY_CI_RLLIB_AFFECTED"]
-  commands:
-    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
-    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --build_tests_only
-      --test_tag_filters=quick_train
-      --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
-      rllib/...
-    # Test everything that does not have any of the "main" labels:
-    # "learning_tests|quick_train|examples|tests_dir".
-    - bazel test --config=ci $(./scripts/bazel_export_options)
-      --build_tests_only
-      --test_tag_filters=-learning_tests_tf,-learning_tests_torch,-quick_train,-examples,-tests_dir
-      --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
-      rllib/...
-- label: ":brain: RLlib: rllib/examples/"
-  conditions: ["RAY_CI_RLLIB_AFFECTED"]
-  commands:
-    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
-      --test_tag_filters=examples_A,examples_B --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/...
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
-      --test_tag_filters=examples_C,examples_D --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1 rllib/...
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
-      --test_tag_filters=examples_E,examples_F,examples_G,examples_H,examples_I,examples_J,examples_K,examples_L,examples_M,examples_N,examples_O,examples_P --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
-      rllib/...
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
-      --test_tag_filters=examples_Q,examples_R,examples_S,examples_T,examples_U,examples_V,examples_W,examples_X,examples_Y,examples_Z --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
-      rllib/...
-- label: ":brain: RLlib: rllib/tests/ (A-L)"
-  conditions: ["RAY_CI_RLLIB_AFFECTED"]
-  commands:
-    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
-      --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I,tests_dir_J,tests_dir_K,tests_dir_L --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
-      rllib/...
-- label: ":brain: RLlib: rllib/tests/ (M-Z)"
-  conditions: ["RAY_CI_RLLIB_AFFECTED"]
-  commands:
-    - RLLIB_TESTING=1 TF_VERSION=2.1.0 TFP_VERSION=0.8 TORCH_VERSION=1.6 ./ci/travis/install-dependencies.sh
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only
-      --test_tag_filters=tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z --test_env=RAY_USE_MULTIPROCESSING_CPU_COUNT=1
-      rllib/...
-
-- label: ":octopus: Tune tests and examples"
-  conditions: ["RAY_CI_TUNE_AFFECTED"]
-  commands:
-    - TUNE_TESTING=1 ./ci/travis/install-dependencies.sh
-    - bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-jenkins_only,-example python/ray/tune/...
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=example,-tf,-pytorch,-py37,-flaky python/ray/tune/...
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37,-flaky python/ray/tune/...
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37,-flaky python/ray/tune/...
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-py37,flaky python/ray/tune/...
-
-- label: ":octopus: SGD tests and examples"
-  conditions: ["RAY_CI_SGD_AFFECTED"]
-  commands:
-    - SGD_TESTING=1 ./ci/travis/install-dependencies.sh
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37 python/ray/util/sgd/...
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37 python/ray/util/sgd/...
-
-- label: ":octopus: Tune/SGD tests and examples. Python 3.7"
-  conditions: ["RAY_CI_TUNE_AFFECTED", "RAY_CI_SGD_AFFECTED"]
-  commands:
-    - TUNE_TESTING=1 PYTHON=3.7 INSTALL_HOROVOD=1 ./ci/travis/install-dependencies.sh
-    # Bcause Python version changed, we need to re-install Ray here
-    - rm -rf ./python/ray/thirdparty_files; ./ci/travis/ci.sh build
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=py37 python/ray/tune/...
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/util/xgboost/...
-
-- label: ":book: Doc tests and examples"
-  conditions:
-    ["RAY_CI_PYTHON_AFFECTED", "RAY_CI_TUNE_AFFECTED", "RAY_CI_DOC_AFFECTED"]
-  commands:
-    - DOC_TESTING=1 ./ci/travis/install-dependencies.sh
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,-pytorch,-py37 doc/...
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=tf,-pytorch,-py37 doc/...
-    - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=-tf,pytorch,-py37 doc/...
+  - bazel test --config=ci $(./scripts/bazel_export_options) python/ray/new_dashboard/...

.travis.yml

@@ -78,9 +78,7 @@ matrix:
         - . ./ci/travis/ci.sh build
       script:
         # Run all C++ unit tests with ASAN enabled. ASAN adds too much overhead to run Python tests.
-        # NOTE: core_worker_test is out-of-date and should already covered by
-        # Python tests.
-        - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only -- //:all -core_worker_test
+        - bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only -- //:all
 
     - os: osx
       osx_image: xcode7
@@ -197,7 +195,6 @@ matrix:
       env:
         # - PYTHON=3.6
         - LINUX_WHEELS=1 LINUX_JARS=1
-        - DOCKER_BUILD_PY37=1
         - PYTHONWARNINGS=ignore
         - RAY_INSTALL_JAVA=1
       language: java
@@ -210,32 +207,10 @@ matrix:
         - . ./ci/travis/ci.sh test_wheels
         - export PATH="$HOME/miniconda3/bin:$PATH"
         - python -m pip install docker
-        - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py PY37; fi
+        - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py; fi
         - bash ./java/build-jar-multiplatform.sh linux
       cache: false
 
-
-    # Build Py36 & Py38 Docker Images
-    - os: linux
-      env:
-        - LINUX_WHEELS=1
-        - DOCKER_BUILD_PY36_38=1
-        - PYTHONWARNINGS=ignore
-      language: java
-      jdk: openjdk8
-      install:
-        - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED
-      before_script:
-        - . ./ci/travis/ci.sh build
-      script:
-        - wget --quiet "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" -O miniconda3.sh
-        - bash miniconda3.sh -b -p "$HOME/miniconda3"
-        - export PATH="$HOME/miniconda3/bin:$PATH"
-        - conda install -y python=3.7.6
-        - python -m pip install docker
-        - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py PY36_PY38; fi
-      cache: false
-
     # Build and deploy multi-platform jars.
     - os: linux
       env:
@@ -443,7 +418,6 @@ matrix:
       script:
         - ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only --test_tag_filters=py37 python/ray/tune/...
         - ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/util/xgboost/...
-        - ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only python/ray/util/lightning_accelerators/...
         # There are no python 3.7 tests for RaySGD at the moment
         # - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=py37 python/ray/util/sgd/...
         # - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=py37 doc/...
@@ -461,10 +435,11 @@ matrix:
       script:
         - . ./ci/travis/ci.sh test_cpp
 script:
+  # raylet integration tests (core_worker_tests included in bazel tests below)
+  - ./ci/suppress_output bash src/ray/test/run_object_manager_tests.sh
+
   # cc bazel tests (w/o RLlib)
-  # NOTE: core_worker_test is out-of-date and should already covered by Python
-  # tests.
-  - ./ci/suppress_output bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only -- //:all -rllib/... -core_worker_test
+  - ./ci/suppress_output bazel test --config=ci $(./scripts/bazel_export_options) --build_tests_only -- //:all -rllib/...
 
   # ray serve tests
   - if [ $RAY_CI_SERVE_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-jenkins_only python/ray/serve/...; fi
@@ -494,7 +469,7 @@ deploy:
     on:
       repo: ray-project/ray
       all_branches: true
-      condition: ($LINUX_WHEELS = 1 && $DOCKER_BUILD_PY37=1) || $MAC_WHEELS = 1
+      condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1
 
   - provider: s3
     edge: true # This supposedly opts in to deploy v2.
@@ -510,16 +485,16 @@ deploy:
     on:
       branch: master
       repo: ray-project/ray
-      condition: ($LINUX_WHEELS = 1 && $DOCKER_BUILD_PY37=1) || $MAC_WHEELS = 1
+      condition: $LINUX_WHEELS = 1 || $MAC_WHEELS = 1
 
   - provider: script
     edge: true # This supposedly opts in to deploy v2.
-    script: export PATH="$HOME/miniconda3/bin:$PATH"; ./ci/keep_alive python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py PY37
+    script: export PATH="$HOME/miniconda3/bin:$PATH"; ./ci/keep_alive python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py
     skip_cleanup: true
     on:
       repo: ray-project/ray
       all_branches: true
-      condition: $LINUX_WHEELS = 1 && $DOCKER_BUILD_PY37 = 1
+      condition: $LINUX_WHEELS = 1
 
   # Upload jars so that we can debug locally for every commit
   - provider: s3
@@ -553,12 +528,3 @@ deploy:
       repo: ray-project/ray
       branch: master
       condition: $MULTIPLATFORM_JARS = 1 || $MAC_JARS = 1 || $LINUX_JARS = 1
-
-  - provider: script
-    edge: true # This supposedly opts in to deploy v2.
-    script: export PATH="$HOME/miniconda3/bin:$PATH"; ./ci/keep_alive python $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.py PY36_PY38
-    skip_cleanup: true
-    on:
-      repo: ray-project/ray
-      all_branches: true
-      condition: $LINUX_WHEELS = 1 && $DOCKER_BUILD_PY36_38 = 1