ray-project · krfricke · Jan 9, 2023 · Jan 10, 2023
diff --git a/.buildkite/pipeline.arm64.yml b/.buildkite/pipeline.arm64.yml
@@ -0,0 +1,56 @@
+- label: ":mechanical_arm: :ferris_wheel: ARM64 Linux wheels"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    # Build the wheels
+    - UPLOAD_WHEELS_AS_ARTIFACTS=1 LINUX_WHEELS=1 ./ci/ci.sh build
+    # Upload the wheels
+    # We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated.
+    - if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    # Upload to branch directory.
+    - python .buildkite/copy_files.py --destination branch_wheels --path ./.whl
+    # Upload to latest directory.
+    - if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi
+
+- label: ":mechanical_arm: :ferris_wheel: ARM64 Post-wheels tests"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    - ./ci/env/env_info.sh
+    - bazel test --config=ci $(./ci/run/bazel_export_options)
+      --test_tag_filters=post_wheel_build
+      --test_env=CONDA_EXE
+      --test_env=CONDA_PYTHON_EXE
+      --test_env=CONDA_SHLVL
+      --test_env=CONDA_PREFIX
+      --test_env=CONDA_DEFAULT_ENV
+      --test_env=CI
+      --test_env=RAY_CI_POST_WHEEL_TESTS=True
+      python/ray/tests/... python/ray/serve/... python/ray/tune/... rllib/... doc/...
+
+
+# This currently takes ~3 hours and times out often.
+# Enable this when ARM debug wheels are actually needed (and look into speeding this up).
+
+#- label: ":mechanical_arm: :ferris_wheel: ARM64 Debug Wheels"
+#  conditions:
+#    [
+#        "RAY_CI_LINUX_WHEELS_AFFECTED",
+#    ]
+#  instance_size: arm64-medium
+#  commands:
+#    # Build the debug wheels
+#    - RAY_DEBUG_BUILD=debug LINUX_WHEELS=1 ./ci/ci.sh build
+#    # Upload the wheels.
+#    # We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated.
+#    - if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi
+#    - pip install -q docker aws_requests_auth boto3
+#    - ./ci/env/env_info.sh
+#    # Upload to branch directory.
+#    - python .buildkite/copy_files.py --destination branch_wheels --path ./.whl
+#    # Upload to latest directory.
+#    - if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi
diff --git a/.buildkite/pipeline.build.yml b/.buildkite/pipeline.build.yml
@@ -200,7 +200,9 @@
   instance_size: medium
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - bazel test --config=ci --config=ubsan $(./ci/run/bazel_export_options)
+    # Unset CC CXX vars set in Dockerfile. Clang currently runs into problems with ubsan builds, this will revert to
+    # using GCC instead.
+    - unset CC CXX && bazel test --config=ci --config=ubsan $(./ci/run/bazel_export_options)
       --build_tests_only
       --jobs=2
       -- //:all -//:core_worker_test -//:logging_test -//:ray_syncer_test

diff --git a/ci/ci.sh b/ci/ci.sh
@@ -455,11 +455,14 @@ build_wheels() {
         -e "RAY_DEBUG_BUILD=${RAY_DEBUG_BUILD:-}"
       )
 
+      IMAGE_NAME="quay.io/pypa/manylinux2014_${HOSTTYPE}"
+      IMAGE_TAG="2021-11-07-28723f3"
+
       if [ -z "${BUILDKITE-}" ]; then
         # This command should be kept in sync with ray/python/README-building-wheels.md,
         # except the "${MOUNT_BAZEL_CACHE[@]}" part.
         docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" \
-        quay.io/pypa/manylinux2014_x86_64:2021-11-07-28723f3 /ray/python/build-wheel-manylinux2014.sh
+        "${IMAGE_NAME}:${IMAGE_TAG}" /ray/python/build-wheel-manylinux2014.sh
       else
         rm -rf /ray-mount/*
         rm -rf /ray-mount/.whl || true
@@ -469,7 +472,7 @@ build_wheels() {
         docker run --rm -v /ray:/ray-mounted ubuntu:focal ls /
         docker run --rm -v /ray:/ray-mounted ubuntu:focal ls /ray-mounted
         docker run --rm -w /ray -v /ray:/ray "${MOUNT_BAZEL_CACHE[@]}" \
-          quay.io/pypa/manylinux2014_x86_64:2021-11-07-28723f3 /ray/python/build-wheel-manylinux2014.sh
+          "${IMAGE_NAME}:${IMAGE_TAG}" /ray/python/build-wheel-manylinux2014.sh
         cp -rT /ray-mount /ray # copy new files back here
         find . | grep whl # testing
 

diff --git a/ci/docker/base.test.Dockerfile b/ci/docker/base.test.Dockerfile
@@ -1,4 +1,5 @@
-FROM ubuntu:focal
+ARG DOCKER_IMAGE_BASE_UBUNTU=ubuntu:focal
+FROM $DOCKER_IMAGE_BASE_UBUNTU
 
 ARG REMOTE_CACHE_URL
 ARG BUILDKITE_PULL_REQUEST
@@ -33,6 +34,7 @@ RUN apt-get install -y -qq \
     sudo unzip unrar apt-utils dialog tzdata wget rsync \
     language-pack-en tmux cmake gdb vim htop \
     libgtk2.0-dev zlib1g-dev libgl1-mesa-dev \
+    liblz4-dev libunwind-dev libncurses5 \
     clang-format-12 jq \
     clang-tidy-12 clang-12
 # Make using GCC 9 explicit.

diff --git a/ci/docker/build.Dockerfile b/ci/docker/build.Dockerfile
@@ -9,6 +9,9 @@ ENV BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
 ENV BUILDKITE_COMMIT=${BUILDKITE_COMMIT}
 ENV BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
 ENV TRAVIS_COMMIT=${BUILDKITE_COMMIT}
+# Set compiler here to build Ray with CLANG/LLVM
+ENV CC=clang
+ENV CXX=clang++-12
 
 # Move out of working dir /ray
 # Delete stale data
@@ -25,9 +28,8 @@ RUN env
 
 # init also calls install-dependencies.sh
 RUN BUILD=1 bash --login -i ./ci/ci.sh init
-RUN bash --login -i ./ci/ci.sh build
 
-RUN export CC=clang CXX=clang++-12
+RUN bash --login -i ./ci/ci.sh build
 
 # Run determine test to run
 RUN bash --login -i -c "python ./ci/pipeline/determine_tests_to_run.py --output=json > affected_set.json"

diff --git a/ci/env/install-bazel.sh b/ci/env/install-bazel.sh
@@ -5,7 +5,7 @@ ROOT_DIR=$(cd "$(dirname "$0")/$(dirname "$(test -L "$0" && readlink "$0" || ech
 
 arg1="${1-}"
 
-achitecture="${HOSTTYPE}"
+
 platform="unknown"
 case "${OSTYPE}" in
   msys)
@@ -26,6 +26,9 @@ case "${OSTYPE}" in
     exit 1
 esac
 
+architecture="${HOSTTYPE}"
+echo "Architecture is $architecture"
+
 if [ "${BAZEL_CONFIG_ONLY-}" != "1" ]; then
   # Sanity check: Verify we have symlinks where we expect them, or Bazel can produce weird "missing input file" errors.
   # This is most likely to occur on Windows, where symlinks are sometimes disabled by default.
@@ -50,26 +53,57 @@ if [ "${BAZEL_CONFIG_ONLY-}" != "1" ]; then
   if [ "${OSTYPE}" = "msys" ]; then
     target="${MINGW_DIR-/usr}/bin/bazel.exe"
     mkdir -p "${target%/*}"
-    curl -f -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-${platform}-${achitecture}.exe"
+    curl -f -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-${platform}-${architecture}.exe"
   else
-    target="./install.sh"
-    curl -f -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-installer-${platform}-${achitecture}.sh"
-    chmod +x "${target}"
+
+    # Buildkite mac instances
     if [[ -n "${BUILDKITE-}" ]] && [ "${platform}" = "darwin" ]; then
-      "${target}" --user
+      mkdir -p "$HOME/bin"
       # Add bazel to the path.
       # shellcheck disable=SC2016
       printf '\nexport PATH="$HOME/bin:$PATH"\n' >> ~/.zshrc
       # shellcheck disable=SC1090
       source ~/.zshrc
+      INSTALL_USER=1
+    # Buildkite linux instance
     elif [ "${CI-}" = true ] || [ "${arg1-}" = "--system" ]; then
-      "$(command -v sudo || echo command)" "${target}" > /dev/null  # system-wide install for CI
+      INSTALL_USER=0
+    # User
     else
-      "${target}" --user > /dev/null
+      mkdir -p "$HOME/bin"
+      INSTALL_USER=1
       export PATH=$PATH:"$HOME/bin"
     fi
-    which bazel
-    rm -f "${target}"
+
+    if [ "${architecture}" = "aarch64" ]; then
+      # architecture is "aarch64", but the bazel tag is "arm64"
+      url="https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-${platform}-arm64"
+
+      if [ "$INSTALL_USER" = "1" ]; then
+        target="$HOME/bin/bazel"
+        curl -f -s -L -R -o "${target}" "${url}"
+        chmod +x "${target}"
+      else
+        target="/bin/bazel"
+        sudo curl -f -s -L -R -o "${target}" "${url}"
+        sudo chmod +x "${target}"
+      fi
+
+      which bazel
+
+    else
+      target="./install.sh"
+      curl -f -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-installer-${platform}-${architecture}.sh"
+      chmod +x "${target}"
+
+      if [ "$INSTALL_USER" = "1" ]; then
+        "${target}" --user
+      else
+        "$(command -v sudo || echo command)" "${target}" > /dev/null  # system-wide install for CI
+      fi
+      which bazel
+      rm -f "${target}"
+    fi
   fi
 fi
 

diff --git a/ci/env/install-dependencies.sh b/ci/env/install-dependencies.sh
@@ -287,10 +287,15 @@ download_mnist() {
 install_pip_packages() {
 
   # Install modules needed in all jobs.
+  # shellcheck disable=SC2262
   alias pip="python -m pip"
 
   if [ "${MINIMAL_INSTALL-}" != 1 ]; then
+    # Some architectures will build dm-tree from source.
+    # Move bazelrc to a different location temporarily to disable --config=ci settings
+    mv "$HOME/.bazelrc" "$HOME/._brc" || true
     pip install --no-clean dm-tree==0.1.5  # --no-clean is due to: https://github.com/deepmind/tree/issues/5
+    mv "$HOME/._brc" "$HOME/.bazelrc" || true
   fi
 
   if { [ -n "${PYTHON-}" ] || [ "${DL-}" = "1" ]; } && [ "${MINIMAL_INSTALL-}" != 1 ]; then

diff --git a/ci/env/install-llvm-binaries.sh b/ci/env/install-llvm-binaries.sh
@@ -25,7 +25,14 @@ log_err() {
 
 trap '[ $? -eq 0 ] || log_err' EXIT
 
-LLVM_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz"
+FILE_NAME="clang+llvm-12.0.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz"
+
+if [ "$HOSTTYPE" = "aarch64" ]; then
+  FILE_NAME="clang+llvm-12.0.1-aarch64-linux-gnu.tar.xz "
+fi
+
+LLVM_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-12.0.1/${FILE_NAME}"
+
 TARGET_DIR="/opt/llvm"
 
 install_llvm() {
@@ -67,7 +74,7 @@ install_llvm() {
         WGET_OPTIONS="-nv"
       fi
 
-      wget ${WGET_OPTIONS} -c $url -O llvm.tar.xz
+      wget ${WGET_OPTIONS} -c "$url" -O llvm.tar.xz
 
       printInfo "Installing LLVM to ${targetdir}"
       mkdir -p "${targetdir}"

@@ -3,7 +3,7 @@
 Installing Ray
 ==============
 
-Ray currently officially supports x86_64 and Apple silicon (M1) hardware.
+Ray currently officially supports x86_64, aarch64 (ARM) for Linux, and Apple silicon (M1) hardware.
 Ray on Windows is currently in beta.
 
 Official Releases
@@ -49,15 +49,16 @@ You can install the nightly Ray wheels via the following links. These daily rele
   # pip install -U LINK_TO_WHEEL.whl
 
 
-====================  ====================  =======================
-       Linux                 MacOS          Windows (beta)
-====================  ====================  =======================
-`Linux Python 3.10`_  `MacOS Python 3.10`_  `Windows Python 3.10`_
-`Linux Python 3.9`_   `MacOS Python 3.9`_   `Windows Python 3.9`_
-`Linux Python 3.8`_   `MacOS Python 3.8`_   `Windows Python 3.8`_
-`Linux Python 3.7`_   `MacOS Python 3.7`_   `Windows Python 3.7`_
-`Linux Python 3.6`_   `MacOS Python 3.6`_
-====================  ====================  =======================
+=============================================== ================================================  ====================  =======================
+       Linux (x86_64)                                   Linux (arm64/aarch64)                      MacOS                 Windows (beta)
+=============================================== ================================================  ====================  =======================
+`Linux Python 3.10 (x86_64)`_                    `Linux Python 3.10 (aarch64)`_                   `MacOS Python 3.10`_  `Windows Python 3.10`_
+`Linux Python 3.9 (x86_64)`_                     `Linux Python 3.9 (aarch64)`_                    `MacOS Python 3.9`_   `Windows Python 3.9`_
+`Linux Python 3.8 (x86_64)`_                     `Linux Python 3.8 (aarch64)`_                    `MacOS Python 3.8`_   `Windows Python 3.8`_
+`Linux Python 3.7 (x86_64)`_                     `Linux Python 3.7 (aarch64)`_                    `MacOS Python 3.7`_   `Windows Python 3.7`_
+`Linux Python 3.6 (x86_64)`_                     `Linux Python 3.6 (aarch64)`_                    `MacOS Python 3.6`_
+`Linux Python 3.11 (x86_64) (EXPERIMENTAL)`_     `Linux Python 3.11 (aarch64) (EXPERIMENTAL)`_
+=============================================== ================================================  ====================  =======================
 
 .. note::
 
@@ -68,11 +69,24 @@ You can install the nightly Ray wheels via the following links. These daily rele
 
   :ref:`Usage stats <ref-usage-stats>` collection is enabled by default (can be :ref:`disabled <usage-disable>`) for nightly wheels including both local clusters started via ``ray.init()`` and remote clusters via cli.
 
-.. _`Linux Python 3.10`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp310-cp310-manylinux2014_x86_64.whl
-.. _`Linux Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp39-cp39-manylinux2014_x86_64.whl
-.. _`Linux Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
-.. _`Linux Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
-.. _`Linux Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
+.. note::
+
+  Python 3.11 support is experimental.
+
+.. _`Linux Python 3.11 (x86_64) (EXPERIMENTAL)`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp311-cp311-manylinux2014_x86_64.whl
+.. _`Linux Python 3.10 (x86_64)`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp310-cp310-manylinux2014_x86_64.whl
+.. _`Linux Python 3.9 (x86_64)`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp39-cp39-manylinux2014_x86_64.whl
+.. _`Linux Python 3.8 (x86_64)`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
+.. _`Linux Python 3.7 (x86_64)`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
+.. _`Linux Python 3.6 (x86_64)`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
+
+.. _`Linux Python 3.11 (aarch64) (EXPERIMENTAL)`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp311-cp311-manylinux2014_aarch64.whl
+.. _`Linux Python 3.10 (aarch64)`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp310-cp310-manylinux2014_aarch64.whl
+.. _`Linux Python 3.9 (aarch64)`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp39-cp39-manylinux2014_aarch64.whl
+.. _`Linux Python 3.8 (aarch64)`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp38-cp38-manylinux2014_aarch64.whl
+.. _`Linux Python 3.7 (aarch64)`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp37-cp37m-manylinux2014_aarch64.whl
+.. _`Linux Python 3.6 (aarch64)`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp36-cp36m-manylinux2014_aarch64.whl
+
 
 .. _`MacOS Python 3.10`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp310-cp310-macosx_10_15_universal2.whl
 .. _`MacOS Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp39-cp39-macosx_10_15_x86_64.whl

diff --git a/python/build-wheel-manylinux2014.sh b/python/build-wheel-manylinux2014.sh
@@ -27,11 +27,12 @@ NUMPY_VERSIONS=("1.14.5"
 yum -y install unzip zip sudo
 yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel xz
 yum -y install openssl
-yum install libasan-4.8.5-44.el7.x86_64 -y
-yum install libubsan-7.3.1-5.10.el7.x86_64 -y
-yum install devtoolset-8-libasan-devel.x86_64 -y
-
 
+if [ "${HOSTTYPE-}" = "x86_64" ]; then
+  yum install "libasan-4.8.5-44.el7.${HOSTTYPE}" -y
+  yum install "libubsan-7.3.1-5.10.el7.${HOSTTYPE}" -y
+  yum install "devtoolset-8-libasan-devel.${HOSTTYPE}" -y
+fi
 
 java -version
 java_bin=$(readlink -f "$(command -v java)")

diff --git a/python/ray/_private/utils.py b/python/ray/_private/utils.py
@@ -9,6 +9,7 @@
 import logging
 import multiprocessing
 import os
+import platform
 import re
 import signal
 import subprocess
@@ -1246,6 +1247,7 @@ def get_wheel_filename(
     sys_platform: str = sys.platform,
     ray_version: str = ray.__version__,
     py_version: str = f"{sys.version_info.major}{sys.version_info.minor}",
+    architecture: Optional[str] = None,
 ) -> str:
     """Returns the filename used for the nightly Ray wheel.
 
@@ -1257,17 +1259,27 @@ def get_wheel_filename(
         py_version (str):
             The major and minor Python versions concatenated.  Examples: "36",
             "37", "38", "39"
+        architecture: Architecture, e.g. ``x86_64`` or ``aarch64``. If None, will
+            be determined by calling ``platform.processor()``.
+
     Returns:
         The wheel file name.  Examples:
             ray-3.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
     """
     assert py_version in ["36", "37", "38", "39"], py_version
 
+    architecture = architecture or platform.processor()
+
+    if architecture == "aarch64":
+        linux_os_string = "manylinux2014_aarch64"
+    else:
+        linux_os_string = "manylinux2014_x86_64"
+
     os_strings = {
         "darwin": "macosx_10_15_x86_64"
         if py_version in ["38", "39"]
         else "macosx_10_15_intel",
-        "linux": "manylinux2014_x86_64",
+        "linux": linux_os_string,
         "win32": "win_amd64",
     }