Add benchmarks to CI and make nightlies more comprehensive (#257)

Summary:
Pull Request resolved: #257

Adds a benchmarks sanity check to the our CI to make sure they do not break. It also updates our nightly tests to correspond with our pull request tests such that we don't have to worry about creating broken tarballs.

Test Plan: Imported from OSS

Reviewed By: d4l3k, anirbanr-fb-r2p

Differential Revision: D41010380

Pulled By: PaliC

fbshipit-source-id: 98e3b938b252923bea1dbf9010e071cbf21d2838

Author: PaliC

.github/workflows/runtime_nightly.yaml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         python3-minor-version: [7,8,9,10]
-        platform: [ubuntu-18.04]
+        platform: [linux.4xlarge.nvidia.gpu]
       fail-fast: false
     runs-on: ${{ matrix.platform }}
     steps:
@@ -24,18 +24,40 @@ jobs:
         with:
           github-secret: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Clean up previous CUDA driver installations
+        shell: bash
+        run: |
+          set -x
+          yum list installed | grep nvidia || true
+          yum list installed | grep cuda || true
+          sudo yum remove -y cuda || true
+          sudo yum remove -y cuda-drivers || true
+          sudo yum remove -y "*nvidia*" || true
+      - name: Install nvidia driver, nvidia-docker runtime, set GPU_FLAG
+        run: |
+          bash .github/scripts/install_nvidia_utils_linux.sh || true
+          echo "GPU_FLAG=--gpus all" >> "${GITHUB_ENV}"
+
       - name: Build
         env:
           DOCKER_BUILDKIT: 1
-        run: docker build -t multipy --progress=plain --build-arg PYTHON_3_MINOR_VERSION=${{ matrix.python3-minor-version }} .
+        run: nvidia-docker build -t multipy --progress=plain --build-arg PYTHON_3_MINOR_VERSION=${{ matrix.python3-minor-version }} --build-arg BUILD_CUDA_TESTS=1 .
 
       - name: Test
         run: |
           docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && multipy/runtime/build/test_deploy"
+          nvidia-docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && multipy/runtime/build/test_deploy_gpu"
+
+
+      - name: Examples
+        run: |
+          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && examples/build/hello_world_example"
+          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && python3 examples/quickstart/gen_package.py && ./examples/build/quickstart my_package.pt"
+          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && ./examples/build/movable_example"
 
-      - name: Compat Tests
+      - name: Benchmark
         run: |
-          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -gt 7 ]]; then pip install -r compat-requirements.txt && multipy/runtime/build/interactive_embedded_interpreter --pyscript multipy/runtime/test_compat.py; fi"
+          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && ./multipy/runtime/build/deploy_benchmark 2 none jit multipy/runtime/example/generated/resnet"
 
       - name: Set Python Version
         run: |

.github/workflows/runtime_tests.yaml

@@ -44,6 +44,10 @@ jobs:
           docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && python3 examples/quickstart/gen_package.py && ./examples/build/quickstart my_package.pt"
           docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && ./examples/build/movable_example"
 
+      - name: Benchmark
+        run: |
+          docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -lt 8 ]]; then source ~/venvs/multipy/bin/activate; fi && ./multipy/runtime/build/deploy_benchmark 2 none jit multipy/runtime/example/generated/resnet"
+
       - name: Compat Tests
         run: |
           docker run --rm multipy bash -c "if [[ ${{ matrix.python3-minor-version }} -gt 7 ]]; then pip install -r compat-requirements.txt && multipy/runtime/build/interactive_embedded_interpreter --pyscript multipy/runtime/test_compat.py; fi"