From 05767590b8fbdf4fcaff3ed6d8e837963aef9d9d Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Wed, 8 Mar 2023 13:12:02 -0500 Subject: [PATCH 01/12] Use checkout/upload-artifact v3 actions with up-to-date Node --- .github/workflows/build-cachelib-centos-8-1.yml | 4 ++-- .github/workflows/build-cachelib-centos-8-5.yml | 4 ++-- .github/workflows/build-cachelib-debian-10.yml | 4 ++-- .github/workflows/build-cachelib-fedora-36.yml | 4 ++-- .github/workflows/build-cachelib-rockylinux-8.yml | 4 ++-- .github/workflows/build-cachelib-rockylinux-9.yml | 4 ++-- .github/workflows/build-cachelib-ubuntu-18.yml | 3 ++- .github/workflows/build-cachelib-ubuntu-20.yml | 4 ++-- .github/workflows/build-cachelib-ubuntu-22.yml | 4 ++-- .github/workflows/clang-format-check.yml | 2 +- 10 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-cachelib-centos-8-1.yml b/.github/workflows/build-cachelib-centos-8-1.yml index 3983e0c78b..19b642f0bf 100644 --- a/.github/workflows/build-cachelib-centos-8-1.yml +++ b/.github/workflows/build-cachelib-centos-8-1.yml @@ -73,7 +73,7 @@ jobs: echo === gcc -v === gcc -v - name: "checkout sources" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "Install Prerequisites" run: ./contrib/build.sh -S -B - name: "Test: update-submodules" @@ -101,7 +101,7 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs diff --git a/.github/workflows/build-cachelib-centos-8-5.yml b/.github/workflows/build-cachelib-centos-8-5.yml index 4e6c2d12e1..14ab8cfa74 100644 --- a/.github/workflows/build-cachelib-centos-8-5.yml +++ b/.github/workflows/build-cachelib-centos-8-5.yml @@ -72,7 +72,7 @@ jobs: echo === gcc -v === gcc -v - name: "checkout sources" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "Install Prerequisites" run: ./contrib/build.sh -S -B - name: "Test: update-submodules" @@ -100,7 +100,7 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs diff --git a/.github/workflows/build-cachelib-debian-10.yml b/.github/workflows/build-cachelib-debian-10.yml index 7f0ab29a6c..484833fa39 100644 --- a/.github/workflows/build-cachelib-debian-10.yml +++ b/.github/workflows/build-cachelib-debian-10.yml @@ -51,7 +51,7 @@ jobs: echo === g++ -v === g++ - || true - name: "checkout sources" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "Add Git safe directory" # Workaround for Docker image bug (GitHub issue #199). run: git config --system --add safe.directory $GITHUB_WORKSPACE @@ -82,7 +82,7 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs diff --git a/.github/workflows/build-cachelib-fedora-36.yml b/.github/workflows/build-cachelib-fedora-36.yml index f8c0424400..b9f4372ac6 100644 --- a/.github/workflows/build-cachelib-fedora-36.yml +++ b/.github/workflows/build-cachelib-fedora-36.yml @@ -54,7 +54,7 @@ jobs: echo === gcc -v === gcc -v - name: "checkout sources" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "git: ignore unsafe-directories" run: git config --global --add safe.directory '*' - name: "Install Prerequisites" @@ -84,7 +84,7 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs diff --git a/.github/workflows/build-cachelib-rockylinux-8.yml b/.github/workflows/build-cachelib-rockylinux-8.yml index c8af12327d..17d60883d1 100644 --- a/.github/workflows/build-cachelib-rockylinux-8.yml +++ b/.github/workflows/build-cachelib-rockylinux-8.yml @@ -56,7 +56,7 @@ jobs: echo === gcc -v === gcc -v - name: "checkout sources" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "Install Prerequisites" run: ./contrib/build.sh -S -B - name: "Test: update-submodules" @@ -84,7 +84,7 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs diff --git a/.github/workflows/build-cachelib-rockylinux-9.yml b/.github/workflows/build-cachelib-rockylinux-9.yml index e26eac6ff1..23c0218eca 100644 --- a/.github/workflows/build-cachelib-rockylinux-9.yml +++ b/.github/workflows/build-cachelib-rockylinux-9.yml @@ -54,7 +54,7 @@ jobs: echo === gcc -v === gcc -v - name: "checkout sources" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "Install Prerequisites" run: ./contrib/build.sh -S -B - name: "Test: update-submodules" @@ -85,7 +85,7 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs diff --git a/.github/workflows/build-cachelib-ubuntu-18.yml b/.github/workflows/build-cachelib-ubuntu-18.yml index ad068278a4..da4801b368 100644 --- a/.github/workflows/build-cachelib-ubuntu-18.yml +++ b/.github/workflows/build-cachelib-ubuntu-18.yml @@ -72,6 +72,7 @@ jobs: - name: "git: ignore unsafe-directories" run: git config --global --add safe.directory '*' - name: "checkout sources" + # v1 needed for Ubuntu 18 due to old Git version uses: actions/checkout@v1 - name: "Install Prerequisites" run: ./contrib/build.sh -S -B @@ -100,7 +101,7 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -t cachelib - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs diff --git a/.github/workflows/build-cachelib-ubuntu-20.yml b/.github/workflows/build-cachelib-ubuntu-20.yml index a8380fdb96..a2623ad8f5 100644 --- a/.github/workflows/build-cachelib-ubuntu-20.yml +++ b/.github/workflows/build-cachelib-ubuntu-20.yml @@ -52,7 +52,7 @@ jobs: echo === g++ -v === g++ -v - name: "checkout sources" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "Install Prerequisites" run: ./contrib/build.sh -S -B - name: "Test: update-submodules" @@ -80,7 +80,7 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -t cachelib - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs diff --git a/.github/workflows/build-cachelib-ubuntu-22.yml b/.github/workflows/build-cachelib-ubuntu-22.yml index 4db194431d..76f197b81e 100644 --- a/.github/workflows/build-cachelib-ubuntu-22.yml +++ b/.github/workflows/build-cachelib-ubuntu-22.yml @@ -52,7 +52,7 @@ jobs: echo === g++ -v === g++ -v - name: "checkout sources" - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: "Install Prerequisites" run: ./contrib/build.sh -S -B - name: "Test: update-submodules" @@ -80,7 +80,7 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v3 if: failure() with: name: cachelib-cmake-logs diff --git a/.github/workflows/clang-format-check.yml b/.github/workflows/clang-format-check.yml index 4b4897b610..26d942d182 100644 --- a/.github/workflows/clang-format-check.yml +++ b/.github/workflows/clang-format-check.yml @@ -11,7 +11,7 @@ jobs: - 'cachelib' - 'examples' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Run clang-format style check for C/C++ programs. uses: jidicula/clang-format-action@v4.6.2 with: From edffda45490047348cc12aa50cb769a5fbbbb405 Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Wed, 8 Mar 2023 13:32:48 -0500 Subject: [PATCH 02/12] Cache built submodules to speed up builds --- .github/workflows/build-cachelib-centos-8-1.yml | 14 ++++++++++++++ .github/workflows/build-cachelib-centos-8-5.yml | 14 ++++++++++++++ .github/workflows/build-cachelib-debian-10.yml | 14 ++++++++++++++ .github/workflows/build-cachelib-fedora-36.yml | 14 ++++++++++++++ .github/workflows/build-cachelib-rockylinux-8.yml | 14 ++++++++++++++ .github/workflows/build-cachelib-rockylinux-9.yml | 14 ++++++++++++++ .github/workflows/build-cachelib-ubuntu-18.yml | 14 ++++++++++++++ .github/workflows/build-cachelib-ubuntu-20.yml | 14 ++++++++++++++ .github/workflows/build-cachelib-ubuntu-22.yml | 14 ++++++++++++++ 9 files changed, 126 insertions(+) diff --git a/.github/workflows/build-cachelib-centos-8-1.yml b/.github/workflows/build-cachelib-centos-8-1.yml index 19b642f0bf..e284dbfa24 100644 --- a/.github/workflows/build-cachelib-centos-8-1.yml +++ b/.github/workflows/build-cachelib-centos-8-1.yml @@ -78,6 +78,16 @@ jobs: run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -92,12 +102,16 @@ jobs: run: ./contrib/build-package.sh -j -v -i fmt - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib diff --git a/.github/workflows/build-cachelib-centos-8-5.yml b/.github/workflows/build-cachelib-centos-8-5.yml index 14ab8cfa74..735bc69dd5 100644 --- a/.github/workflows/build-cachelib-centos-8-5.yml +++ b/.github/workflows/build-cachelib-centos-8-5.yml @@ -77,6 +77,16 @@ jobs: run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -91,12 +101,16 @@ jobs: run: ./contrib/build-package.sh -j -v -i fmt - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib diff --git a/.github/workflows/build-cachelib-debian-10.yml b/.github/workflows/build-cachelib-debian-10.yml index 484833fa39..77c9c1ed95 100644 --- a/.github/workflows/build-cachelib-debian-10.yml +++ b/.github/workflows/build-cachelib-debian-10.yml @@ -59,6 +59,16 @@ jobs: run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -73,12 +83,16 @@ jobs: run: ./contrib/build-package.sh -j -v -i fmt - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib diff --git a/.github/workflows/build-cachelib-fedora-36.yml b/.github/workflows/build-cachelib-fedora-36.yml index b9f4372ac6..097c53f2ba 100644 --- a/.github/workflows/build-cachelib-fedora-36.yml +++ b/.github/workflows/build-cachelib-fedora-36.yml @@ -61,6 +61,16 @@ jobs: run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -75,12 +85,16 @@ jobs: run: ./contrib/build-package.sh -j -v -i fmt - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib diff --git a/.github/workflows/build-cachelib-rockylinux-8.yml b/.github/workflows/build-cachelib-rockylinux-8.yml index 17d60883d1..6fe7eca185 100644 --- a/.github/workflows/build-cachelib-rockylinux-8.yml +++ b/.github/workflows/build-cachelib-rockylinux-8.yml @@ -61,6 +61,16 @@ jobs: run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -75,12 +85,16 @@ jobs: run: ./contrib/build-package.sh -j -v -i fmt - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib diff --git a/.github/workflows/build-cachelib-rockylinux-9.yml b/.github/workflows/build-cachelib-rockylinux-9.yml index 23c0218eca..c5dbefb99c 100644 --- a/.github/workflows/build-cachelib-rockylinux-9.yml +++ b/.github/workflows/build-cachelib-rockylinux-9.yml @@ -59,6 +59,16 @@ jobs: run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -76,12 +86,16 @@ jobs: # see: https://aur.archlinux.org/packages/folly#comment-862543 sed -i '$aset(LIBDWARF_INCLUDE_DIRS "/usr/include/libdwarf-0")' cachelib/external/folly/CMake/FindLibDwarf.cmake ./contrib/build-package.sh -S -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib diff --git a/.github/workflows/build-cachelib-ubuntu-18.yml b/.github/workflows/build-cachelib-ubuntu-18.yml index da4801b368..563b629ff7 100644 --- a/.github/workflows/build-cachelib-ubuntu-18.yml +++ b/.github/workflows/build-cachelib-ubuntu-18.yml @@ -78,6 +78,16 @@ jobs: run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -92,12 +102,16 @@ jobs: run: ./contrib/build-package.sh -j -v -i fmt - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -t cachelib diff --git a/.github/workflows/build-cachelib-ubuntu-20.yml b/.github/workflows/build-cachelib-ubuntu-20.yml index a2623ad8f5..e38b849ffb 100644 --- a/.github/workflows/build-cachelib-ubuntu-20.yml +++ b/.github/workflows/build-cachelib-ubuntu-20.yml @@ -57,6 +57,16 @@ jobs: run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -71,12 +81,16 @@ jobs: run: ./contrib/build-package.sh -j -v -i fmt - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -t cachelib diff --git a/.github/workflows/build-cachelib-ubuntu-22.yml b/.github/workflows/build-cachelib-ubuntu-22.yml index 76f197b81e..0b025407b9 100644 --- a/.github/workflows/build-cachelib-ubuntu-22.yml +++ b/.github/workflows/build-cachelib-ubuntu-22.yml @@ -57,6 +57,16 @@ jobs: run: ./contrib/build.sh -S -B - name: "Test: update-submodules" run: ./contrib/update-submodules.sh + - name: "Use cache for built libraries of submodules" + uses: actions/cache@v3 + id: cache-submodules + with: + key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }} + path: | + opt/cachelib/lib + opt/cachelib/include + opt/cachelib/bin/thrift1 + opt/cachelib/bin/fizz* - name: "Install dependency: zstd" run: ./contrib/build-package.sh -j -v -i zstd - name: "Install dependency: googleflags" @@ -71,12 +81,16 @@ jobs: run: ./contrib/build-package.sh -j -v -i fmt - name: "Install dependency: folly" run: ./contrib/build-package.sh -j -v -i folly + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fizz" run: ./contrib/build-package.sh -j -v -i fizz + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: wangle" run: ./contrib/build-package.sh -j -v -i wangle + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "Install dependency: fbthrift" run: ./contrib/build-package.sh -j -v -i fbthrift + if: steps.cache-submodules.outputs.cache-hit != 'true' - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib From 096d39128fe88ba9e57afce3e62467487bd37923 Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Wed, 8 Mar 2023 19:17:10 +0000 Subject: [PATCH 03/12] Run tests after build --- .github/run_tests.sh | 211 ++++++++++++++++++ .../workflows/build-cachelib-centos-8-1.yml | 8 + .../workflows/build-cachelib-centos-8-5.yml | 8 + .../workflows/build-cachelib-debian-10.yml | 8 + .../workflows/build-cachelib-fedora-36.yml | 8 + .../workflows/build-cachelib-rockylinux-8.yml | 8 + .../workflows/build-cachelib-rockylinux-9.yml | 8 + .../workflows/build-cachelib-ubuntu-18.yml | 8 + .../workflows/build-cachelib-ubuntu-20.yml | 8 + .../workflows/build-cachelib-ubuntu-22.yml | 8 + cachelib/cmake/Makefile.tests.in | 3 + 11 files changed, 286 insertions(+) create mode 100755 .github/run_tests.sh diff --git a/.github/run_tests.sh b/.github/run_tests.sh new file mode 100755 index 0000000000..e6b49adf3e --- /dev/null +++ b/.github/run_tests.sh @@ -0,0 +1,211 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ -z "$GITHUB_STEP_SUMMARY" ]]; then + echo "run_tests.sh is for use by CI (selected tests, timeout)." + echo "Users should go to opt/cachelib/tests and run make." + echo +fi + +# Optional (e.g., flaky) tests. Issues a warning instead of an error. +OPTIONAL=() +OPTIONAL+=("allocator-test-AllocationClassTest") # Ubuntu 18 (segfault) +OPTIONAL+=("allocator-test-AllocatorResizeTypeTest") # Rocky 8.6 +OPTIONAL+=("allocator-test-AllocatorTypeTest") # CentOS 8.5, Rocky 8.6 +OPTIONAL+=("allocator-test-MemoryAllocatorTest") # Ubuntu 18 (segfault) +OPTIONAL+=("allocator-test-MM2QTest") # Ubuntu 18 (segfault) +OPTIONAL+=("allocator-test-BlockCacheTest") # Rocky 8.6 +# CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18 +OPTIONAL+=("allocator-test-NavySetupTest") +# CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18 +OPTIONAL+=("common-test-UtilTests") +# CentOS 8.1, Rocky 9, Ubuntu 18 +OPTIONAL+=("navy-test-BlockCacheTest") +# CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Ubuntu 18, Ubuntu 20, Ubuntu 22 +OPTIONAL+=("navy-test-DriverTest") +# CentOS 8.5, Rocky 9, Ubuntu 20 +OPTIONAL+=("navy-test-MockJobSchedulerTest") +# CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18, Ubuntu 20, Ubuntu 22 +# Large pages need to be enabled +OPTIONAL+=("shm-test-test_page_size") + +# Skip long-running benchmarks. +TO_SKIP=() +# TO_SKIP+=("allocator-test-AllocatorTypeTest") # 12 mins. +TO_SKIP+=("benchmark-test-CompactCacheBench") # 26 mins. +TO_SKIP+=("benchmark-test-MutexBench") # 60 mins. + +TEST_TIMEOUT=20m +BENCHMARK_TIMEOUT=20m +PARALLELISM=10 + +OPTIONAL_LIST=$(printf -- '%s\n' ${OPTIONAL[@]}) +TO_SKIP_LIST=$(printf -- '%s\n' ${TO_SKIP[@]}) + +MD_OUT=${GITHUB_STEP_SUMMARY:-$PWD/summary.md} +if [[ "$MD_OUT" != "$GITHUB_STEP_SUMMARY" ]]; then + echo "Markdown summary will be saved in $MD_OUT. Truncating it." + echo + echo "Time started: $(date)" > $MD_OUT +fi + +dir=$(dirname "$0") +cd "$dir/.." || die "failed to change-dir into $dir/.." +test -d cachelib || die "failed to change-dir to expected root directory" + +PREFIX="$PWD/opt/cachelib" +LD_LIBRARY_PATH="$PREFIX/lib:${LD_LIBRARY_PATH:-}" +export LD_LIBRARY_PATH + +echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" + +cd opt/cachelib/tests || die "failed to change-dir into opt/cachelib/tests" + +TESTS_TO_RUN=$(find * -type f -not -name "*bench*" -executable \ + | grep -vF "$TO_SKIP_LIST" \ + | awk ' { print $1 ".log" } ') +N_TESTS=$(echo $TESTS_TO_RUN | wc -w) + +echo +echo "::group::Running tests for CI (total: $N_TESTS, max: $TEST_TIMEOUT)" +timeout --preserve-status $TEST_TIMEOUT make -j $PARALLELISM -s $TESTS_TO_RUN +echo "::endgroup::" +echo "Successful tests: $(find -name '*.ok' | wc -l)" +echo "Failed tests: $(find -name '*.fail' | wc -l)" +echo + +BENCHMARKS_TO_RUN=$(find * -type f -name "*bench*" -executable \ + | grep -vF "$TO_SKIP_LIST" \ + | awk ' { print $1 ".log" } ') +N_BENCHMARKS=$(echo $BENCHMARKS_TO_RUN | wc -w) + +echo "::group::Running benchmarks for CI (total: $N_BENCHMARKS, max: $BENCHMARK_TIMEOUT)" +timeout --preserve-status $BENCHMARK_TIMEOUT make -j $PARALLELISM -s $BENCHMARKS_TO_RUN +echo "::endgroup::" +echo "Successful benchmarks: $(find -name '*bench*.ok' | wc -l)" +echo "Failed benchmarks: $(find -name '*bench*.fail' | wc -l)" + +TESTS_PASSED=$(find * -name '*.log.ok' | sed 's/\.log\.ok$//') +TESTS_FAILED=$(find * -name '*.log.fail' | sed 's/\.log\.fail$//') +TESTS_TIMEOUT=$(find * -type f -executable \ + | grep -vF "$TESTS_PASSED" \ + | grep -vF "$TESTS_FAILED" \ + | grep -vF "$TO_SKIP_LIST") +TESTS_IGNORED=$(echo "$TESTS_FAILED" | grep -F "$OPTIONAL_LIST") +FAILURES_UNIGNORED=$(echo "$TESTS_FAILED" | grep -vF "$OPTIONAL_LIST") + +N_TIMEOUT=$(echo $TESTS_TIMEOUT | wc -w) +N_PASSED=$(echo $TESTS_PASSED | wc -w) +N_FAILED=$(echo $TESTS_FAILED | wc -w) +N_IGNORED=$(echo $TESTS_IGNORED | wc -w) +N_FAILURES_UNIGNORED=$(echo $FAILURES_UNIGNORED | wc -w) +N_SKIPPED=$(echo $TO_SKIP_LIST | wc -w) + +echo "## Test summary" >> $MD_OUT +echo "|Workflow| Passed | Failed | Ignored | Timeout | Skipped" >> $MD_OUT +echo "|--------|--------|--------|---------|---------|---------|" >> $MD_OUT +echo "| $GITHUB_JOB | $N_PASSED | $N_FAILED | $N_IGNORED | $N_TIMEOUT | $N_SKIPPED |" >> $MD_OUT + +STATUS=0 + +if [[ $N_FAILED -ne 0 ]]; then + if [[ $N_IGNORED -ne 0 ]]; then + echo + echo "::group::Ignored test failures " + echo "$TESTS_IGNORED" + echo "::endgroup::" + echo "::warning ::$N_IGNORED tests/benchmarks failed and ignored." + + echo >> $MD_OUT + echo "## Ignored test failures" >> $MD_OUT + echo "$TESTS_IGNORED" | awk ' { print "1. " $1 } ' >> $MD_OUT + fi + + if [ $N_FAILURES_UNIGNORED -eq 0 ]; then + echo "Only ignored tests failed." + else + STATUS=1 + echo + echo "== Failing tests ==" + echo "$FAILURES_UNIGNORED" + + echo >> $MD_OUT + echo "## Failing tests" >> $MD_OUT + echo "$FAILURES_UNIGNORED" | awk ' { print "1. " $1 } ' >> $MD_OUT + + echo "::error ::$N_FAILURES_UNIGNORED tests/benchmarks failed." + fi + + echo + echo "::group::Failed tests" + grep "Segmentation fault" *.log || true + grep "FAILED.*ms" *.log || true + echo "::endgroup::" + + echo >> $MD_OUT + echo "## Failures summary" >> $MD_OUT + echo "\`\`\`" >> $MD_OUT + grep "Segmentation fault" *.log >> $MD_OUT || true + grep "FAILED.*ms" *.log >> $MD_OUT || true + echo "\`\`\`" >> $MD_OUT + + echo + echo "=== Failure logs with context ===" + echo >> $MD_OUT + echo "### Failure logs with context" >> $MD_OUT + for faillog in *.log.fail; do + logfile="${faillog/\.fail/}" + echo + echo "::group::Logs:$logfile" + grep -Pazo "(?s)\[ RUN[^\[]+\[ FAILED[^\n]+ms\)\n" $logfile + grep "Segmentation fault" -B 3 $logfile + echo "::endgroup::" + + echo "#### $logfile" >> $MD_OUT + echo "\`\`\`" >> $MD_OUT + grep -Pazo "(?s)\[ RUN[^\[]+\[ FAILED[^\n]+ms\)\n" $logfile \ + | sed 's/\x0/---------------\n/g' >> $MD_OUT + grep "Segmentation fault" -B 3 $logfile >> $MD_OUT + echo "\`\`\`" >> $MD_OUT + echo >> $MD_OUT + done +else + echo + echo "All tests passed." +fi + +echo +echo "::group::Skipped tests" +echo "$TO_SKIP_LIST" +echo "::endgroup::" + +echo >> $MD_OUT +echo "## Skipped tests" >> $MD_OUT +echo "$TO_SKIP_LIST" | awk ' { print "1. " $1 } ' >> $MD_OUT + +if [[ $N_TIMEOUT -ne 0 ]]; then + echo + echo "::error ::$N_TIMEOUT tests exceeded time limit." \ + " Consider adding them to TO_SKIP or increasing TEST_TIMEOUT/BENCHMARK_TIMEOUT." + echo "::group::Timed out tests" + echo "$TESTS_TIMEOUT" + echo "::endgroup::" + + echo "## Tests timed out" >> $MD_OUT + echo "$TESTS_TIMEOUT" | awk ' { print "1. " $1 } ' >> $MD_OUT +fi + +# Comment out if you do not want (unignored) failing tests to fail the build +exit $STATUS diff --git a/.github/workflows/build-cachelib-centos-8-1.yml b/.github/workflows/build-cachelib-centos-8-1.yml index e284dbfa24..8dbe5017e9 100644 --- a/.github/workflows/build-cachelib-centos-8-1.yml +++ b/.github/workflows/build-cachelib-centos-8-1.yml @@ -115,6 +115,8 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib + - name: "Run tests" + run: ./.github/run_tests.sh - uses: actions/upload-artifact@v3 if: failure() with: @@ -126,3 +128,9 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/build-cachelib-centos-8-5.yml b/.github/workflows/build-cachelib-centos-8-5.yml index 735bc69dd5..eb11d68102 100644 --- a/.github/workflows/build-cachelib-centos-8-5.yml +++ b/.github/workflows/build-cachelib-centos-8-5.yml @@ -114,6 +114,8 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib + - name: "Run tests" + run: ./.github/run_tests.sh - uses: actions/upload-artifact@v3 if: failure() with: @@ -125,3 +127,9 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/build-cachelib-debian-10.yml b/.github/workflows/build-cachelib-debian-10.yml index 77c9c1ed95..0281e0fe8c 100644 --- a/.github/workflows/build-cachelib-debian-10.yml +++ b/.github/workflows/build-cachelib-debian-10.yml @@ -96,6 +96,8 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib + - name: "Run tests" + run: ./.github/run_tests.sh - uses: actions/upload-artifact@v3 if: failure() with: @@ -107,3 +109,9 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/build-cachelib-fedora-36.yml b/.github/workflows/build-cachelib-fedora-36.yml index 097c53f2ba..c6228f8430 100644 --- a/.github/workflows/build-cachelib-fedora-36.yml +++ b/.github/workflows/build-cachelib-fedora-36.yml @@ -98,6 +98,8 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib + - name: "Run tests" + run: ./.github/run_tests.sh - uses: actions/upload-artifact@v3 if: failure() with: @@ -109,3 +111,9 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/build-cachelib-rockylinux-8.yml b/.github/workflows/build-cachelib-rockylinux-8.yml index 6fe7eca185..605c3de122 100644 --- a/.github/workflows/build-cachelib-rockylinux-8.yml +++ b/.github/workflows/build-cachelib-rockylinux-8.yml @@ -98,6 +98,8 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib + - name: "Run tests" + run: ./.github/run_tests.sh - uses: actions/upload-artifact@v3 if: failure() with: @@ -109,3 +111,9 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/build-cachelib-rockylinux-9.yml b/.github/workflows/build-cachelib-rockylinux-9.yml index c5dbefb99c..f097041574 100644 --- a/.github/workflows/build-cachelib-rockylinux-9.yml +++ b/.github/workflows/build-cachelib-rockylinux-9.yml @@ -99,6 +99,8 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib + - name: "Run tests" + run: ./.github/run_tests.sh - uses: actions/upload-artifact@v3 if: failure() with: @@ -110,3 +112,9 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/build-cachelib-ubuntu-18.yml b/.github/workflows/build-cachelib-ubuntu-18.yml index 563b629ff7..59a402e14d 100644 --- a/.github/workflows/build-cachelib-ubuntu-18.yml +++ b/.github/workflows/build-cachelib-ubuntu-18.yml @@ -115,6 +115,8 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -t cachelib + - name: "Run tests" + run: ./.github/run_tests.sh - uses: actions/upload-artifact@v3 if: failure() with: @@ -126,3 +128,9 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/build-cachelib-ubuntu-20.yml b/.github/workflows/build-cachelib-ubuntu-20.yml index e38b849ffb..c6d29320cc 100644 --- a/.github/workflows/build-cachelib-ubuntu-20.yml +++ b/.github/workflows/build-cachelib-ubuntu-20.yml @@ -94,6 +94,8 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -t cachelib + - name: "Run tests" + run: ./.github/run_tests.sh - uses: actions/upload-artifact@v3 if: failure() with: @@ -105,3 +107,9 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/build-cachelib-ubuntu-22.yml b/.github/workflows/build-cachelib-ubuntu-22.yml index 0b025407b9..8e8756030e 100644 --- a/.github/workflows/build-cachelib-ubuntu-22.yml +++ b/.github/workflows/build-cachelib-ubuntu-22.yml @@ -94,6 +94,8 @@ jobs: - name: "build CacheLib" # Build cachelib in debug mode (-d) and with all tests (-t) run: ./contrib/build-package.sh -j -v -i -d -t cachelib + - name: "Run tests" + run: ./.github/run_tests.sh - uses: actions/upload-artifact@v3 if: failure() with: @@ -105,3 +107,9 @@ jobs: build-cachelib/**/Makefile if-no-files-found: warn retention-days: 1 + - uses: actions/upload-artifact@v3 + with: + name: cachelib-test-logs + path: opt/cachelib/tests/*.log + if-no-files-found: warn + retention-days: 7 diff --git a/cachelib/cmake/Makefile.tests.in b/cachelib/cmake/Makefile.tests.in index 4be43a2890..b0320d380c 100644 --- a/cachelib/cmake/Makefile.tests.in +++ b/cachelib/cmake/Makefile.tests.in @@ -20,6 +20,8 @@ MARKERS_FAIL=$(LOGS:%=%.fail) OUTPUT=cachelib-test-summary.log .PHONY: all clean +# Avoid deleting test logs if interrupted (e.g., timeout) +.PRECIOUS: ($LOGS) $(OUTPUT) all: $(OUTPUT) @@ -86,6 +88,7 @@ $(LOGS): %.log: % else \ touch "$@.fail" ; \ fi + @echo Completed $^ clean: @rm -f $(LOGS) $(MARKERS_OK) $(MARKERS_FAIL) $(OUTPUT) From 2dccb1897cee3abc08f035a27d0fb15c6de4e57f Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Wed, 8 Mar 2023 16:24:27 -0500 Subject: [PATCH 04/12] Shorten GitHub job name to avoid cut-off --- .github/workflows/build-cachelib-centos-8-1.yml | 2 +- .github/workflows/build-cachelib-centos-8-5.yml | 2 +- .github/workflows/build-cachelib-debian-10.yml | 2 +- .github/workflows/build-cachelib-fedora-36.yml | 2 +- .github/workflows/build-cachelib-rockylinux-8.yml | 2 +- .github/workflows/build-cachelib-rockylinux-9.yml | 2 +- .github/workflows/build-cachelib-ubuntu-18.yml | 2 +- .github/workflows/build-cachelib-ubuntu-20.yml | 2 +- .github/workflows/build-cachelib-ubuntu-22.yml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-cachelib-centos-8-1.yml b/.github/workflows/build-cachelib-centos-8-1.yml index 8dbe5017e9..a280d2a7d7 100644 --- a/.github/workflows/build-cachelib-centos-8-1.yml +++ b/.github/workflows/build-cachelib-centos-8-1.yml @@ -20,7 +20,7 @@ on: jobs: build-cachelib-centos8-1-1911: #if: "!contains(github.event.head_commit.author.name, 'svcscm')" - name: "CentOS/8.1.1911 - Build CacheLib with all dependencies" + name: "CentOS/8.1.1911 Build & Test" runs-on: ubuntu-latest # Docker container image name container: "centos:8.1.1911" diff --git a/.github/workflows/build-cachelib-centos-8-5.yml b/.github/workflows/build-cachelib-centos-8-5.yml index eb11d68102..b55e6dfedd 100644 --- a/.github/workflows/build-cachelib-centos-8-5.yml +++ b/.github/workflows/build-cachelib-centos-8-5.yml @@ -19,7 +19,7 @@ on: - cron: '0 9 * * *' jobs: build-cachelib-centos8-latest: - name: "CentOS/8.5 - Build CacheLib with all dependencies" + name: "CentOS/8.5 Build & Test" runs-on: ubuntu-latest # Docker container image name container: "centos:latest" diff --git a/.github/workflows/build-cachelib-debian-10.yml b/.github/workflows/build-cachelib-debian-10.yml index 0281e0fe8c..989cec432f 100644 --- a/.github/workflows/build-cachelib-debian-10.yml +++ b/.github/workflows/build-cachelib-debian-10.yml @@ -19,7 +19,7 @@ on: - cron: '0 13 * * *' jobs: build-cachelib-debian-10: - name: "Debian-10 (Buster) - Build CacheLib with all dependencies" + name: "Debian-10 (Buster) Build & Test" runs-on: ubuntu-latest # Docker container image name container: "debian:buster-slim" diff --git a/.github/workflows/build-cachelib-fedora-36.yml b/.github/workflows/build-cachelib-fedora-36.yml index c6228f8430..569da00df1 100644 --- a/.github/workflows/build-cachelib-fedora-36.yml +++ b/.github/workflows/build-cachelib-fedora-36.yml @@ -19,7 +19,7 @@ on: - cron: '0 19 * * *' jobs: build-cachelib-fedora-36: - name: "Fedora 36 - Build CacheLib with all dependencies" + name: "Fedora 36 Build & Test" runs-on: ubuntu-latest # Docker container image name container: "fedora:36" diff --git a/.github/workflows/build-cachelib-rockylinux-8.yml b/.github/workflows/build-cachelib-rockylinux-8.yml index 605c3de122..b4f774f65f 100644 --- a/.github/workflows/build-cachelib-rockylinux-8.yml +++ b/.github/workflows/build-cachelib-rockylinux-8.yml @@ -19,7 +19,7 @@ on: - cron: '0 15 * * 2,4,6' jobs: build-cachelib-centos8-latest: - name: "RockyLinux-8.6 - Build CacheLib with all dependencies" + name: "RockyLinux-8.6 Build & Test" runs-on: ubuntu-latest # Docker container image name container: "rockylinux:8.6" diff --git a/.github/workflows/build-cachelib-rockylinux-9.yml b/.github/workflows/build-cachelib-rockylinux-9.yml index f097041574..1e3e7b3e4a 100644 --- a/.github/workflows/build-cachelib-rockylinux-9.yml +++ b/.github/workflows/build-cachelib-rockylinux-9.yml @@ -19,7 +19,7 @@ on: - cron: '0 17 * * *' jobs: build-cachelib-rockylinux-9: - name: "RockyLinux-9.0 - Build CacheLib with all dependencies" + name: "RockyLinux-9.0 Build & Test" runs-on: ubuntu-latest # Docker container image name container: "rockylinux:9.0" diff --git a/.github/workflows/build-cachelib-ubuntu-18.yml b/.github/workflows/build-cachelib-ubuntu-18.yml index 59a402e14d..31ee90ee7c 100644 --- a/.github/workflows/build-cachelib-ubuntu-18.yml +++ b/.github/workflows/build-cachelib-ubuntu-18.yml @@ -24,7 +24,7 @@ on: - cron: '0 5 * * 2,4,6' jobs: build-cachelib-ubuntu-18: - name: "Ubuntu-18 - Build CacheLib with all dependencies" + name: "Ubuntu-18 Build & Test" runs-on: ubuntu-latest container: "ubuntu:18.04" steps: diff --git a/.github/workflows/build-cachelib-ubuntu-20.yml b/.github/workflows/build-cachelib-ubuntu-20.yml index c6d29320cc..b71bd22b50 100644 --- a/.github/workflows/build-cachelib-ubuntu-20.yml +++ b/.github/workflows/build-cachelib-ubuntu-20.yml @@ -20,7 +20,7 @@ on: - cron: '0 5 * * 1,3,5' jobs: build-cachelib-ubuntu-20: - name: "Ubuntu-20 - Build CacheLib with all dependencies" + name: "Ubuntu-20 Build & Test" runs-on: ubuntu-20.04 steps: - name: "Show Head Commit" diff --git a/.github/workflows/build-cachelib-ubuntu-22.yml b/.github/workflows/build-cachelib-ubuntu-22.yml index 8e8756030e..eb4d2f0365 100644 --- a/.github/workflows/build-cachelib-ubuntu-22.yml +++ b/.github/workflows/build-cachelib-ubuntu-22.yml @@ -20,7 +20,7 @@ on: - cron: '0 7 * * *' jobs: build-cachelib-ubuntu-22: - name: "Ubuntu-22 - Build CacheLib with all dependencies" + name: "Ubuntu-22 Build & Test" runs-on: ubuntu-22.04 steps: - name: "Show Head Commit" From 2b09eeeecfeb39a6ef5e8bd065229f0740698c07 Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Wed, 8 Mar 2023 16:30:19 -0500 Subject: [PATCH 05/12] Redirect people to summary page Going via the GitHub Actions tab brings people to the summary page first, but clicking Details in a PR will bring people to the log directly and skip the summary page. Print a message telling people to look at the summary. --- .github/run_tests.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/run_tests.sh b/.github/run_tests.sh index e6b49adf3e..253ea46007 100755 --- a/.github/run_tests.sh +++ b/.github/run_tests.sh @@ -61,6 +61,10 @@ if [[ "$MD_OUT" != "$GITHUB_STEP_SUMMARY" ]]; then echo "Time started: $(date)" > $MD_OUT fi +echo "See Summary page of job for a table of test results and log excerpts" +echo + + dir=$(dirname "$0") cd "$dir/.." || die "failed to change-dir into $dir/.." test -d cachelib || die "failed to change-dir to expected root directory" From fe4531c482fe265fea08611ba7e371a35ba6bd5a Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Wed, 8 Mar 2023 19:54:50 -0500 Subject: [PATCH 06/12] Add more flaky tests - datatype-test-MapTest (Ubuntu 20) - allocator-test-SlabAllocatorTest (Ubuntu 18) --- .github/run_tests.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/run_tests.sh b/.github/run_tests.sh index 253ea46007..0edd0e20e1 100755 --- a/.github/run_tests.sh +++ b/.github/run_tests.sh @@ -24,13 +24,15 @@ OPTIONAL=() OPTIONAL+=("allocator-test-AllocationClassTest") # Ubuntu 18 (segfault) OPTIONAL+=("allocator-test-AllocatorResizeTypeTest") # Rocky 8.6 OPTIONAL+=("allocator-test-AllocatorTypeTest") # CentOS 8.5, Rocky 8.6 +OPTIONAL+=("allocator-test-BlockCacheTest") # Rocky 8.6 OPTIONAL+=("allocator-test-MemoryAllocatorTest") # Ubuntu 18 (segfault) OPTIONAL+=("allocator-test-MM2QTest") # Ubuntu 18 (segfault) -OPTIONAL+=("allocator-test-BlockCacheTest") # Rocky 8.6 # CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18 OPTIONAL+=("allocator-test-NavySetupTest") +OPTIONAL+=("allocator-test-SlabAllocatorTest") # Ubuntu 18 # CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18 OPTIONAL+=("common-test-UtilTests") +OPTIONAL+=("datatype-test-MapTest") # Ubuntu 20 # CentOS 8.1, Rocky 9, Ubuntu 18 OPTIONAL+=("navy-test-BlockCacheTest") # CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Ubuntu 18, Ubuntu 20, Ubuntu 22 From d4aaf9cf5f1543aaeb01e75fc91b3ce8342beb83 Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Wed, 8 Mar 2023 21:26:38 -0500 Subject: [PATCH 07/12] Increase TEST_TIMEOUT for allocator-test-AllocatorTypeTest --- .github/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/run_tests.sh b/.github/run_tests.sh index 0edd0e20e1..459d4d8e59 100755 --- a/.github/run_tests.sh +++ b/.github/run_tests.sh @@ -49,7 +49,7 @@ TO_SKIP=() TO_SKIP+=("benchmark-test-CompactCacheBench") # 26 mins. TO_SKIP+=("benchmark-test-MutexBench") # 60 mins. -TEST_TIMEOUT=20m +TEST_TIMEOUT=30m BENCHMARK_TIMEOUT=20m PARALLELISM=10 From 694e993a2099761d052365294c5a534f45659bb1 Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Mon, 13 Mar 2023 18:42:07 -0400 Subject: [PATCH 08/12] Ignore NvmCacheTests, reformat test logs - Group logs with each warning/error annotation - 1 warning annotation per failing/ignored test --- .github/run_tests.sh | 100 +++++++++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 42 deletions(-) diff --git a/.github/run_tests.sh b/.github/run_tests.sh index 459d4d8e59..e25493bafa 100755 --- a/.github/run_tests.sh +++ b/.github/run_tests.sh @@ -29,6 +29,7 @@ OPTIONAL+=("allocator-test-MemoryAllocatorTest") # Ubuntu 18 (segfault) OPTIONAL+=("allocator-test-MM2QTest") # Ubuntu 18 (segfault) # CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18 OPTIONAL+=("allocator-test-NavySetupTest") +OPTIONAL+=("allocator-test-NvmCacheTests") # Rocky 8.6 OPTIONAL+=("allocator-test-SlabAllocatorTest") # Ubuntu 18 # CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18 OPTIONAL+=("common-test-UtilTests") @@ -53,6 +54,24 @@ TEST_TIMEOUT=30m BENCHMARK_TIMEOUT=20m PARALLELISM=10 +print_test_log() { + logfile=$1 + echo "::group::Logs:$logfile" + grep -Pazo "(?s)\[ RUN[^\[]+\[ FAILED[^\n]+ms\)\n" $logfile + grep "Segmentation fault" -B 3 $logfile + echo + echo "::endgroup::" + echo + + echo "#### $logfile" >> $MD_OUT + echo "\`\`\`" >> $MD_OUT + grep -Pazo "(?s)\[ RUN[^\[]+\[ FAILED[^\n]+ms\)\n" $logfile \ + | sed 's/\x0/---------------\n/g' >> $MD_OUT + grep "Segmentation fault" -B 3 $logfile >> $MD_OUT + echo "\`\`\`" >> $MD_OUT + echo >> $MD_OUT +} + OPTIONAL_LIST=$(printf -- '%s\n' ${OPTIONAL[@]}) TO_SKIP_LIST=$(printf -- '%s\n' ${TO_SKIP[@]}) @@ -63,10 +82,9 @@ if [[ "$MD_OUT" != "$GITHUB_STEP_SUMMARY" ]]; then echo "Time started: $(date)" > $MD_OUT fi -echo "See Summary page of job for a table of test results and log excerpts" +echo "See Summary page of job for a table of test results and log excerpts." echo - dir=$(dirname "$0") cd "$dir/.." || die "failed to change-dir into $dir/.." test -d cachelib || die "failed to change-dir to expected root directory" @@ -84,6 +102,13 @@ TESTS_TO_RUN=$(find * -type f -not -name "*bench*" -executable \ | awk ' { print $1 ".log" } ') N_TESTS=$(echo $TESTS_TO_RUN | wc -w) +if [[ $(< /proc/sys/vm/nr_hugepages) == "0" ]]; then + # GitHub's runners have 7GB of RAM (as of 2023) + echo + echo "Trying to allocate a 1GB huge page pool for shm-test-test_page_size" + sudo sysctl -w vm.nr_hugepages=512 +fi + echo echo "::group::Running tests for CI (total: $N_TESTS, max: $TEST_TIMEOUT)" timeout --preserve-status $TEST_TIMEOUT make -j $PARALLELISM -s $TESTS_TO_RUN @@ -127,17 +152,19 @@ echo "| $GITHUB_JOB | $N_PASSED | $N_FAILED | $N_IGNORED | $N_TIMEOUT | $N_SKIPP STATUS=0 if [[ $N_FAILED -ne 0 ]]; then - if [[ $N_IGNORED -ne 0 ]]; then - echo - echo "::group::Ignored test failures " - echo "$TESTS_IGNORED" - echo "::endgroup::" - echo "::warning ::$N_IGNORED tests/benchmarks failed and ignored." - echo >> $MD_OUT - echo "## Ignored test failures" >> $MD_OUT - echo "$TESTS_IGNORED" | awk ' { print "1. " $1 } ' >> $MD_OUT - fi + echo + echo "::group::Failures at a glance" + grep "Segmentation fault" *.log || true + grep "FAILED.*ms" *.log || true + echo "::endgroup::" + + echo >> $MD_OUT + echo "## Failures at a glance" >> $MD_OUT + echo "\`\`\`" >> $MD_OUT + grep "Segmentation fault" *.log >> $MD_OUT || true + grep "FAILED.*ms" *.log >> $MD_OUT || true + echo "\`\`\`" >> $MD_OUT if [ $N_FAILURES_UNIGNORED -eq 0 ]; then echo "Only ignored tests failed." @@ -145,48 +172,37 @@ if [[ $N_FAILED -ne 0 ]]; then STATUS=1 echo echo "== Failing tests ==" + echo "::error ::$N_FAILURES_UNIGNORED tests/benchmarks failed." + echo "$FAILURES_UNIGNORED" echo >> $MD_OUT echo "## Failing tests" >> $MD_OUT echo "$FAILURES_UNIGNORED" | awk ' { print "1. " $1 } ' >> $MD_OUT - echo "::error ::$N_FAILURES_UNIGNORED tests/benchmarks failed." - fi - - echo - echo "::group::Failed tests" - grep "Segmentation fault" *.log || true - grep "FAILED.*ms" *.log || true - echo "::endgroup::" - echo >> $MD_OUT - echo "## Failures summary" >> $MD_OUT - echo "\`\`\`" >> $MD_OUT - grep "Segmentation fault" *.log >> $MD_OUT || true - grep "FAILED.*ms" *.log >> $MD_OUT || true - echo "\`\`\`" >> $MD_OUT + for failedtest in $FAILURES_UNIGNORED; do + echo "::error ::$failedtest failed. See job summary or log for details." + print_test_log "$failedtest.log" + done + fi - echo - echo "=== Failure logs with context ===" - echo >> $MD_OUT - echo "### Failure logs with context" >> $MD_OUT - for faillog in *.log.fail; do - logfile="${faillog/\.fail/}" + if [[ $N_IGNORED -ne 0 ]]; then echo - echo "::group::Logs:$logfile" - grep -Pazo "(?s)\[ RUN[^\[]+\[ FAILED[^\n]+ms\)\n" $logfile - grep "Segmentation fault" -B 3 $logfile + echo "::group::Ignored test failures " + echo "$TESTS_IGNORED" echo "::endgroup::" + echo "::warning ::$N_IGNORED tests/benchmarks failed and ignored." - echo "#### $logfile" >> $MD_OUT - echo "\`\`\`" >> $MD_OUT - grep -Pazo "(?s)\[ RUN[^\[]+\[ FAILED[^\n]+ms\)\n" $logfile \ - | sed 's/\x0/---------------\n/g' >> $MD_OUT - grep "Segmentation fault" -B 3 $logfile >> $MD_OUT - echo "\`\`\`" >> $MD_OUT echo >> $MD_OUT - done + echo "## Ignored test failures" >> $MD_OUT + echo "$TESTS_IGNORED" | awk ' { print "1. " $1 } ' >> $MD_OUT + + for failedtest in $TESTS_IGNORED; do + echo "::warning ::$failedtest failed & ignored. See job summary or log for details." + print_test_log "$failedtest.log" + done + fi else echo echo "All tests passed." From 4da4dca236113eab2f63fc7017ca2d9ffd3d389d Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Tue, 14 Mar 2023 03:25:02 -0400 Subject: [PATCH 09/12] Attach test logs on failure too --- .github/workflows/build-cachelib-centos-8-1.yml | 1 + .github/workflows/build-cachelib-centos-8-5.yml | 1 + .github/workflows/build-cachelib-debian-10.yml | 1 + .github/workflows/build-cachelib-fedora-36.yml | 1 + .github/workflows/build-cachelib-rockylinux-8.yml | 1 + .github/workflows/build-cachelib-rockylinux-9.yml | 1 + .github/workflows/build-cachelib-ubuntu-18.yml | 1 + .github/workflows/build-cachelib-ubuntu-20.yml | 1 + .github/workflows/build-cachelib-ubuntu-22.yml | 1 + 9 files changed, 9 insertions(+) diff --git a/.github/workflows/build-cachelib-centos-8-1.yml b/.github/workflows/build-cachelib-centos-8-1.yml index a280d2a7d7..6295194b3d 100644 --- a/.github/workflows/build-cachelib-centos-8-1.yml +++ b/.github/workflows/build-cachelib-centos-8-1.yml @@ -134,3 +134,4 @@ jobs: path: opt/cachelib/tests/*.log if-no-files-found: warn retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/build-cachelib-centos-8-5.yml b/.github/workflows/build-cachelib-centos-8-5.yml index b55e6dfedd..ab848c28b7 100644 --- a/.github/workflows/build-cachelib-centos-8-5.yml +++ b/.github/workflows/build-cachelib-centos-8-5.yml @@ -133,3 +133,4 @@ jobs: path: opt/cachelib/tests/*.log if-no-files-found: warn retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/build-cachelib-debian-10.yml b/.github/workflows/build-cachelib-debian-10.yml index 989cec432f..e001deb096 100644 --- a/.github/workflows/build-cachelib-debian-10.yml +++ b/.github/workflows/build-cachelib-debian-10.yml @@ -115,3 +115,4 @@ jobs: path: opt/cachelib/tests/*.log if-no-files-found: warn retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/build-cachelib-fedora-36.yml b/.github/workflows/build-cachelib-fedora-36.yml index 569da00df1..f616217f91 100644 --- a/.github/workflows/build-cachelib-fedora-36.yml +++ b/.github/workflows/build-cachelib-fedora-36.yml @@ -117,3 +117,4 @@ jobs: path: opt/cachelib/tests/*.log if-no-files-found: warn retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/build-cachelib-rockylinux-8.yml b/.github/workflows/build-cachelib-rockylinux-8.yml index b4f774f65f..8c01c8e591 100644 --- a/.github/workflows/build-cachelib-rockylinux-8.yml +++ b/.github/workflows/build-cachelib-rockylinux-8.yml @@ -117,3 +117,4 @@ jobs: path: opt/cachelib/tests/*.log if-no-files-found: warn retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/build-cachelib-rockylinux-9.yml b/.github/workflows/build-cachelib-rockylinux-9.yml index 1e3e7b3e4a..5707be387b 100644 --- a/.github/workflows/build-cachelib-rockylinux-9.yml +++ b/.github/workflows/build-cachelib-rockylinux-9.yml @@ -118,3 +118,4 @@ jobs: path: opt/cachelib/tests/*.log if-no-files-found: warn retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/build-cachelib-ubuntu-18.yml b/.github/workflows/build-cachelib-ubuntu-18.yml index 31ee90ee7c..05263a800b 100644 --- a/.github/workflows/build-cachelib-ubuntu-18.yml +++ b/.github/workflows/build-cachelib-ubuntu-18.yml @@ -134,3 +134,4 @@ jobs: path: opt/cachelib/tests/*.log if-no-files-found: warn retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/build-cachelib-ubuntu-20.yml b/.github/workflows/build-cachelib-ubuntu-20.yml index b71bd22b50..593070c2df 100644 --- a/.github/workflows/build-cachelib-ubuntu-20.yml +++ b/.github/workflows/build-cachelib-ubuntu-20.yml @@ -113,3 +113,4 @@ jobs: path: opt/cachelib/tests/*.log if-no-files-found: warn retention-days: 7 + if: success() || failure() diff --git a/.github/workflows/build-cachelib-ubuntu-22.yml b/.github/workflows/build-cachelib-ubuntu-22.yml index eb4d2f0365..d7cf74c1db 100644 --- a/.github/workflows/build-cachelib-ubuntu-22.yml +++ b/.github/workflows/build-cachelib-ubuntu-22.yml @@ -113,3 +113,4 @@ jobs: path: opt/cachelib/tests/*.log if-no-files-found: warn retention-days: 7 + if: success() || failure() From 8ed987702d8a16420e083cad40e5529b33eeca4a Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Tue, 14 Mar 2023 03:26:11 -0400 Subject: [PATCH 10/12] Mark RefCountTest flaky (Rocky 8.6) --- .github/run_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/run_tests.sh b/.github/run_tests.sh index e25493bafa..d049c9f1a4 100755 --- a/.github/run_tests.sh +++ b/.github/run_tests.sh @@ -30,6 +30,7 @@ OPTIONAL+=("allocator-test-MM2QTest") # Ubuntu 18 (segfault) # CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18 OPTIONAL+=("allocator-test-NavySetupTest") OPTIONAL+=("allocator-test-NvmCacheTests") # Rocky 8.6 +OPTIONAL+=("allocator-test-RefCountTest") # Rocky 8.6 OPTIONAL+=("allocator-test-SlabAllocatorTest") # Ubuntu 18 # CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18 OPTIONAL+=("common-test-UtilTests") From 7c1ad3728858aa637769075a59b2c798dd7e1ecf Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Tue, 14 Mar 2023 03:28:53 -0400 Subject: [PATCH 11/12] Fix typo on RefCountTest (Ubuntu 18) --- .github/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/run_tests.sh b/.github/run_tests.sh index d049c9f1a4..0815a9823b 100755 --- a/.github/run_tests.sh +++ b/.github/run_tests.sh @@ -30,7 +30,7 @@ OPTIONAL+=("allocator-test-MM2QTest") # Ubuntu 18 (segfault) # CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18 OPTIONAL+=("allocator-test-NavySetupTest") OPTIONAL+=("allocator-test-NvmCacheTests") # Rocky 8.6 -OPTIONAL+=("allocator-test-RefCountTest") # Rocky 8.6 +OPTIONAL+=("allocator-test-RefCountTest") # Ubuntu 18 OPTIONAL+=("allocator-test-SlabAllocatorTest") # Ubuntu 18 # CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18 OPTIONAL+=("common-test-UtilTests") From acf7fc254f3e245fa95ff1bc7b7f0e62b7d68417 Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Tue, 14 Mar 2023 18:47:32 -0400 Subject: [PATCH 12/12] Capture test log before core dumps Generalize from segfaults to other things that cause core dumps --- .github/run_tests.sh | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/run_tests.sh b/.github/run_tests.sh index 0815a9823b..cfb36457a9 100755 --- a/.github/run_tests.sh +++ b/.github/run_tests.sh @@ -57,18 +57,24 @@ PARALLELISM=10 print_test_log() { logfile=$1 + # Print last failed test + EXP_LOG=$(grep -Pazo \ + "(?s)\[ RUN[^\[]+\[ FAILED[^\n]+ms\)\n" $logfile \ + | sed 's/\x0/---------------\n/g') + # And contents of last test before core dumps + if grep -q -R "core dumped" $logfile; then + EXP_LOG+=$'\n' + EXP_LOG+=$(tac $logfile | sed '/\[ RUN \]/q' | tac) + fi echo "::group::Logs:$logfile" - grep -Pazo "(?s)\[ RUN[^\[]+\[ FAILED[^\n]+ms\)\n" $logfile - grep "Segmentation fault" -B 3 $logfile + echo "$EXP_LOG" echo echo "::endgroup::" echo echo "#### $logfile" >> $MD_OUT echo "\`\`\`" >> $MD_OUT - grep -Pazo "(?s)\[ RUN[^\[]+\[ FAILED[^\n]+ms\)\n" $logfile \ - | sed 's/\x0/---------------\n/g' >> $MD_OUT - grep "Segmentation fault" -B 3 $logfile >> $MD_OUT + echo "$EXP_LOG" >> $MD_OUT echo "\`\`\`" >> $MD_OUT echo >> $MD_OUT } @@ -156,14 +162,14 @@ if [[ $N_FAILED -ne 0 ]]; then echo echo "::group::Failures at a glance" - grep "Segmentation fault" *.log || true + grep "core dumped" *.log || true grep "FAILED.*ms" *.log || true echo "::endgroup::" echo >> $MD_OUT echo "## Failures at a glance" >> $MD_OUT echo "\`\`\`" >> $MD_OUT - grep "Segmentation fault" *.log >> $MD_OUT || true + grep "core dumped" *.log >> $MD_OUT || true grep "FAILED.*ms" *.log >> $MD_OUT || true echo "\`\`\`" >> $MD_OUT