diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index e4154b66bae0..38597db239a3 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -58,11 +58,35 @@ jobs: displayName: Build JS - script: .dotnet/dotnet build ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-restore displayName: Build - - script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build --filter 'Quarantined!=true|Quarantined=false' - -p:VsTestUseMSBuildOutput=false - --logger:"trx%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" - --logger:"html%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.html" - --results-directory $(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined + - script: | + set -o pipefail + + .dotnet/dotnet test ./src/Components/test/E2ETest \ + -c $(BuildConfiguration) \ + --no-build \ + --filter 'Quarantined!=true|Quarantined=false' \ + -p:VsTestUseMSBuildOutput=false \ + --logger:"trx%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" \ + --logger:"html%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.html" \ + --results-directory $(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined \ + | tee e2e-test-output.log + + if grep -q "No test matches the given testcase filter" e2e-test-output.log + then + echo "##vso[task.logissue type=error] No tests matched the filter." + + exit 1 + fi + + # Check total tests run to detect abnormalities. In case the number of tests changes significantly, we should adjust the threshold. + # Extract total from the summary line "Failed: xx, Passed: yy, Skipped: zz, Total: NNN, Duration: ..." + total=$(sed -nE 's/.*Failed:[[:space:]]*[0-9]+,[[:space:]]*Passed:[[:space:]]*[0-9]+,[[:space:]]*Skipped:[[:space:]]*[0-9]+,[[:space:]]*Total:[[:space:]]*([0-9]+).*/\1/p' e2e-test-output.log) + min_total=1000 + if [ -z "$total" ] || [ "$total" -lt "$min_total" ] + then + echo "##vso[task.logissue type=error] Insufficient total test count: $total. We expect at least $min_total tests to run." + exit 1 + fi displayName: Run E2E tests - script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build --filter 'Quarantined=true' -p:RunQuarantinedTests=true -p:VsTestUseMSBuildOutput=false