From 96ff268249afee2f3d8b50c9f2903fa5754f17ad Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Tue, 3 Jun 2025 07:52:03 +0000 Subject: [PATCH 01/17] Change to build.sh -test. --- .azure/pipelines/components-e2e-tests.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index e4154b66bae0..2382f7130e35 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -58,11 +58,13 @@ jobs: displayName: Build JS - script: .dotnet/dotnet build ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-restore displayName: Build - - script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build --filter 'Quarantined!=true|Quarantined=false' - -p:VsTestUseMSBuildOutput=false - --logger:"trx%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" - --logger:"html%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.html" - --results-directory $(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined + - script: ./eng/build.sh -no-build -test -projects src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj \ + -c $(BuildConfiguration) \ + -p:TestCaseFilter="Quarantined!=true|Quarantined=false" \ + -p:VsTestUseMSBuildOutput=false \ + -p:VSTestLogger="trx;LogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" \ + -p:VSTestLogger="html;LogFileName=Microsoft.AspNetCore.Components.E2ETests.html" \ + -p:TestResultsDirectory=$(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined displayName: Run E2E tests - script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build --filter 'Quarantined=true' -p:RunQuarantinedTests=true -p:VsTestUseMSBuildOutput=false From e7e5d7a609933c27b01d912784ce9ae614a2ada6 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Tue, 3 Jun 2025 08:33:18 +0000 Subject: [PATCH 02/17] Syntax. --- .azure/pipelines/components-e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index 2382f7130e35..fc7813360913 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -58,7 +58,7 @@ jobs: displayName: Build JS - script: .dotnet/dotnet build ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-restore displayName: Build - - script: ./eng/build.sh -no-build -test -projects src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj \ + - script: ./eng/build.sh --no-build --test --projects src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj \ -c $(BuildConfiguration) \ -p:TestCaseFilter="Quarantined!=true|Quarantined=false" \ -p:VsTestUseMSBuildOutput=false \ From 102f8c35d96f6525b05bc6171509bedbab7ed04e Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Tue, 3 Jun 2025 08:50:42 +0000 Subject: [PATCH 03/17] Full switch name. --- .azure/pipelines/components-e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index fc7813360913..a81b996946ed 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -59,7 +59,7 @@ jobs: - script: .dotnet/dotnet build ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-restore displayName: Build - script: ./eng/build.sh --no-build --test --projects src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj \ - -c $(BuildConfiguration) \ + --configuration $(BuildConfiguration) \ -p:TestCaseFilter="Quarantined!=true|Quarantined=false" \ -p:VsTestUseMSBuildOutput=false \ -p:VSTestLogger="trx;LogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" \ From 9a901c031960d84993cb63df1e95eaa7e5814808 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Tue, 3 Jun 2025 08:51:29 +0000 Subject: [PATCH 04/17] Change order. --- .azure/pipelines/components-e2e-tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index a81b996946ed..ac3e1a81c456 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -58,8 +58,7 @@ jobs: displayName: Build JS - script: .dotnet/dotnet build ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-restore displayName: Build - - script: ./eng/build.sh --no-build --test --projects src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj \ - --configuration $(BuildConfiguration) \ + - script: ./eng/build.sh --no-build --test --configuration $(BuildConfiguration) --projects src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj \ -p:TestCaseFilter="Quarantined!=true|Quarantined=false" \ -p:VsTestUseMSBuildOutput=false \ -p:VSTestLogger="trx;LogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" \ From a5758aef0e097394893c4bff6b6d3aac7e936d00 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Tue, 3 Jun 2025 09:33:03 +0000 Subject: [PATCH 05/17] test --- .azure/pipelines/components-e2e-tests.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index ac3e1a81c456..6cc90f0e98d5 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -58,13 +58,18 @@ jobs: displayName: Build JS - script: .dotnet/dotnet build ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-restore displayName: Build - - script: ./eng/build.sh --no-build --test --configuration $(BuildConfiguration) --projects src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj \ - -p:TestCaseFilter="Quarantined!=true|Quarantined=false" \ - -p:VsTestUseMSBuildOutput=false \ - -p:VSTestLogger="trx;LogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" \ - -p:VSTestLogger="html;LogFileName=Microsoft.AspNetCore.Components.E2ETests.html" \ - -p:TestResultsDirectory=$(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined - displayName: Run E2E tests + - script: ./eng/build.sh --ci \ + --no-build \ + --test \ + --no-build-repo-tasks \ + --projects "$(Build.SourcesDirectory)/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj" \ + --configuration "$(BuildConfiguration)" \ + -p:TestCaseFilter="Quarantined!=true|Quarantined=false" \ + -p:VsTestUseMSBuildOutput=false \ + -p:VSTestLogger="trx;LogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" \ + -p:VSTestLogger="html;LogFileName=Microsoft.AspNetCore.Components.E2ETests.html" \ + -p:TestResultsDirectory="$(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined" + displayName: Run E2E tests - script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build --filter 'Quarantined=true' -p:RunQuarantinedTests=true -p:VsTestUseMSBuildOutput=false --logger:"trx%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" From bbde679416cdb8f3f292e39909184e6791d1daa2 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Tue, 3 Jun 2025 10:22:45 +0000 Subject: [PATCH 06/17] Whitespace was missing. --- .azure/pipelines/components-e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index 6cc90f0e98d5..43ec20ebfeca 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -69,7 +69,7 @@ jobs: -p:VSTestLogger="trx;LogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" \ -p:VSTestLogger="html;LogFileName=Microsoft.AspNetCore.Components.E2ETests.html" \ -p:TestResultsDirectory="$(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined" - displayName: Run E2E tests + displayName: Run E2E tests - script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build --filter 'Quarantined=true' -p:RunQuarantinedTests=true -p:VsTestUseMSBuildOutput=false --logger:"trx%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" From 8c8b7fd2468031e184763f4bc7e0a1e8a1fbfba8 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Tue, 3 Jun 2025 10:46:37 +0000 Subject: [PATCH 07/17] One dash. --- .azure/pipelines/components-e2e-tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index 43ec20ebfeca..4bdda418daf4 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -58,12 +58,12 @@ jobs: displayName: Build JS - script: .dotnet/dotnet build ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-restore displayName: Build - - script: ./eng/build.sh --ci \ - --no-build \ - --test \ - --no-build-repo-tasks \ - --projects "$(Build.SourcesDirectory)/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj" \ - --configuration "$(BuildConfiguration)" \ + - script: ./eng/build.sh -ci \ + -no-build \ + -test \ + -no-build-repo-tasks \ + -projects "$(Build.SourcesDirectory)/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj" \ + -configuration "$(BuildConfiguration)" \ -p:TestCaseFilter="Quarantined!=true|Quarantined=false" \ -p:VsTestUseMSBuildOutput=false \ -p:VSTestLogger="trx;LogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" \ From cf309f9357a29dac93f2cd33047373de48900ecd Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Tue, 3 Jun 2025 11:07:24 +0000 Subject: [PATCH 08/17] Whitespaces + no break line slashes. --- .azure/pipelines/components-e2e-tests.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index 4bdda418daf4..ab3e4886386e 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -58,17 +58,17 @@ jobs: displayName: Build JS - script: .dotnet/dotnet build ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-restore displayName: Build - - script: ./eng/build.sh -ci \ - -no-build \ - -test \ - -no-build-repo-tasks \ - -projects "$(Build.SourcesDirectory)/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj" \ - -configuration "$(BuildConfiguration)" \ - -p:TestCaseFilter="Quarantined!=true|Quarantined=false" \ - -p:VsTestUseMSBuildOutput=false \ - -p:VSTestLogger="trx;LogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" \ - -p:VSTestLogger="html;LogFileName=Microsoft.AspNetCore.Components.E2ETests.html" \ - -p:TestResultsDirectory="$(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined" + - script: ./eng/build.sh -ci + -no-build + -test + -no-build-repo-tasks + -projects "$(Build.SourcesDirectory)/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj" + -configuration "$(BuildConfiguration)" + -p:TestCaseFilter="Quarantined!=true|Quarantined=false" + -p:VsTestUseMSBuildOutput=false + -p:VSTestLogger="trx;LogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" + -p:VSTestLogger="html;LogFileName=Microsoft.AspNetCore.Components.E2ETests.html" + -p:TestResultsDirectory="$(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined" displayName: Run E2E tests - script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build --filter 'Quarantined=true' -p:RunQuarantinedTests=true -p:VsTestUseMSBuildOutput=false From 2ade1eb894e9304bbfc4448d38426346860f22b5 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Tue, 3 Jun 2025 14:21:06 +0000 Subject: [PATCH 09/17] Unify build step with test step. --- .azure/pipelines/components-e2e-tests.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index ab3e4886386e..a31e534b5c01 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -56,11 +56,14 @@ jobs: displayName: NPM install - script: npm run build displayName: Build JS - - script: .dotnet/dotnet build ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-restore - displayName: Build - script: ./eng/build.sh -ci + -projects "$(Build.SourcesDirectory)/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj" + -c $(BuildConfiguration) + -no-restore + displayName: Build + - script: ./eng/build.sh -test -no-build - -test + -ci -no-build-repo-tasks -projects "$(Build.SourcesDirectory)/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj" -configuration "$(BuildConfiguration)" From 38d22061faa9d963c4c320332ff1e253d87b4c0a Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Tue, 3 Jun 2025 15:06:33 +0000 Subject: [PATCH 10/17] Revert to dotnet test but with a binlog. --- .azure/pipelines/components-e2e-tests.yml | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index a31e534b5c01..5f39d78f201c 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -56,22 +56,13 @@ jobs: displayName: NPM install - script: npm run build displayName: Build JS - - script: ./eng/build.sh -ci - -projects "$(Build.SourcesDirectory)/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj" - -c $(BuildConfiguration) - -no-restore + - script: .dotnet/dotnet build ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-restore displayName: Build - - script: ./eng/build.sh -test - -no-build - -ci - -no-build-repo-tasks - -projects "$(Build.SourcesDirectory)/src/Components/test/E2ETest/Microsoft.AspNetCore.Components.E2ETests.csproj" - -configuration "$(BuildConfiguration)" - -p:TestCaseFilter="Quarantined!=true|Quarantined=false" + - script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build --filter 'Quarantined!=true|Quarantined=false' /bl:$(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined/e2e.binlog -p:VsTestUseMSBuildOutput=false - -p:VSTestLogger="trx;LogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" - -p:VSTestLogger="html;LogFileName=Microsoft.AspNetCore.Components.E2ETests.html" - -p:TestResultsDirectory="$(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined" + --logger:"trx%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" + --logger:"html%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.html" + --results-directory $(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined displayName: Run E2E tests - script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build --filter 'Quarantined=true' -p:RunQuarantinedTests=true -p:VsTestUseMSBuildOutput=false From 64b69e8730e294a683cfe114a25c6d9c3520cd55 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Wed, 4 Jun 2025 15:33:33 +0000 Subject: [PATCH 11/17] Make sure e2e fail if no tests were run. --- .azure/pipelines/components-e2e-tests.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index 5f39d78f201c..e784cce1f832 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -58,11 +58,7 @@ jobs: displayName: Build JS - script: .dotnet/dotnet build ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-restore displayName: Build - - script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build --filter 'Quarantined!=true|Quarantined=false' /bl:$(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined/e2e.binlog - -p:VsTestUseMSBuildOutput=false - --logger:"trx%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" - --logger:"html%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.html" - --results-directory $(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined + - script: ./eng/scripts/run-unquarantined-e2e-tests.sh $(BuildConfiguration) $(Build.SourcesDirectory) displayName: Run E2E tests - script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build --filter 'Quarantined=true' -p:RunQuarantinedTests=true -p:VsTestUseMSBuildOutput=false From c4431d3d90a442a207b05d4c809af3f0482e210a Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Wed, 4 Jun 2025 15:34:28 +0000 Subject: [PATCH 12/17] Missing script for the lastest commit. --- eng/scripts/run-unquarantined-e2e-tests.sh | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 eng/scripts/run-unquarantined-e2e-tests.sh diff --git a/eng/scripts/run-unquarantined-e2e-tests.sh b/eng/scripts/run-unquarantined-e2e-tests.sh new file mode 100644 index 000000000000..2feb6de81c81 --- /dev/null +++ b/eng/scripts/run-unquarantined-e2e-tests.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Run unquarantined Blazor E2E tests and fail if no tests match the filter +set -euo pipefail + +CONFIGURATION="$1" +SOURCES_DIR="$2" +OUTPUT_LOG="e2e-test-output.log" + +.dotnet/dotnet test ./src/Components/test/E2ETest \ + -c "${CONFIGURATION}" \ + --no-build \ + --filter 'Quarantined!=true|Quarantined=false' \ + -p:VsTestUseMSBuildOutput=false \ + --logger:"trx;LogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" \ + --logger:"html;LogFileName=Microsoft.AspNetCore.Components.E2ETests.html" \ + --results-directory "${SOURCES_DIR}/artifacts/TestResults/${CONFIGURATION}/Unquarantined" \ + | tee "${OUTPUT_LOG}" + +if grep -q "No test matches the given testcase filte" "${OUTPUT_LOG}"; then + echo "##vso[task.logissue type=error] No tests matched the filter." + exit 1 +fi From fe62c515e83274df2b7bf5f8eb6ba2dc6e99ccdb Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Wed, 4 Jun 2025 16:04:46 +0000 Subject: [PATCH 13/17] Permission denied. --- .azure/pipelines/components-e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index e784cce1f832..a6f3cfa633dc 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -58,7 +58,7 @@ jobs: displayName: Build JS - script: .dotnet/dotnet build ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-restore displayName: Build - - script: ./eng/scripts/run-unquarantined-e2e-tests.sh $(BuildConfiguration) $(Build.SourcesDirectory) + - bash: ./eng/scripts/run-unquarantined-e2e-tests.sh $(BuildConfiguration) $(Build.SourcesDirectory) displayName: Run E2E tests - script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build --filter 'Quarantined=true' -p:RunQuarantinedTests=true -p:VsTestUseMSBuildOutput=false From cd3a331b1b208fb30210b1e8e861c7739e71ce2a Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Wed, 4 Jun 2025 16:32:17 +0000 Subject: [PATCH 14/17] Embedded. --- .azure/pipelines/components-e2e-tests.yml | 20 +++++++++++++++++++- eng/scripts/run-unquarantined-e2e-tests.sh | 22 ---------------------- 2 files changed, 19 insertions(+), 23 deletions(-) delete mode 100644 eng/scripts/run-unquarantined-e2e-tests.sh diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index a6f3cfa633dc..371297f5a578 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -58,7 +58,25 @@ jobs: displayName: Build JS - script: .dotnet/dotnet build ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-restore displayName: Build - - bash: ./eng/scripts/run-unquarantined-e2e-tests.sh $(BuildConfiguration) $(Build.SourcesDirectory) + - script: | + set -o pipefail + + .dotnet/dotnet test ./src/Components/test/E2ETest \ + -c $(BuildConfiguration) \ + --no-build \ + --filter 'Quarantined!=true|Quarantined=false' \ + -p:VsTestUseMSBuildOutput=false \ + --logger:"trx%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" \ + --logger:"html%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.html" \ + --results-directory $(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined \ + | tee e2e-test-output.log + + if grep -q "No test matches the given testcase filte" e2e-test-output.log + then + echo "##vso[task.logissue type=error] No tests matched the filter string was found in output." + + exit 1 + fi displayName: Run E2E tests - script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build --filter 'Quarantined=true' -p:RunQuarantinedTests=true -p:VsTestUseMSBuildOutput=false diff --git a/eng/scripts/run-unquarantined-e2e-tests.sh b/eng/scripts/run-unquarantined-e2e-tests.sh deleted file mode 100644 index 2feb6de81c81..000000000000 --- a/eng/scripts/run-unquarantined-e2e-tests.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -# Run unquarantined Blazor E2E tests and fail if no tests match the filter -set -euo pipefail - -CONFIGURATION="$1" -SOURCES_DIR="$2" -OUTPUT_LOG="e2e-test-output.log" - -.dotnet/dotnet test ./src/Components/test/E2ETest \ - -c "${CONFIGURATION}" \ - --no-build \ - --filter 'Quarantined!=true|Quarantined=false' \ - -p:VsTestUseMSBuildOutput=false \ - --logger:"trx;LogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" \ - --logger:"html;LogFileName=Microsoft.AspNetCore.Components.E2ETests.html" \ - --results-directory "${SOURCES_DIR}/artifacts/TestResults/${CONFIGURATION}/Unquarantined" \ - | tee "${OUTPUT_LOG}" - -if grep -q "No test matches the given testcase filte" "${OUTPUT_LOG}"; then - echo "##vso[task.logissue type=error] No tests matched the filter." - exit 1 -fi From bf7003371900d2495bafdc959f2bfa9d3c6efa1e Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Thu, 5 Jun 2025 09:02:44 +0000 Subject: [PATCH 15/17] Fix typos. --- .azure/pipelines/components-e2e-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index 371297f5a578..a7c6341b643f 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -71,9 +71,9 @@ jobs: --results-directory $(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined \ | tee e2e-test-output.log - if grep -q "No test matches the given testcase filte" e2e-test-output.log + if grep -q "No test matches the given testcase filter" e2e-test-output.log then - echo "##vso[task.logissue type=error] No tests matched the filter string was found in output." + echo "##vso[task.logissue type=error] No tests matched the filter." exit 1 fi From 0fb32f169a5199fb5c2efe49cad85deab7534106 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Fri, 6 Jun 2025 07:03:55 +0000 Subject: [PATCH 16/17] Check if threshold mechanism is working: use the threshold higher than the total test number to make the CI fail. --- .azure/pipelines/components-e2e-tests.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index a7c6341b643f..b7cd5153e211 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -77,6 +77,16 @@ jobs: exit 1 fi + + # Check total tests run to detect abnormalities. In case the number of tests changes significantly, we should adjust the threshold. + # Extract total from the summary line "Failed: xx, Passed: yy, Skipped: zz, Total: NNN, Duration: ..." + total=$(sed -nE 's/.*Failed:[[:space:]]*[0-9]+,[[:space:]]*Passed:[[:space:]]*[0-9]+,[[:space:]]*Skipped:[[:space:]]*[0-9]+,[[:space:]]*Total:[[:space:]]*([0-9]+).*/\1/p' e2e-test-output.log) + min_total=2000 + if [ -z "$total" ] || [ "$total" -lt "$min_total" ] + then + echo "##vso[task.logissue type=error] Insufficient total test count: $total. We expect at least $min_total tests to run." + exit 1 + fi displayName: Run E2E tests - script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build --filter 'Quarantined=true' -p:RunQuarantinedTests=true -p:VsTestUseMSBuildOutput=false From a1f8a782bd0985304aff3557f13eb20f7e24e117 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Fri, 6 Jun 2025 08:20:34 +0000 Subject: [PATCH 17/17] Set the threshold to 1k. --- .azure/pipelines/components-e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure/pipelines/components-e2e-tests.yml b/.azure/pipelines/components-e2e-tests.yml index b7cd5153e211..38597db239a3 100644 --- a/.azure/pipelines/components-e2e-tests.yml +++ b/.azure/pipelines/components-e2e-tests.yml @@ -81,7 +81,7 @@ jobs: # Check total tests run to detect abnormalities. In case the number of tests changes significantly, we should adjust the threshold. # Extract total from the summary line "Failed: xx, Passed: yy, Skipped: zz, Total: NNN, Duration: ..." total=$(sed -nE 's/.*Failed:[[:space:]]*[0-9]+,[[:space:]]*Passed:[[:space:]]*[0-9]+,[[:space:]]*Skipped:[[:space:]]*[0-9]+,[[:space:]]*Total:[[:space:]]*([0-9]+).*/\1/p' e2e-test-output.log) - min_total=2000 + min_total=1000 if [ -z "$total" ] || [ "$total" -lt "$min_total" ] then echo "##vso[task.logissue type=error] Insufficient total test count: $total. We expect at least $min_total tests to run."