diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7279c904..fd19b3a7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,10 +30,24 @@ jobs: run: npm install - name: Build run: npm run build + - name: Set environment variable for log path + ## Use /tmp for beacon logging as less prone to delays + run: echo "LOG_PATH=/tmp/beacons.log" >> $GITHUB_ENV - name: Run server run: npm run test:server & - name: Run e2e tests for chrome - run: npm run test:e2e -- --browsers=chrome + ## Retire up to 5 times as tests on GitHub Actions are flakey :-( + run: | + for i in 1 2 3 4 5; do + echo "Attempt #$i" + if npm run test:e2e -- --browsers=chrome; then + exit 0 + fi + echo "Attempt #$i failed" >&2 + sleep 5 + done + echo "All attempts failed." + exit 1 firefox-tests: name: Run Firefox e2e tests # Runs best on macos for CI as linux requires extra setup @@ -45,10 +59,24 @@ jobs: run: npm install - name: Build run: npm run build + - name: Set environment variable for log path + ## Use /tmp for beacon logging as less prone to delays + run: echo "LOG_PATH=/tmp/beacons.log" >> $GITHUB_ENV - name: Run server run: npm run test:server & - name: Run e2e tests for firefox - run: npm run test:e2e -- --browsers=firefox + ## Retire up to 5 times as tests on GitHub Actions are flakey :-( + run: | + for i in 1 2 3 4 5; do + echo "Attempt #$i" + if npm run test:e2e -- --browsers=firefox; then + exit 0 + fi + echo "Attempt #$i failed" >&2 + sleep 5 + done + echo "All attempts failed" + exit 1 safari-tests: name: Run Safari e2e tests # Requires macos @@ -60,7 +88,21 @@ jobs: run: npm install - name: Build run: npm run build + - name: Set environment variable for log path + ## Use /tmp for beacon logging as less prone to delays + run: echo "LOG_PATH=/tmp/beacons.log" >> $GITHUB_ENV - name: Run server run: npm run test:server & - name: Run e2e tests for safari - run: npm run test:e2e -- --browsers=safari + ## Retire up to 5 times as tests on GitHub Actions are flakey :-( + run: | + for i in 1 2 3 4 5; do + echo "Attempt #$i" + if npm run test:e2e -- --browsers=safari; then + exit 0 + fi + echo "Attempt #$i failed" >&2 + sleep 5 + done + echo "All attempts failed" + exit 1 diff --git a/test/e2e/onFCP-test.js b/test/e2e/onFCP-test.js index d7748136..d3764caf 100644 --- a/test/e2e/onFCP-test.js +++ b/test/e2e/onFCP-test.js @@ -130,7 +130,7 @@ describe('onFCP()', async function () { it('does not report if the document changes to hidden before the first entry', async function () { if (!browserSupportsFCP) this.skip(); - await navigateTo('/test/fcp?invisible=1', {readyState: 'interactive'}); + await navigateTo('/test/fcp?invisible=1', {readyState: 'complete'}); await stubVisibilityChange('hidden'); await stubVisibilityChange('visible'); @@ -211,7 +211,7 @@ describe('onFCP()', async function () { it('reports if the page is restored from bfcache even when the document was hidden at page load time', async function () { if (!browserSupportsFCP) this.skip(); - await navigateTo('/test/fcp?hidden=1', {readyState: 'interactive'}); + await navigateTo('/test/fcp?hidden=1', {readyState: 'complete'}); await stubVisibilityChange('visible'); diff --git a/test/e2e/onINP-test.js b/test/e2e/onINP-test.js index 4db92e47..6974dfdf 100644 --- a/test/e2e/onINP-test.js +++ b/test/e2e/onINP-test.js @@ -43,7 +43,7 @@ describe('onINP()', async function () { it('reports the correct value on visibility hidden after interactions (reportAllChanges === false)', async function () { if (!browserSupportsINP) this.skip(); - await navigateTo('/test/inp?click=100', {readyState: 'interactive'}); + await navigateTo('/test/inp?click=100', {readyState: 'complete'}); const h1 = await $('h1'); await simulateUserLikeClick(h1); @@ -67,7 +67,7 @@ describe('onINP()', async function () { if (!browserSupportsINP) this.skip(); await navigateTo('/test/inp?click=100&reportAllChanges=1', { - readyState: 'interactive', + readyState: 'complete', }); const h1 = await $('h1'); @@ -89,9 +89,9 @@ describe('onINP()', async function () { it('reports the correct value when script is loaded late (reportAllChanges === false)', async function () { if (!browserSupportsINP) this.skip(); - // Don't await the `interactive` ready state because DCL is delayed until + // Don't await the `complete` ready state because DCL is delayed until // after user input. - await navigateTo('/test/inp?click=100&loadAfterInput=1'); + await navigateTo('/test/inp?click=150&loadAfterInput=1'); // Wait until await nextFrame(); @@ -117,9 +117,9 @@ describe('onINP()', async function () { it('reports the correct value when loaded late (reportAllChanges === true)', async function () { if (!browserSupportsINP) this.skip(); - // Don't await the `interactive` ready state because DCL is delayed until + // Don't await the `complete` ready state because DCL is delayed until // after user input. - await navigateTo('/test/inp?click=100&reportAllChanges=1&loadAfterInput=1'); + await navigateTo('/test/inp?click=150&reportAllChanges=1&loadAfterInput=1'); // Wait until await nextFrame(); @@ -143,7 +143,7 @@ describe('onINP()', async function () { it('reports the correct value on page unload after interactions (reportAllChanges === false)', async function () { if (!browserSupportsINP) this.skip(); - await navigateTo('/test/inp?click=100', {readyState: 'interactive'}); + await navigateTo('/test/inp?click=100', {readyState: 'complete'}); const h1 = await $('h1'); await simulateUserLikeClick(h1); @@ -167,7 +167,7 @@ describe('onINP()', async function () { if (!browserSupportsINP) this.skip(); await navigateTo('/test/inp?click=100&reportAllChanges=1', { - readyState: 'interactive', + readyState: 'complete', }); const h1 = await $('h1'); @@ -192,7 +192,7 @@ describe('onINP()', async function () { if (!browserSupportsINP) this.skip(); await navigateTo('/test/inp?click=60&pointerdown=600', { - readyState: 'interactive', + readyState: 'complete', }); const h1 = await $('h1'); @@ -254,7 +254,7 @@ describe('onINP()', async function () { if (!browserSupportsINP) this.skip(); await navigateTo('/test/inp?click=60&pointerdown=600&reportAllChanges=1', { - readyState: 'interactive', + readyState: 'complete', }); const h1 = await $('h1'); @@ -293,7 +293,7 @@ describe('onINP()', async function () { it('reports a new interaction after bfcache restore', async function () { if (!browserSupportsINP) this.skip(); - await navigateTo('/test/inp', {readyState: 'interactive'}); + await navigateTo('/test/inp', {readyState: 'complete'}); await setBlockingTime('click', 100); @@ -379,7 +379,7 @@ describe('onINP()', async function () { it('does not report if there were no interactions', async function () { if (!browserSupportsINP) this.skip(); - await navigateTo('/test/inp', {readyState: 'interactive'}); + await navigateTo('/test/inp', {readyState: 'complete'}); await stubVisibilityChange('hidden'); @@ -394,7 +394,7 @@ describe('onINP()', async function () { if (!browserSupportsINP) this.skip(); await navigateTo('/test/inp?click=100&prerender=1', { - readyState: 'interactive', + readyState: 'complete', }); const h1 = await $('h1'); @@ -419,7 +419,7 @@ describe('onINP()', async function () { if (!browserSupportsINP) this.skip(); await navigateTo('/test/inp?click=100&wasDiscarded=1', { - readyState: 'interactive', + readyState: 'complete', }); const h1 = await $('h1'); @@ -618,7 +618,7 @@ describe('onINP()', async function () { await beaconCountIs(1); const [inp1] = await getBeacons(); - assert.equal(inp1.attribution.loadState, 'dom-interactive'); + assert.match(inp1.attribution.loadState, /^(loading|dom-interactive)$/); await clearBeacons(); @@ -646,7 +646,7 @@ describe('onINP()', async function () { if (!browserSupportsINP) this.skip(); await navigateTo('/test/inp?attribution=1&mouseup=100&click=50', { - readyState: 'interactive', + readyState: 'complete', }); const h1 = await $('h1'); @@ -672,7 +672,7 @@ describe('onINP()', async function () { if (!browserSupportsINP) this.skip(); await navigateTo('/test/inp?attribution=1&mouseup=100&click=50', { - readyState: 'interactive', + readyState: 'complete', }); const button = await $('#reset'); @@ -700,7 +700,7 @@ describe('onINP()', async function () { if (!browserSupportsLoAF) this.skip(); await navigateTo('/test/inp?attribution=1&pointerdown=100', { - readyState: 'interactive', + readyState: 'complete', }); // Click on the