Skip to content

Commit da62195

Browse files
NateowamiNateowami
committed
Add e2e tests
1 parent 15bb5cc commit da62195

25 files changed

+2798
-9
lines changed

‎.github/workflows/e2e-tests.yml

+120
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
name: E2E Tests
2+
permissions: {}
3+
4+
on:
5+
# push:
6+
# branches: [develop, master, sf-qa, sf-live]
7+
pull_request:
8+
workflow_dispatch:
9+
10+
jobs:
11+
e2e_tests:
12+
name: "E2E Tests"
13+
environment: "e2e_tests"
14+
strategy:
15+
matrix:
16+
os: ["ubuntu-22.04"]
17+
dotnet_version: ["8.0.x"]
18+
node_version: ["22.13.0"]
19+
npm_version: ["10.9.2"]
20+
runs-on: ${{matrix.os}}
21+
steps:
22+
- name: Checkout
23+
uses: actions/checkout@v4
24+
with:
25+
persist-credentials: false
26+
27+
- name: "Deps: .NET"
28+
uses: actions/setup-dotnet@v4
29+
with:
30+
dotnet-version: ${{matrix.dotnet_version}}
31+
cache: true
32+
cache-dependency-path: src/SIL.XForge.Scripture/packages.lock.json
33+
- name: "Deps: Node"
34+
uses: actions/setup-node@v4
35+
with:
36+
node-version: ${{matrix.node_version}}
37+
cache: "npm"
38+
cache-dependency-path: |
39+
src/SIL.XForge.Scripture/ClientApp/package-lock.json
40+
src/RealtimeServer/package-lock.json
41+
- name: "Deps: npm"
42+
env:
43+
NPM_VERSION: ${{matrix.npm_version}}
44+
run: |
45+
set -xueo pipefail
46+
npm install --global npm@${NPM_VERSION}
47+
- name: Pre-build report
48+
run: |
49+
set -xueo pipefail
50+
lsb_release -a
51+
which dotnet
52+
dotnet --version
53+
dpkg -l dotnet\*
54+
dotnet --list-sdks
55+
dotnet --list-runtimes
56+
which node
57+
node --version
58+
which npm
59+
npm --version
60+
which chromium-browser
61+
chromium-browser --version
62+
- name: "Ensure desired tool versions"
63+
# The build machine may come with newer tools than we are ready for.
64+
env:
65+
NODE_VERSION: ${{matrix.node_version}}
66+
NPM_VERSION: ${{matrix.npm_version}}
67+
run: |
68+
set -xueo pipefail
69+
[[ $(node --version) == v${NODE_VERSION} ]]
70+
[[ $(npm --version) == ${NPM_VERSION} ]]
71+
72+
- name: "Deps: RealtimeServer npm"
73+
run: cd src/RealtimeServer && (npm ci || (sleep 3m && npm ci))
74+
- name: "Deps: Backend nuget"
75+
run: dotnet restore
76+
- name: "Deps: Frontend npm"
77+
run: cd src/SIL.XForge.Scripture/ClientApp && (npm ci || (sleep 3m && npm ci))
78+
79+
- name: "Build: Backend, RealtimeServer"
80+
run: dotnet build xForge.sln
81+
- name: "Build: Frontend"
82+
run: cd src/SIL.XForge.Scripture/ClientApp && npm run build
83+
84+
# Set up secrets
85+
- name: "Configure secrets"
86+
env:
87+
PARATEXT_CLIENT_ID: ${{ secrets.PARATEXT_CLIENT_ID }}
88+
PARATEXT_CLIENT_SECRET: ${{ secrets.PARATEXT_CLIENT_SECRET }}
89+
AUTH0_BACKEND_CLIENT_SECRET: ${{ secrets.AUTH0_BACKEND_CLIENT_SECRET }}
90+
PARATEXT_RESOURCE_PASSWORD_HASH: ${{ secrets.PARATEXT_RESOURCE_PASSWORD_HASH }}
91+
PARATEXT_RESOURCE_PASSWORD_BASE64: ${{ secrets.PARATEXT_RESOURCE_PASSWORD_BASE64 }}
92+
SERVAL_CLIENT_ID: ${{ secrets.SERVAL_CLIENT_ID }}
93+
SERVAL_CLIENT_SECRET: ${{ secrets.SERVAL_CLIENT_SECRET }}
94+
E2E_SECRETS_JSON_BASE64: ${{ secrets.E2E_SECRETS_JSON_BASE64 }}
95+
run: |
96+
set -xueo pipefail
97+
cd src/SIL.XForge.Scripture/
98+
dotnet user-secrets set "Paratext:ClientId" "${PARATEXT_CLIENT_ID}"
99+
dotnet user-secrets set "Paratext:ClientSecret" "${PARATEXT_CLIENT_SECRET}"
100+
dotnet user-secrets set "Auth:BackendClientSecret" "${AUTH0_BACKEND_CLIENT_SECRET}"
101+
dotnet user-secrets set "Paratext:ResourcePasswordHash" "${PARATEXT_RESOURCE_PASSWORD_HASH}"
102+
dotnet user-secrets set "Paratext:ResourcePasswordBase64" "${PARATEXT_RESOURCE_PASSWORD_BASE64}"
103+
dotnet user-secrets set "Serval:ClientId" "${SERVAL_CLIENT_ID}"
104+
dotnet user-secrets set "Serval:ClientSecret" "${SERVAL_CLIENT_SECRET}",
105+
base64 --decode - <<< "${E2E_SECRETS_JSON_BASE64}" > ./ClientApp/e2e/secrets.json
106+
107+
- name: Set up Deno
108+
uses: denoland/setup-deno@v2
109+
with:
110+
deno-version: v2.x
111+
112+
- name: Playwright install browsers"
113+
run: npx playwright install
114+
115+
- name: Run E2E tests
116+
run: |
117+
set -xueo pipefail
118+
cd src/SIL.XForge.Scripture
119+
dotnet run &
120+
cd ./ClientApp/e2e/ && ./e2e.mts

‎src/SIL.XForge.Scripture/ClientApp/e2e/.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
test_output
2+
videos
3+
secrets.json

‎src/SIL.XForge.Scripture/ClientApp/e2e/README.md

+114
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
# Scripture Forge End-to-End Tests
2+
3+
## Testing philosophy
4+
5+
### The testing pyramid
6+
7+
The greater focus on integration tests rather than E2E tests in this version of Scripture Forge came from this Google
8+
developer blog post: https://testing.googleblog.com/2015/04/just-say-no-to-more-end-to-end-tests.html
9+
10+
The main point is to use unit tests as much as possible, use integration tests for what unit tests can't cover, and use
11+
E2E tests for what only E2E tests can cover. This is mainly because unit tests are faster, more reliable, and pinpoint
12+
the source of the problem more accurately.
13+
14+
It's not that E2E tests are bad, but that they come with trade-offs that should be considered.
15+
16+
### A pyramid approach to the E2E tests themselves
17+
18+
While the article above focuses on different types of tests, the same principle can be applied to the E2E tests
19+
themselves. Once a test is written, it may be possible to run that test across multiple browsers, viewport sizes,
20+
localization languages, and depending on the test type, different user roles. Being able to test every possible
21+
permutation of these variables is extremely powerful, but it also takes much longer to run (and the probability of
22+
failure due to flakey tests increases).
23+
24+
Rather than choosing to always have a ton of tests or only a few tests, we can scale the number of tests based on the
25+
situation. In general, we should run as many tests as possible without sacrificing efficiency. In general this means:
26+
27+
1. Pull requests should run as many E2E tests as can be run without slowing down the process (i.e. they need to be
28+
reliable and take no longer than the other checks that are run on pull requests)
29+
2. Pull requests that make major changes should have more tests run (this could be a manual step or somehow configured
30+
in the CI by setting a tag on the pull request).
31+
3. Release candidates should run as many tests as possible.
32+
33+
### Other goals
34+
35+
Instrumentation for E2E tests can be used for more than automated testing. We can also use it to create screenshots for
36+
visual regression testing, and to keep screenshots in documentation up to date and localized. Some of this would incur
37+
additional effort to implement, but the instrumentation should be designed with this future in mind.
38+
39+
## Implementation
40+
41+
Playwright is being used for the E2E tests. It comes with both a library for driving the browser, and a test runner. For
42+
the most part, I have avoided using the test runner, opting instead to use the library directly. This gives a lot more
43+
flexibility in controlling what tests are run. The Playwright test runner is powerful, allowing for permutations of
44+
tests to be run, and multiple browsers run in parallel. However, there are also scenarios where more flexibility is
45+
needed, such as when running smoke tests for each user role. The admin user needs to be able to create the share links
46+
that are then used for invitations, having one test use the output of another.
47+
48+
I opted to use Deno rather than Node for the E2E tests, though this comes with some drawbacks (see the "Working with
49+
Deno" section below).
50+
51+
There are two types of tests that have been created so far:
52+
53+
- Smoke tests: The tests log in as each user role, navigate to each page, and take a screenshot on each.
54+
- Workflow: A specific workflow that a user may perform is tested from start to finish.
55+
56+
## Running tests
57+
58+
A test plan is defined in `e2e-globals.ts` as a "Preset". It defines which locales to test, which browser engines,
59+
user roles, whether to take screenshots, etc. It also defines which categories of tests should be run (e.g. smoke tests,
60+
generating a draft, community checking). When the tests are executed, the run sheet should be followed to the degree
61+
that is possible. For example, the smoke tests should test only the user roles specified in the run sheet, but
62+
certain tests are specific to a given role (for example, you have to be an admin to set up a community checking
63+
project), and won't need to consider the specified roles.
64+
65+
To run the tests, make any necessary edits to the run sheet, then run `e2e.ts`.
66+
67+
Screenshots are saved in the `screenshots` directory, in a subfolder specified by the run sheet. The default subfolder
68+
name is the timestamp when the tests started.
69+
70+
A file named `run_log.json` is saved to the directory with information about the test run and metadata regarding each of
71+
the screenshots.
72+
73+
## Other notes
74+
75+
### Working with Deno
76+
77+
Unfortunately, I have not found a good way to make Deno play nicely with Node and Angular. In VS Code, I always run the
78+
`Deno: Enable` command when working with files that will be run by Deno, and then run `Deno: Disable` when switching to
79+
other TypeScript files. When Deno is disabled the language server complains about problems in the files intended to be
80+
run by Deno, and when Deno is enabled the language server complains about problems in the other files.
81+
82+
Hopefully a better solution is available.
83+
84+
### Making utility functions wait for completion
85+
86+
A utility function that performs an action should also wait for for any side effects of the action to complete before
87+
returning. For example, a function that deletes the current project should wait until the user is redirected to the my
88+
projects page before returning. This can be done by waiting for the URL to change. This has two main benefits:
89+
90+
1. Whatever action runs next does not need to wait for the side effects of the previous action to complete.
91+
2. When failures occur (such as if the redirect following the deletion doesn't happen), it's much easier to determine
92+
where things went wrong, because the failure will occur in the function where the problem originated.
93+
94+
### Recording tests
95+
96+
In general it does not work well to just record a test with Playwright and then consider it a finished test. However, it
97+
can be much quicker to have Playwright record a test and then use that as a starting point. You can record a test by
98+
running `npx playwright codegen`, or by calling `await page.pause()` in a test, which stops execution and opens a second
99+
inspector window, which allows recording of tests, or using Playwright's locator tool.
100+
101+
## Future plans
102+
103+
Workflow tests that should be created:
104+
105+
- Community checking
106+
- Editing, including simultaneous editing and change in network status
107+
- Serval admins
108+
- Site admins
109+
110+
Other use-cases for the E2E tests:
111+
112+
- Automated screenshot comparison
113+
- Localized screenshots for documentation
114+
- Help videos

‎src/SIL.XForge.Scripture/ClientApp/e2e/characterization-trace-generate_draft-2025-04-01T18:36:22.742Z.zip

3.57 MB
Binary file not shown.

‎src/SIL.XForge.Scripture/ClientApp/e2e/characterize-tests.mts

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#!/usr/bin/env -S deno run --allow-run --allow-env --allow-sys --allow-read --allow-write --unstable-sloppy-imports
2+
import { chromium } from "npm:playwright";
3+
import { preset } from "./e2e-globals.ts";
4+
import { numberOfTimesToAttemptTest } from "./pass-probability-ts";
5+
import { ScreenshotContext } from "./presets.ts";
6+
import { tests } from "./test-definitions.ts";
7+
8+
const resultFilePath = "test_characterization.json";
9+
const testNames = Object.keys(tests) as (keyof typeof tests)[];
10+
let mostRecentResultData = JSON.parse(await Deno.readTextFile(resultFilePath));
11+
12+
printRetriesForEachTest();
13+
14+
while (true) {
15+
const testName = nextTestToRun();
16+
const testFunction = tests[testName];
17+
const browser = await chromium.launch({ headless: true });
18+
const browserContext = await browser.newContext();
19+
await browserContext.grantPermissions(["clipboard-read", "clipboard-write"]);
20+
await browserContext.tracing.start({ screenshots: true, snapshots: true, sources: true });
21+
const page = await browserContext.newPage();
22+
const screenshotContext: ScreenshotContext = { engine: "chromium" };
23+
try {
24+
const attempts = numberOfTimesToAttemptTest(testName, mostRecentResultData);
25+
console.log(`Running test: ${testName}, which currently has ${attempts} attempts`);
26+
27+
await testFunction(chromium, page, screenshotContext);
28+
await saveResult("success", testName);
29+
} catch (error: unknown) {
30+
console.error(error);
31+
await saveResult("failure", testName);
32+
const tracePath = `${preset.outputDir}/characterization-trace-${testName}-${new Date().toISOString()}.zip`;
33+
console.log(`Saving trace to ${tracePath}`);
34+
browserContext.tracing.stop({ path: tracePath });
35+
} finally {
36+
printRetriesForEachTest();
37+
}
38+
}
39+
40+
async function saveResult(result: "success" | "failure", testName: string): Promise<void> {
41+
mostRecentResultData = JSON.parse(await Deno.readTextFile(resultFilePath));
42+
mostRecentResultData[testName] ??= {};
43+
mostRecentResultData[testName][result] ??= 0;
44+
mostRecentResultData[testName][result]++;
45+
await Deno.writeTextFile(resultFilePath, JSON.stringify(mostRecentResultData, null, 2) + "\n");
46+
console.log(
47+
`%c✔ Test ${testName} finished with result: ${result}`,
48+
`color: ${result === "success" ? "green" : "red"}`
49+
);
50+
}
51+
52+
function nextTestToRun(): keyof typeof tests {
53+
return testNames
54+
.slice()
55+
.sort(
56+
(a, b) =>
57+
numberOfTimesToAttemptTest(b, mostRecentResultData) - numberOfTimesToAttemptTest(a, mostRecentResultData)
58+
)[0];
59+
}
60+
61+
function printRetriesForEachTest(): void {
62+
const info: { [key: string]: number } = {};
63+
for (const testName of testNames) {
64+
info[testName] = numberOfTimesToAttemptTest(testName, mostRecentResultData);
65+
}
66+
console.log("Retries for each test:");
67+
console.table(info);
68+
}

‎src/SIL.XForge.Scripture/ClientApp/e2e/e2e-globals.ts

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
import { E2ETestRunLogger } from './e2e-test-run-logger.ts';
2+
import { presets, TestPreset } from './presets.ts';
3+
import secrets from './secrets.json' with { type: 'json' };
4+
5+
const _allBrowsers = ['chromium', 'firefox', 'webkit'] as const;
6+
export type BrowserName = (typeof _allBrowsers)[number];
7+
export const allRoles = [
8+
'pt_administrator',
9+
'pt_translator',
10+
'pt_consultant',
11+
'pt_observer',
12+
'community_checker',
13+
'commenter',
14+
'viewer'
15+
] as const;
16+
export type UserRole = (typeof allRoles)[number];
17+
18+
export interface ScreenshotContext {
19+
engine: BrowserName;
20+
role?: UserRole;
21+
pageName?: string;
22+
locale?: string;
23+
}
24+
25+
// TODO Create a separate project for each test
26+
export const DEFAULT_PROJECT_SHORTNAME = 'Stp22';
27+
export const CHECKING_PROJECT_NAMES = 'SEEC2';
28+
29+
export const logger = new E2ETestRunLogger();
30+
31+
console.log(Deno.args);
32+
function getPreset(): TestPreset {
33+
const presetName = Deno.args[0] ?? 'default';
34+
const availablePresets = Object.keys(presets);
35+
if (!availablePresets.includes(presetName)) {
36+
console.error(`Usage: ./e2e.mts <preset> <test1> <test2> ...`);
37+
throw new Error(`Invalid preset "${presetName}". Available presets: ${availablePresets.join(', ')}`);
38+
}
39+
const preset = (presets as any)[presetName];
40+
console.log(`Using preset: ${presetName}`);
41+
console.log(preset);
42+
return preset;
43+
}
44+
45+
export const preset = getPreset();
46+
47+
export const ptUsersByRole = {
48+
[allRoles[0]]: secrets.users[0],
49+
[allRoles[1]]: secrets.users[1],
50+
[allRoles[2]]: secrets.users[2],
51+
[allRoles[3]]: secrets.users[3]
52+
} as const;

‎src/SIL.XForge.Scripture/ClientApp/e2e/e2e-test-run-logger.ts

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import { preset, ScreenshotContext } from './e2e-globals.ts';
2+
3+
interface ScreenshotEvent {
4+
fileName: string;
5+
context: ScreenshotContext;
6+
}
7+
8+
export class E2ETestRunLogger {
9+
private timeStarted = new Date();
10+
private screenshotEvents: ScreenshotEvent[] = [];
11+
12+
async saveToFile(): Promise<void> {
13+
const data = {
14+
timeStarted: this.timeStarted,
15+
timeEnded: new Date(),
16+
preset,
17+
screenshotEvents: this.screenshotEvents
18+
};
19+
20+
const filePath = `${preset.outputDir}/run_log.json`;
21+
console.log(`Saving run log to ${filePath}...`);
22+
await Deno.mkdir(preset.outputDir, { recursive: true });
23+
await Deno.writeTextFile(filePath, JSON.stringify(data, null, 2));
24+
}
25+
26+
logScreenshot(fileName: string, context: ScreenshotContext): void {
27+
this.screenshotEvents.push({ fileName, context });
28+
}
29+
}

0 commit comments

Comments
 (0)