Skip to content

Commit 292cd03

Browse files
mockersfmockersf
committed
run examples in wasm in CI (#4818)
# Objective - Run examples in WASM in CI - Fix #4817 ## Solution - on feature `bevy_ci_testing` - add an extra log message before exiting - when building for wasm, read CI config file at compile time - add a simple [playwright](https://playwright.dev) test script that opens the browser then waits for the success log, and takes a screenshot - add a CI job that runs the playwright test for Chromium and Firefox on one example (lighting) and save the screenshots - Firefox screenshot is good (with some clusters visible) - Chromium screenshot is gray, I don't know why but it's logging `GPU stall due to ReadPixels` - Webkit is not enabled for now, to revisit once https://bugs.webkit.org/show_bug.cgi?id=234926 is fixed or worked around - the CI job only runs on bors validation example run: https://github.com/mockersf/bevy/actions/runs/2361673465. The screenshots can be downloaded
1 parent 85cd0eb commit 292cd03

File tree

10 files changed

+395
-20
lines changed

10 files changed

+395
-20
lines changed

‎.github/bors.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ status = [
88
"build-android",
99
"markdownlint",
1010
"run-examples",
11+
"run-examples-on-wasm",
1112
"check-doc",
1213
"check-missing-examples-in-docs",
1314
"check-unused-dependencies",

‎.github/start-wasm-example/.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
node_modules/
2+
/test-results/
3+
/playwright-report/
4+
/playwright/.cache/

‎.github/start-wasm-example/package-lock.json

Lines changed: 76 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.github/start-wasm-example/package.json

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"name": "start-wasm-example",
3+
"version": "1.0.0",
4+
"description": "",
5+
"main": "index.js",
6+
"scripts": {},
7+
"keywords": [],
8+
"author": "",
9+
"license": "ISC",
10+
"devDependencies": {
11+
"@playwright/test": "^1.22.1"
12+
},
13+
"dependencies": {
14+
"dotenv": "^16.0.1"
15+
}
16+
}

‎.github/start-wasm-example/playwright.config.ts

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import type { PlaywrightTestConfig } from '@playwright/test';
2+
import { devices } from '@playwright/test';
3+
4+
/**
5+
* Read environment variables from file.
6+
* https://github.com/motdotla/dotenv
7+
*/
8+
require('dotenv').config();
9+
10+
/**
11+
* See https://playwright.dev/docs/test-configuration.
12+
*/
13+
const config: PlaywrightTestConfig = {
14+
testDir: './tests',
15+
/* Maximum time one test can run for. */
16+
timeout: 300_000,
17+
expect: {
18+
/**
19+
* Maximum time expect() should wait for the condition to be met.
20+
* For example in `await expect(locator).toHaveText();`
21+
*/
22+
timeout: 5000
23+
},
24+
/* Run tests in files in parallel */
25+
fullyParallel: true,
26+
/* Fail the build on CI if you accidentally left test.only in the source code. */
27+
forbidOnly: !!process.env.CI,
28+
/* Retry on CI only */
29+
retries: 0,
30+
/* Opt out of parallel tests on CI. */
31+
workers: 1,
32+
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
33+
reporter: 'list',
34+
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
35+
use: {
36+
/* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
37+
actionTimeout: 0,
38+
/* Base URL to use in actions like `await page.goto('/')`. */
39+
// baseURL: 'http://localhost:3000',
40+
41+
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
42+
trace: 'on-first-retry',
43+
},
44+
45+
/* Configure projects for major browsers */
46+
projects: [
47+
{
48+
name: 'chromium',
49+
use: {
50+
...devices['Desktop Chrome'],
51+
},
52+
},
53+
54+
{
55+
name: 'firefox',
56+
use: {
57+
...devices['Desktop Firefox'],
58+
},
59+
},
60+
61+
{
62+
name: 'webkit',
63+
use: {
64+
...devices['Desktop Safari'],
65+
},
66+
},
67+
68+
/* Test against mobile viewports. */
69+
// {
70+
// name: 'Mobile Chrome',
71+
// use: {
72+
// ...devices['Pixel 5'],
73+
// },
74+
// },
75+
// {
76+
// name: 'Mobile Safari',
77+
// use: {
78+
// ...devices['iPhone 12'],
79+
// },
80+
// },
81+
82+
/* Test against branded browsers. */
83+
// {
84+
// name: 'Microsoft Edge',
85+
// use: {
86+
// channel: 'msedge',
87+
// },
88+
// },
89+
// {
90+
// name: 'Google Chrome',
91+
// use: {
92+
// channel: 'chrome',
93+
// },
94+
// },
95+
],
96+
97+
/* Folder for test artifacts such as screenshots, videos, traces, etc. */
98+
// outputDir: 'test-results/',
99+
100+
/* Run your local dev server before starting the tests */
101+
// webServer: {
102+
// command: 'npm run start',
103+
// port: 3000,
104+
// },
105+
};
106+
107+
export default config;

‎.github/start-wasm-example/tests/wasm_example.spec.ts

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import { test, expect, Page } from '@playwright/test';
2+
3+
test.beforeEach(async ({ page }) => {
4+
await page.goto('http://localhost:8000/');
5+
});
6+
7+
const MAX_TIMEOUT_FOR_TEST = 300_000;
8+
9+
test.describe('WASM example', () => {
10+
test('Wait for success', async ({ page }, test_info) => {
11+
let start = new Date().getTime();
12+
13+
let found = false;
14+
while (new Date().getTime() - start < MAX_TIMEOUT_FOR_TEST) {
15+
let msg = await promise_with_timeout(100, on_console(page), "no log found");
16+
if (msg.includes("no log found")) {
17+
continue;
18+
}
19+
console.log(msg);
20+
if (msg.includes("Test successful")) {
21+
let prefix = process.env.SCREENSHOT_PREFIX === undefined ? "screenshot" : process.env.SCREENSHOT_PREFIX;
22+
await page.screenshot({ path: `${prefix}-${test_info.project.name}.png`, fullPage: true });
23+
found = true;
24+
break;
25+
}
26+
}
27+
28+
expect(found).toBe(true);
29+
});
30+
31+
});
32+
33+
function on_console(page) {
34+
return new Promise(resolve => {
35+
page.on('console', msg => resolve(msg.text()));
36+
});
37+
}
38+
39+
async function promise_with_timeout(time_limit, task, failure_value) {
40+
let timeout;
41+
const timeout_promise = new Promise((resolve, reject) => {
42+
timeout = setTimeout(() => {
43+
resolve(failure_value);
44+
}, time_limit);
45+
});
46+
const response = await Promise.race([task, timeout_promise]);
47+
if (timeout) {
48+
clearTimeout(timeout);
49+
}
50+
return response;
51+
}

‎.github/workflows/validation-jobs.yml

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,3 +97,61 @@ jobs:
9797
time CI_TESTING_CONFIG=$example cargo run --example $example_name --features "bevy_ci_testing"
9898
sleep 10
9999
done
100+
101+
run-examples-on-wasm:
102+
runs-on: ubuntu-latest
103+
timeout-minutes: 60
104+
steps:
105+
- uses: actions/checkout@v3
106+
107+
- uses: actions-rs/toolchain@v1
108+
with:
109+
toolchain: stable
110+
target: wasm32-unknown-unknown
111+
override: true
112+
113+
- uses: actions/cache@v3
114+
with:
115+
path: |
116+
~/.cargo/bin/
117+
~/.cargo/registry/index/
118+
~/.cargo/registry/cache/
119+
~/.cargo/git/db/
120+
~/.github/start-wasm-example/node_modules
121+
target/
122+
key: ${{ runner.os }}-wasm-run-examples-${{ hashFiles('**/Cargo.toml') }}
123+
124+
- name: install xvfb, llvmpipe and lavapipe
125+
run: |
126+
sudo apt-get update -y -qq
127+
sudo add-apt-repository ppa:oibaf/graphics-drivers -y
128+
sudo apt-get update
129+
sudo apt install -y xvfb libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers
130+
131+
- name: Install wasm-bindgen
132+
run: cargo install --force wasm-bindgen-cli
133+
134+
- name: Setup playwright
135+
run: |
136+
cd .github/start-wasm-example
137+
npm install
138+
npx playwright install --with-deps
139+
cd ../..
140+
141+
- name: First WASM build
142+
run: |
143+
cargo build --release --example ui --target wasm32-unknown-unknown
144+
145+
- name: Run examples
146+
shell: bash
147+
run: |
148+
# start a webserver
149+
python3 -m http.server --directory examples/wasm &
150+
151+
xvfb-run cargo run -p build-wasm-example -- --browsers chromium --browsers firefox --frames 25 --test shapes lighting text_debug breakout
152+
153+
- name: Save screenshots
154+
uses: actions/upload-artifact@v3
155+
with:
156+
name: screenshots
157+
path: .github/start-wasm-example/screenshot-*.png

‎crates/bevy_app/src/ci_testing.rs

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
use crate::{app::AppExit, App};
22
use serde::Deserialize;
33

4+
use bevy_utils::tracing::info;
5+
46
/// A configuration struct for automated CI testing.
57
///
68
/// It gets used when the `bevy_ci_testing` feature is enabled to automatically
@@ -20,18 +22,29 @@ fn ci_testing_exit_after(
2022
if let Some(exit_after) = ci_testing_config.exit_after {
2123
if *current_frame > exit_after {
2224
app_exit_events.send(AppExit);
25+
info!("Exiting after {} frames. Test successful!", exit_after);
2326
}
2427
}
2528
*current_frame += 1;
2629
}
2730

2831
pub(crate) fn setup_app(app: &mut App) -> &mut App {
29-
let filename =
30-
std::env::var("CI_TESTING_CONFIG").unwrap_or_else(|_| "ci_testing_config.ron".to_string());
31-
let config: CiTestingConfig = ron::from_str(
32-
&std::fs::read_to_string(filename).expect("error reading CI testing configuration file"),
33-
)
34-
.expect("error deserializing CI testing configuration file");
32+
#[cfg(not(target_arch = "wasm32"))]
33+
let config: CiTestingConfig = {
34+
let filename = std::env::var("CI_TESTING_CONFIG")
35+
.unwrap_or_else(|_| "ci_testing_config.ron".to_string());
36+
ron::from_str(
37+
&std::fs::read_to_string(filename)
38+
.expect("error reading CI testing configuration file"),
39+
)
40+
.expect("error deserializing CI testing configuration file")
41+
};
42+
#[cfg(target_arch = "wasm32")]
43+
let config: CiTestingConfig = {
44+
let config = include_str!("../../../ci_testing_config.ron");
45+
ron::from_str(config).expect("error deserializing CI testing configuration file")
46+
};
47+
3548
app.insert_resource(config)
3649
.add_system(ci_testing_exit_after);
3750

‎tools/build-wasm-example/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,4 @@ license = "MIT OR Apache-2.0"
99

1010
[dependencies]
1111
xshell = "0.2"
12+
clap = { version = "3.1.12", features = ["derive"] }

0 commit comments

Comments
 (0)