Skip to content

Commit 7d59d99

Browse files
authored
perf: replace jest fs traversal with bazel defined filelist (#301)
Ref #50
1 parent 9387f40 commit 7d59d99

File tree

5 files changed

+181
-3
lines changed

5 files changed

+181
-3
lines changed

jest/defs.bzl

+10
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,14 @@ def jest_test(
215215
visibility = ["//visibility:public"],
216216
)
217217

218+
bazel_haste_map_module = "_{}_bazel_haste_map_module".format(name)
219+
copy_file(
220+
name = bazel_haste_map_module,
221+
src = "@aspect_rules_jest//jest/private:bazel_haste_map.cjs",
222+
out = "_{}_bazel_haste_map_module.cjs".format(name),
223+
visibility = ["//visibility:public"],
224+
)
225+
218226
# This is the primary {name} jest_test test target
219227
_jest_from_node_modules(
220228
jest_rule = jest_test_rule,
@@ -233,6 +241,7 @@ def jest_test(
233241
bazel_sequencer = bazel_sequencer,
234242
bazel_snapshot_reporter = bazel_snapshot_reporter,
235243
bazel_snapshot_resolver = bazel_snapshot_resolver,
244+
bazel_haste_map_module = bazel_haste_map_module,
236245
**kwargs
237246
)
238247

@@ -255,6 +264,7 @@ def jest_test(
255264
bazel_sequencer = bazel_sequencer,
256265
bazel_snapshot_reporter = bazel_snapshot_reporter,
257266
bazel_snapshot_resolver = bazel_snapshot_resolver,
267+
bazel_haste_map_module = bazel_haste_map_module,
258268
tags = tags + ["manual"], # tagged manual so it is not built unless the {name}_update_snapshot target is run
259269
**kwargs
260270
)

jest/private/BUILD.bazel

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
33

44
exports_files([
55
"jest_config_template.mjs",
6+
"bazel_haste_map.cjs",
67
"bazel_sequencer.cjs",
78
"bazel_snapshot_reporter.cjs",
89
"bazel_snapshot_resolver.cjs",

jest/private/bazel_haste_map.cjs

+124
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
const { join, dirname, extname } = require("path");
2+
const { readFileSync } = require("fs");
3+
4+
// to require jest-haste-map we need to hop from jest-cli => @jest/core => jest-haste-map in the virtual store
5+
const jestCliPackage = dirname(require.resolve("jest-cli/package.json"));
6+
const jestConfigPackage = dirname(
7+
require.resolve(join(jestCliPackage, "../@jest/core/package.json")),
8+
);
9+
const HasteMap = require(
10+
join(jestConfigPackage, "../../jest-haste-map"),
11+
).default;
12+
const fastPath = require(
13+
join(jestConfigPackage, "../../jest-haste-map/build/lib/fast_path.js"),
14+
);
15+
16+
// The path to the rules_jest files list file
17+
const WORKSPACE_RUNFILES = join(
18+
process.env.TEST_SRCDIR,
19+
process.env.TEST_WORKSPACE,
20+
);
21+
const BAZEL_FILELIST_JSON_FULL_PATH = join(
22+
WORKSPACE_RUNFILES,
23+
global.BAZEL_FILELIST_JSON_SHORT_PATH,
24+
);
25+
26+
/**
27+
* Extend the standard jest HasteMap to use rules_jest
28+
*/
29+
module.exports = class BazelHasteMap extends HasteMap {
30+
constructor(options) {
31+
super({
32+
// Override some default HasteMap options.
33+
// These are HasteMap.Options, not Config.Options.haste
34+
dependencyExtractor: null,
35+
computeDependencies: false,
36+
resetCache: true,
37+
38+
...options,
39+
});
40+
41+
// Override the jest HasteMap._crawl() to not invoke `find` or `fs.*`
42+
this._crawl = this.bazelCrawl.bind(this);
43+
44+
// Override & remove the HastMap._persist() method to disable persisting the cache due to:
45+
// - when `config.cacheDirectory` is not persisted across jest_test invocations caching is ineffective
46+
// - when extending `HasteMap` the `BazelHasteMap` construction does not invoke the standard `HasteMap` factory
47+
// logic including `await HasteMap.setupCachePath()` which is required for `._persist()` to work.
48+
this._persist = function bazelNoopPersist() {};
49+
}
50+
51+
/**
52+
* A rules_jest replacement for the standard jest `crawl` function.
53+
*
54+
* Modified to:
55+
* - use the rules_jest replacement for `find` to avoid walking the filesystem
56+
* - disable or skip all jest caching that is not applicable in bazel
57+
* - assume rules_jest did all directory+symlink filtering to avoid stat() calls
58+
*
59+
* See https://github.com/jestjs/jest/blob/v29.7.0/packages/jest-haste-map/src/index.ts#L760-L773
60+
*/
61+
async bazelCrawl(hasteMap /*: InternalHasteMap*/) {
62+
const ignore = this._ignore.bind(this);
63+
const { extensions, rootDir, enableSymlinks, roots } = this._options;
64+
65+
return new Promise(function bazelCrawlExecutor(resolve) {
66+
function findComplete(files) {
67+
const filesMap = new Map();
68+
for (const file of files) {
69+
const relativeFilePath = fastPath.relative(rootDir, file);
70+
filesMap.set(relativeFilePath, [
71+
"", // ID
72+
0, // MTIME
73+
0, // SIZE
74+
0, // VISITED
75+
"", // DEPENDENCIES
76+
null, // SHA1 (disabled by default by rules_jest)
77+
]);
78+
}
79+
hasteMap.files = filesMap;
80+
81+
resolve({
82+
hasteMap: hasteMap,
83+
removedFiles: new Map(),
84+
});
85+
}
86+
87+
find(roots, extensions, ignore, enableSymlinks, findComplete);
88+
});
89+
}
90+
};
91+
92+
/**
93+
* A rules_jest replacement for standard jest fs `find` function.
94+
*
95+
* Differences from standard `find`:
96+
* - all fs info read from the BAZEL_FILELIST_JSON_FULL_PATH file outputted by the jest rule, no fs operations/traversal
97+
*
98+
* See: https://github.com/jestjs/jest/blob/v29.7.0/packages/jest-haste-map/src/crawlers/node.ts#L59-L131
99+
*/
100+
function find(roots, extensions, ignore, enableSymlinks, callback) {
101+
const files = JSON.parse(
102+
readFileSync(BAZEL_FILELIST_JSON_FULL_PATH, { encoding: "utf8" }),
103+
);
104+
105+
// TODO: exclude those not in `roots`?
106+
107+
const result = [];
108+
109+
for (const file of files) {
110+
const ext = extname(file).slice(1);
111+
if (!extensions.includes(ext)) {
112+
continue;
113+
}
114+
115+
const f = join(WORKSPACE_RUNFILES, file);
116+
if (ignore(f)) {
117+
continue;
118+
}
119+
120+
result.push(f);
121+
}
122+
123+
callback(result);
124+
}

jest/private/jest_config_template.mjs

+31-3
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ const bazelSnapshotReporterPath = _resolveRunfilesPath(
1818
const bazelSnapshotResolverPath = _resolveRunfilesPath(
1919
"{{BAZEL_SNAPSHOT_RESOLVER_SHORT_PATH}}",
2020
);
21+
const bazelHasteMapModulePath = _resolveRunfilesPath(
22+
"{{BAZEL_HASTE_MAP_MODULE_SHORT_PATH}}",
23+
);
2124

2225
if (
2326
!updateSnapshots &&
@@ -98,17 +101,42 @@ if (userConfigShortPath) {
98101

99102
_verifyJestConfig(config);
100103

104+
// Templated config that must be persisted globally for use by other rules_jest bazel_*.cjs files.
105+
global.BAZEL_FILELIST_JSON_SHORT_PATH = "{{BAZEL_FILELIST_JSON_SHORT_PATH}}";
106+
101107
// Default to using an isolated tmpdir
102108
config.cacheDirectory ||= path.join(process.env.TEST_TMPDIR, "jest_cache");
103109

104-
// Needed for Jest to walk the filesystem to find inputs.
105-
// See https://github.com/facebook/jest/pull/9351
106-
config.haste = { enableSymlinks: true, ...config.haste };
110+
config.haste = {
111+
// Needed for Jest to walk the filesystem to find inputs.
112+
// See https://github.com/facebook/jest/pull/9351
113+
enableSymlinks: true,
114+
115+
// Do not use external watchman or find, use a custom
116+
// HasteMap module designed for rules_jest
117+
forceNodeFilesystemAPI: true,
118+
hasteMapModulePath: bazelHasteMapModulePath,
119+
120+
// Use of SHA1, computing dependencies etc are all related to caching.
121+
// Disable them unless explicitly enabled by the user `config.haste`.
122+
computeSha1: false,
123+
124+
...config.haste,
125+
};
107126

108127
// https://jestjs.io/docs/cli#--watchman. Whether to use watchman for file crawling. Defaults
109128
// to true. Disable using --no-watchman. Watching is ibazel's job
110129
config.watchman = false;
111130

131+
// Watching and reinvoking tests is rule_jest + ibazel's job.
132+
config.watch = config.watchAll = false;
133+
134+
// Caching is bazel's job.
135+
config.cache = false;
136+
137+
// Change detection is bazel's job.
138+
config.onlyChanged = false;
139+
112140
// Auto configure reporters
113141
if (autoConfReporters) {
114142
// Default reporter should always be configured

jest/private/jest_test.bzl

+15
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ _attrs = dicts.add(js_binary_lib.attrs, {
2626
allow_single_file = True,
2727
mandatory = True,
2828
),
29+
"bazel_haste_map_module": attr.label(
30+
allow_single_file = True,
31+
mandatory = True,
32+
),
2933
"env_inherit": attr.string_list(
3034
doc = """Environment variables to inherit from the external environment.""",
3135
),
@@ -48,15 +52,24 @@ def _impl(ctx):
4852
generated_config = ctx.actions.declare_file("%s__jest.config.mjs" % ctx.label.name)
4953
user_config = copy_file_to_bin_action(ctx, ctx.file.config) if ctx.attr.config else None
5054

55+
filelist = ctx.actions.declare_file("%s__jest.files.json" % ctx.label.name)
56+
ctx.actions.write(
57+
output = filelist,
58+
content = json.encode([f.short_path for f in ctx.files.data if not f.is_directory]),
59+
is_executable = False,
60+
)
61+
5162
ctx.actions.expand_template(
5263
template = ctx.file._jest_config_template,
5364
output = generated_config,
5465
substitutions = {
5566
"{{AUTO_CONF_REPORTERS}}": "1" if ctx.attr.auto_configure_reporters else "",
5667
"{{AUTO_CONF_TEST_SEQUENCER}}": "1" if ctx.attr.auto_configure_test_sequencer else "",
68+
"{{BAZEL_FILELIST_JSON_SHORT_PATH}}": filelist.short_path,
5769
"{{BAZEL_SEQUENCER_SHORT_PATH}}": ctx.file.bazel_sequencer.short_path,
5870
"{{BAZEL_SNAPSHOT_REPORTER_SHORT_PATH}}": ctx.file.bazel_snapshot_reporter.short_path,
5971
"{{BAZEL_SNAPSHOT_RESOLVER_SHORT_PATH}}": ctx.file.bazel_snapshot_resolver.short_path,
72+
"{{BAZEL_HASTE_MAP_MODULE_SHORT_PATH}}": ctx.file.bazel_haste_map_module.short_path,
6073
"{{GENERATED_CONFIG_SHORT_PATH}}": generated_config.short_path,
6174
"{{USER_CONFIG_SHORT_PATH}}": user_config.short_path if user_config else "",
6275
"{{USER_CONFIG_PATH}}": user_config.path if user_config else "",
@@ -126,6 +139,8 @@ def _impl(ctx):
126139
files.append(ctx.file.bazel_sequencer)
127140
files.append(ctx.file.bazel_snapshot_reporter)
128141
files.append(ctx.file.bazel_snapshot_resolver)
142+
files.append(ctx.file.bazel_haste_map_module)
143+
files.append(filelist)
129144

130145
runfiles = ctx.runfiles(
131146
files = files,

0 commit comments

Comments
 (0)