Skip to content

Commit effc827

Browse files
authored
Add invalidation stress test. (#4010)
* Add invalidation stress test. Add test cases for, and fix, two bugs it caught. * Address review comments.
1 parent c7b2fc7 commit effc827

8 files changed

+476
-69
lines changed

build/lib/src/library_cycle_graph/asset_deps_loader.dart

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,12 +72,20 @@ class _InMemoryAssetDepsLoader implements AssetDepsLoader {
7272
);
7373
PhasedAssetDeps phasedAssetDeps;
7474

75-
_InMemoryAssetDepsLoader(this.phasedAssetDeps);
76-
77-
// This is important: it prevents LibraryCycleGraphLoader from trying to load
78-
// data that is not in an incomplete [phasedAssetDeps].
75+
_InMemoryAssetDepsLoader(PhasedAssetDeps phasedAssetDeps)
76+
: phasedAssetDeps = phasedAssetDeps.complete();
77+
78+
// Return very high phase to tell `LibraryCycleGraphLoader` that all data is
79+
// available.
80+
//
81+
// Returning incomplete data would then cause `LibraryCycleGraphLoader` to
82+
// get stuck, which is why `phasedAssetDeps.complete` was called to mark
83+
// all the data complete.
84+
//
85+
// This loader is only used for rebuilding graphs constructed in an earlier
86+
// run, so incomplete data won't actually be used.
7987
@override
80-
int get phase => phasedAssetDeps.phase;
88+
int get phase => 0xffffffff;
8189

8290
@override
8391
ExpiringValue<AssetDeps> _parse(AssetId id, ExpiringValue<String> content) =>

build/lib/src/library_cycle_graph/library_cycle_graph_loader.dart

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -104,9 +104,9 @@ class LibraryCycleGraphLoader {
104104
(_idsToLoad[phase] ??= []).add(id);
105105
}
106106

107-
void _loadAllAtPhase(int phase, Iterable<AssetId> ids) {
107+
void _loadAllAtPhaseZero(Iterable<AssetId> ids) {
108108
if (ids.isEmpty) return;
109-
(_idsToLoad[phase] ??= []).addAll(ids);
109+
(_idsToLoad[0] ??= []).addAll(ids);
110110
}
111111

112112
/// Whether there are assets to load before or at [upToPhase].
@@ -144,17 +144,27 @@ class LibraryCycleGraphLoader {
144144
///
145145
/// Pass a phase and ID from [_nextIdToLoad].
146146
void _removeIdToLoad(int phase, AssetId id) {
147-
// A recursive load might have updated `_idsToLoad` since `_nextIdToLoad`
148-
// was called. If so it fully processed some phases: either `_idsToLoad` is
149-
// now empty at `phase`, in which case there is nothing to do, or it's
150-
// unchanged, in which case `id` is still the last ID.
151147
final ids = _idsToLoad[phase];
152-
if (ids != null) {
153-
if (ids.removeLast() != id) {
154-
throw StateError('$id should still be last in _idsToLoad[$phase]');
148+
149+
// It's possible that a recursive call to an earlier phase fully processed
150+
// the phase, leaving nothing to clean up.
151+
if (ids == null) {
152+
return;
153+
}
154+
155+
// It's possible that a recursive call to an earlier phase encountered a
156+
// reference to an asset generated at this phase, and so added another
157+
// asset to load in this phase. In that case `id` is no longer the last in
158+
// the list: search the whole list to remove it.
159+
if (ids.last == id) {
160+
ids.removeLast();
161+
} else {
162+
final removed = ids.remove(id);
163+
if (!removed) {
164+
throw StateError('Failed to remove $id from _idsToLoad: $_idsToLoad');
155165
}
156-
if (ids.isEmpty) _idsToLoad.remove(phase);
157166
}
167+
if (ids.isEmpty) _idsToLoad.remove(phase);
158168
}
159169

160170
/// Loads [id] and its transitive dependencies at all phases available to
@@ -221,7 +231,7 @@ class LibraryCycleGraphLoader {
221231
// for loading at any phase: if the `_load` that loads them is at a too
222232
// early phase to see generated output they will be queued for
223233
// processing by a later `_load`.
224-
_loadAllAtPhase(0, assetDeps.lastValue.deps);
234+
_loadAllAtPhaseZero(assetDeps.lastValue.deps);
225235
} else {
226236
// It's a generated source that has not yet been generated. Mark it for
227237
// loading later.

build/lib/src/library_cycle_graph/phased_asset_deps.dart

Lines changed: 6 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22
// for details. All rights reserved. Use of this source code is governed by a
33
// BSD-style license that can be found in the LICENSE file.
44

5-
import 'dart:math';
6-
75
import 'package:built_collection/built_collection.dart';
86
import 'package:built_value/built_value.dart';
97
import 'package:built_value/serializer.dart';
@@ -29,9 +27,6 @@ abstract class PhasedAssetDeps
2927
_$PhasedAssetDeps;
3028
PhasedAssetDeps._();
3129

32-
factory PhasedAssetDeps.of(Map<AssetId, PhasedValue<AssetDeps>> assetDeps) =>
33-
_$PhasedAssetDeps._(assetDeps: assetDeps.build());
34-
3530
/// Returns `this` data with [other] added to it.
3631
///
3732
/// For each asset: if [other] has a complete value for that asset, use the
@@ -53,20 +48,12 @@ abstract class PhasedAssetDeps
5348
return result.build();
5449
}
5550

56-
/// The max phase before there is any incomplete data, or 0xffffffff if there
57-
/// is no incomplete data.
58-
@memoized
59-
int get phase {
60-
int? result;
61-
for (final entry in assetDeps.values) {
62-
if (!entry.isComplete) {
63-
if (result == null) {
64-
result = entry.expiresAfter;
65-
} else {
66-
result = min(result, entry.expiresAfter!);
67-
}
51+
PhasedAssetDeps complete() => rebuild((b) {
52+
for (final entry in assetDeps.entries) {
53+
final value = entry.value;
54+
if (!value.isComplete) {
55+
b.assetDeps[entry.key] = PhasedValue.fixed(value.values.single.value);
6856
}
6957
}
70-
return result ?? 0xffffffff;
71-
}
58+
});
7259
}

build/lib/src/library_cycle_graph/phased_asset_deps.g.dart

Lines changed: 0 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.
Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import 'dart:math';
6+
7+
import 'package:build/build.dart' show AssetId;
8+
import 'package:test/test.dart';
9+
10+
import 'invalidation_tester.dart';
11+
12+
/// Tests correctness of invalidation over many randomly generated scenarios.
13+
///
14+
/// The test builders write output that is a list of files read/resolved and their
15+
/// content hashes. This does two things: it ensures that if any input changes,
16+
/// the output changes; and it makes it possible to determine what will
17+
/// invalidate that output.
18+
///
19+
/// In this way the test can know what output changes to assert due to a
20+
/// particular input change.
21+
Future<void> main() async {
22+
for (var iteration = 0; iteration != 500; ++iteration) {
23+
test('invalidation stress test $iteration', () async {
24+
final tester = InvalidationTester()..logSetup();
25+
final random = Random(iteration);
26+
27+
// Whether to change the imports in source files between builds.
28+
final changeImports = iteration % 10 == 0;
29+
30+
// How many checked in sources and how many builders; the build is
31+
// a rectangle of size numberOfSources x numberOfBuilders.
32+
final numberOfSources = random.nextInt(8) + 1;
33+
final numberOfBuilders = random.nextInt(4) + 1;
34+
35+
final sources = [
36+
for (var i = 0; i != numberOfSources; ++i) 'a${i + 1}.1',
37+
];
38+
39+
// Inputs that can be randomly read/resolved.
40+
// `numberOfBuilders + 2` to add some reads/resolves of files that
41+
// don't exist.
42+
final pickableInputs = <String>[];
43+
for (var i = 1; i != (numberOfBuilders + 2); ++i) {
44+
for (final source in sources) {
45+
pickableInputs.add(source.replaceAll('.1', '.$i'));
46+
}
47+
}
48+
49+
// All outputs.
50+
final outputs = <String>[];
51+
for (var i = 2; i != (numberOfBuilders + 1); ++i) {
52+
for (final source in sources) {
53+
outputs.add(source.replaceAll('.1', '.$i'));
54+
}
55+
}
56+
57+
// Picks a list of files to import from `pickableInputs`.
58+
List<String> randomImportList() {
59+
final result = <String>[];
60+
final length = random.nextInt(numberOfSources);
61+
for (var i = 0; i != length; ++i) {
62+
result.add(pickableInputs[random.nextInt(pickableInputs.length)]);
63+
}
64+
return result;
65+
}
66+
67+
tester
68+
..sources(sources)
69+
..pickableInputs(pickableInputs);
70+
71+
// Set up builders.
72+
for (var i = 1; i != numberOfBuilders; ++i) {
73+
tester.builder(
74+
from: '.$i',
75+
to: '.${i + 1}',
76+
// Cover optional+required builders.
77+
isOptional: random.nextBool(),
78+
// Cover builders with hidden+visible output.
79+
outputIsVisible: random.nextBool(),
80+
)
81+
// Use the input as the seed for what additional reads to do,
82+
// so reads won't change between identical runs.
83+
..readsForSeedThenReadsRandomly('.$i')
84+
..writes('.${i + 1}');
85+
}
86+
87+
// Initial random import graph.
88+
final importGraph = {
89+
for (var source in pickableInputs) source: randomImportList(),
90+
};
91+
tester.importGraph(importGraph);
92+
93+
// Initial build should succeed.
94+
expect((await tester.build()).succeeded, true);
95+
96+
// Do five additional builds making changes and checking what was written.
97+
for (var build = 0; build != 5; ++build) {
98+
// Pick which source to change, compute expected outputs.
99+
final sourceToChange = sources[random.nextInt(sources.length)];
100+
final expectedOutputs = <String>{};
101+
102+
Future<void> addExpectedOutputs(String invalidatedInput) async {
103+
for (final output in outputs) {
104+
final assetId = AssetId('pkg', 'lib/$output.dart');
105+
final hiddenAssetId = AssetId(
106+
'pkg',
107+
'.dart_tool/build/generated/pkg/lib/$output.dart',
108+
);
109+
final outputContents =
110+
tester.readerWriter!.testing.exists(assetId)
111+
? tester.readerWriter!.testing.readString(assetId)
112+
: tester.readerWriter!.testing.exists(hiddenAssetId)
113+
? tester.readerWriter!.testing.readString(hiddenAssetId)
114+
: '';
115+
// The test builder output is a list of "$name,$hash" for each input
116+
// that was read, including transitively resolved sources. Check it
117+
// for [input]. If found, this output is invalidated: recursively
118+
// add its invalidated outputs.
119+
if (outputContents.contains('$invalidatedInput.dart,')) {
120+
if (expectedOutputs.add(output)) {
121+
await addExpectedOutputs(output);
122+
}
123+
}
124+
}
125+
}
126+
127+
await addExpectedOutputs(sourceToChange);
128+
129+
// If [changeImports] then change some imports; it's only possible to
130+
// change imports for files that will be output.
131+
if (changeImports && expectedOutputs.isNotEmpty) {
132+
final sourceToChangeImports =
133+
expectedOutputs.toList()[random.nextInt(expectedOutputs.length)];
134+
importGraph[sourceToChangeImports] = randomImportList();
135+
tester.importGraph(importGraph);
136+
}
137+
138+
// Build and check exactly the expected outputs change.
139+
expect(
140+
await tester.build(change: sourceToChange),
141+
Result(written: expectedOutputs),
142+
);
143+
}
144+
});
145+
}
146+
}

0 commit comments

Comments
 (0)