Skip to content

Commit cd8606a

Browse files
onbjergiFrostizz
onbjerg
authored and
iFrostizz
committed
feat: snapshot fuzz tests using determin. seed (foundry-rs#2591)
1 parent e683241 commit cd8606a

File tree

5 files changed

+42
-55
lines changed

5 files changed

+42
-55
lines changed

‎cli/src/cmd/forge/coverage.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,7 @@ impl CoverageArgs {
283283
let local_identifier = LocalTraceIdentifier::new(&runner.known_contracts);
284284

285285
// TODO: Coverage for fuzz tests
286-
let handle = thread::spawn(move || runner.test(&self.filter, Some(tx), false).unwrap());
286+
let handle = thread::spawn(move || runner.test(&self.filter, Some(tx)).unwrap());
287287
for mut result in rx.into_iter().flat_map(|(_, suite)| suite.test_results.into_values()) {
288288
if let Some(hit_map) = result.coverage.take() {
289289
for (_, trace) in &mut result.traces {

‎cli/src/cmd/forge/snapshot.rs

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ use crate::cmd::{
88
Cmd,
99
};
1010
use clap::{Parser, ValueHint};
11+
use ethers::types::U256;
1112
use eyre::Context;
1213
use forge::result::TestKindGas;
1314
use once_cell::sync::Lazy;
@@ -30,6 +31,14 @@ pub static RE_BASIC_SNAPSHOT_ENTRY: Lazy<Regex> = Lazy::new(|| {
3031
Regex::new(r"(?P<file>(.*?)):(?P<sig>(\w+)\s*\((.*?)\))\s*\(((gas:)?\s*(?P<gas>\d+)|(runs:\s*(?P<runs>\d+),\s*μ:\s*(?P<avg>\d+),\s*~:\s*(?P<med>\d+)))\)").unwrap()
3132
});
3233

34+
/// Deterministic fuzzer seed used for gas snapshots.
35+
///
36+
/// The keccak256 hash of "foundry rulez"
37+
pub static SNAPSHOT_FUZZ_SEED: [u8; 32] = [
38+
0x01, 0x00, 0xfa, 0x69, 0xa5, 0xf1, 0x71, 0x0a, 0x95, 0xcd, 0xef, 0x94, 0x88, 0x9b, 0x02, 0x84,
39+
0x5d, 0x64, 0x0b, 0x19, 0xad, 0xf0, 0xe3, 0x57, 0xb8, 0xd4, 0xbe, 0x7d, 0x49, 0xee, 0x70, 0xe6,
40+
];
41+
3342
#[derive(Debug, Clone, Parser)]
3443
pub struct SnapshotArgs {
3544
/// All test arguments are supported
@@ -75,10 +84,6 @@ pub struct SnapshotArgs {
7584
value_name = "SNAPSHOT_FILE"
7685
)]
7786
snap: PathBuf,
78-
79-
/// Include the mean and median gas use of fuzz tests in the snapshot.
80-
#[clap(long, env = "FORGE_INCLUDE_FUZZ_TESTS")]
81-
pub include_fuzz_tests: bool,
8287
}
8388

8489
impl SnapshotArgs {
@@ -102,8 +107,11 @@ impl SnapshotArgs {
102107
impl Cmd for SnapshotArgs {
103108
type Output = ();
104109

105-
fn run(self) -> eyre::Result<()> {
106-
let outcome = custom_run(self.test, self.include_fuzz_tests)?;
110+
fn run(mut self) -> eyre::Result<()> {
111+
// Set fuzz seed so gas snapshots are deterministic
112+
self.test.fuzz_seed = Some(U256::from_big_endian(&SNAPSHOT_FUZZ_SEED));
113+
114+
let outcome = custom_run(self.test)?;
107115
outcome.ensure_ok()?;
108116
let tests = self.config.apply(outcome);
109117

‎cli/src/cmd/forge/test/mod.rs

Lines changed: 11 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,11 @@ pub struct TestArgs {
9696
#[clap(long, short, help_heading = "DISPLAY OPTIONS")]
9797
list: bool,
9898

99-
#[clap(long, help = "Set seed used to generate randomness during your fuzz runs", parse(try_from_str = utils::parse_u256))]
99+
#[clap(
100+
long,
101+
help = "Set seed used to generate randomness during your fuzz runs",
102+
parse(try_from_str = utils::parse_u256)
103+
)]
100104
pub fuzz_seed: Option<U256>,
101105
}
102106

@@ -159,7 +163,7 @@ impl Cmd for TestArgs {
159163

160164
fn run(self) -> eyre::Result<Self::Output> {
161165
trace!(target: "forge::test", "executing test command");
162-
custom_run(self, true)
166+
custom_run(self)
163167
}
164168
}
165169

@@ -295,7 +299,7 @@ fn short_test_result(name: &str, result: &TestResult) {
295299
println!("{} {} {}", status, name, result.kind.gas_used());
296300
}
297301

298-
pub fn custom_run(args: TestArgs, include_fuzz_tests: bool) -> eyre::Result<TestOutcome> {
302+
pub fn custom_run(args: TestArgs) -> eyre::Result<TestOutcome> {
299303
// Merge all configs
300304
let (config, mut evm_opts) = args.config_and_evm_opts()?;
301305

@@ -359,7 +363,7 @@ pub fn custom_run(args: TestArgs, include_fuzz_tests: bool) -> eyre::Result<Test
359363
match runner.count_filtered_tests(&filter) {
360364
1 => {
361365
// Run the test
362-
let results = runner.test(&filter, None, true)?;
366+
let results = runner.test(&filter, None)?;
363367

364368
// Get the result of the single test
365369
let (id, sig, test_kind, counterexample) = results.iter().map(|(id, SuiteResult{ test_results, .. })| {
@@ -403,16 +407,7 @@ pub fn custom_run(args: TestArgs, include_fuzz_tests: bool) -> eyre::Result<Test
403407
} else if args.list {
404408
list(runner, filter, args.json)
405409
} else {
406-
test(
407-
config,
408-
runner,
409-
verbosity,
410-
filter,
411-
args.json,
412-
args.allow_failure,
413-
include_fuzz_tests,
414-
args.gas_report,
415-
)
410+
test(config, runner, verbosity, filter, args.json, args.allow_failure, args.gas_report)
416411
}
417412
}
418413

@@ -443,7 +438,6 @@ fn test(
443438
filter: Filter,
444439
json: bool,
445440
allow_failure: bool,
446-
include_fuzz_tests: bool,
447441
gas_reporting: bool,
448442
) -> eyre::Result<TestOutcome> {
449443
trace!(target: "forge::test", "running all tests");
@@ -468,7 +462,7 @@ fn test(
468462
}
469463

470464
if json {
471-
let results = runner.test(&filter, None, include_fuzz_tests)?;
465+
let results = runner.test(&filter, None)?;
472466
println!("{}", serde_json::to_string(&results)?);
473467
Ok(TestOutcome::new(results, allow_failure))
474468
} else {
@@ -489,8 +483,7 @@ fn test(
489483
let (tx, rx) = channel::<(String, SuiteResult)>();
490484

491485
// Run tests
492-
let handle =
493-
thread::spawn(move || runner.test(&filter, Some(tx), include_fuzz_tests).unwrap());
486+
let handle = thread::spawn(move || runner.test(&filter, Some(tx)).unwrap());
494487

495488
let mut results: BTreeMap<String, SuiteResult> = BTreeMap::new();
496489
let mut gas_report = GasReport::new(config.gas_reports);

‎forge/src/multi_runner.rs

Lines changed: 11 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -119,10 +119,7 @@ impl MultiContractRunner {
119119
&mut self,
120120
filter: &impl TestFilter,
121121
stream_result: Option<Sender<(String, SuiteResult)>>,
122-
include_fuzz_tests: bool,
123122
) -> Result<BTreeMap<String, SuiteResult>> {
124-
tracing::info!(include_fuzz_tests= ?include_fuzz_tests, "running all tests");
125-
126123
let db = Backend::spawn(self.fork.take());
127124

128125
let results =
@@ -156,7 +153,7 @@ impl MultiContractRunner {
156153
executor,
157154
deploy_code.clone(),
158155
libs,
159-
(filter, include_fuzz_tests),
156+
filter
160157
)?;
161158

162159
tracing::trace!(contract= ?identifier, "executed all tests in contract");
@@ -190,7 +187,7 @@ impl MultiContractRunner {
190187
executor: Executor,
191188
deploy_code: Bytes,
192189
libs: &[Bytes],
193-
(filter, include_fuzz_tests): (&impl TestFilter, bool),
190+
filter: &impl TestFilter,
194191
) -> Result<SuiteResult> {
195192
let runner = ContractRunner::new(
196193
executor,
@@ -201,7 +198,7 @@ impl MultiContractRunner {
201198
self.errors.as_ref(),
202199
libs,
203200
);
204-
runner.run_tests(filter, self.fuzzer.clone(), include_fuzz_tests)
201+
runner.run_tests(filter, self.fuzzer.clone())
205202
}
206203
}
207204

@@ -519,7 +516,7 @@ mod tests {
519516
#[test]
520517
fn test_core() {
521518
let mut runner = runner();
522-
let results = runner.test(&Filter::new(".*", ".*", ".*core"), None, true).unwrap();
519+
let results = runner.test(&Filter::new(".*", ".*", ".*core"), None).unwrap();
523520

524521
assert_multiple(
525522
&results,
@@ -594,7 +591,7 @@ mod tests {
594591
#[test]
595592
fn test_logs() {
596593
let mut runner = runner();
597-
let results = runner.test(&Filter::new(".*", ".*", ".*logs"), None, true).unwrap();
594+
let results = runner.test(&Filter::new(".*", ".*", ".*logs"), None).unwrap();
598595

599596
assert_multiple(
600597
&results,
@@ -1157,7 +1154,7 @@ mod tests {
11571154

11581155
// test `setEnv` first, and confirm that it can correctly set environment variables,
11591156
// so that we can use it in subsequent `env*` tests
1160-
runner.test(&Filter::new("testSetEnv", ".*", ".*"), None, true).unwrap();
1157+
runner.test(&Filter::new("testSetEnv", ".*", ".*"), None).unwrap();
11611158
let env_var_key = "_foundryCheatcodeSetEnvTestKey";
11621159
let env_var_val = "_foundryCheatcodeSetEnvTestVal";
11631160
let res = env::var(env_var_key);
@@ -1182,7 +1179,6 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
11821179
&format!(".*cheats{}Fork", RE_PATH_SEPARATOR),
11831180
),
11841181
None,
1185-
true,
11861182
)
11871183
.unwrap();
11881184
assert_eq!(suite_result.len(), 1);
@@ -1206,7 +1202,6 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
12061202
&Filter::new(".*", ".*", &format!(".*cheats{}Fork", RE_PATH_SEPARATOR))
12071203
.exclude_tests(".*Revert"),
12081204
None,
1209-
true,
12101205
)
12111206
.unwrap();
12121207
assert!(!suite_result.is_empty());
@@ -1230,11 +1225,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
12301225
fn test_cheats_local() {
12311226
let mut runner = runner();
12321227
let suite_result = runner
1233-
.test(
1234-
&Filter::new(".*", ".*", &format!(".*cheats{}[^Fork]", RE_PATH_SEPARATOR)),
1235-
None,
1236-
true,
1237-
)
1228+
.test(&Filter::new(".*", ".*", &format!(".*cheats{}[^Fork]", RE_PATH_SEPARATOR)), None)
12381229
.unwrap();
12391230
assert!(!suite_result.is_empty());
12401231

@@ -1258,7 +1249,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
12581249
let cfg = proptest::test_runner::Config { failure_persistence: None, ..Default::default() };
12591250
runner.fuzzer = Some(proptest::test_runner::TestRunner::new(cfg));
12601251

1261-
let suite_result = runner.test(&Filter::new(".*", ".*", ".*fuzz"), None, true).unwrap();
1252+
let suite_result = runner.test(&Filter::new(".*", ".*", ".*fuzz"), None).unwrap();
12621253

12631254
assert!(!suite_result.is_empty());
12641255

@@ -1292,7 +1283,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
12921283
#[test]
12931284
fn test_trace() {
12941285
let mut runner = tracing_runner();
1295-
let suite_result = runner.test(&Filter::new(".*", ".*", ".*trace"), None, true).unwrap();
1286+
let suite_result = runner.test(&Filter::new(".*", ".*", ".*trace"), None).unwrap();
12961287

12971288
// TODO: This trace test is very basic - it is probably a good candidate for snapshot
12981289
// testing.
@@ -1330,7 +1321,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
13301321
fn test_fork() {
13311322
let rpc_url = foundry_utils::rpc::next_http_archive_rpc_endpoint();
13321323
let mut runner = forked_runner(&rpc_url);
1333-
let suite_result = runner.test(&Filter::new(".*", ".*", ".*fork"), None, true).unwrap();
1324+
let suite_result = runner.test(&Filter::new(".*", ".*", ".*fork"), None).unwrap();
13341325

13351326
for (_, SuiteResult { test_results, .. }) in suite_result {
13361327
for (test_name, result) in test_results {
@@ -1351,7 +1342,7 @@ Reason: `setEnv` failed to set an environment variable `{}={}`",
13511342
fn test_doesnt_run_abstract_contract() {
13521343
let mut runner = runner();
13531344
let results = runner
1354-
.test(&Filter::new(".*", ".*", ".*Abstract.t.sol".to_string().as_str()), None, true)
1345+
.test(&Filter::new(".*", ".*", ".*Abstract.t.sol".to_string().as_str()), None)
13551346
.unwrap();
13561347
assert!(results.get("core/Abstract.t.sol:AbstractTestBase").is_none());
13571348
assert!(results.get("core/Abstract.t.sol:AbstractTest").is_some());

‎forge/src/runner.rs

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,6 @@ impl<'a> ContractRunner<'a> {
176176
mut self,
177177
filter: &impl TestFilter,
178178
fuzzer: Option<TestRunner>,
179-
include_fuzz_tests: bool,
180179
) -> Result<SuiteResult> {
181180
tracing::info!("starting tests");
182181
let start = Instant::now();
@@ -247,23 +246,19 @@ impl<'a> ContractRunner<'a> {
247246
.contract
248247
.functions()
249248
.into_iter()
250-
.filter(|func| {
251-
func.name.is_test() &&
252-
filter.matches_test(func.signature()) &&
253-
(include_fuzz_tests || func.inputs.is_empty())
254-
})
255-
.map(|func| (func, func.name.is_test_fail()))
249+
.filter(|func| func.is_test() && filter.matches_test(func.signature()))
250+
.map(|func| (func, func.is_test_fail()))
256251
.collect();
257252

258253
let test_results = tests
259254
.par_iter()
260255
.filter_map(|(func, should_fail)| {
261-
let result = if func.inputs.is_empty() {
262-
Some(self.clone().run_test(func, *should_fail, setup.clone()))
263-
} else {
256+
let result = if func.is_fuzz_test() {
264257
fuzzer.as_ref().map(|fuzzer| {
265258
self.run_fuzz_test(func, *should_fail, fuzzer.clone(), setup.clone())
266259
})
260+
} else {
261+
Some(self.clone().run_test(func, *should_fail, setup.clone()))
267262
};
268263

269264
result.map(|result| Ok((func.signature(), result?)))

0 commit comments

Comments
 (0)