From f37feba232f4feb764ac7bef0f14f6dfe3cacfb5 Mon Sep 17 00:00:00 2001 From: Otto Date: Thu, 2 Nov 2023 08:44:50 +0100 Subject: [PATCH 1/6] Use criterion for `cargo bench` locally, use iai in CI --- .github/workflows/benchmark.yaml | 4 +- Cargo.toml | 2 + benches/my_benchmark.rs | 86 ++++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 68ee71e9..27838522 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -53,11 +53,11 @@ jobs: - name: Revert when benches do not compile on main run: cargo check --benches --no-default-features || git checkout main -- benches/my_benchmark.rs - name: Run benchmarks for main branch - run: cargo bench --no-default-features + run: cargo bench --no-default-features --feature iai - name: Checkout PR branch run: git checkout - - name: Run bench against baseline - run: cargo bench --no-default-features | sed '0,/^test result:/d' | tee bench.txt + run: cargo bench --no-default-features --feature iai | sed '0,/^test result:/d' | tee bench.txt # for testing # - name: create mock results diff --git a/Cargo.toml b/Cargo.toml index 06e0349a..b842576a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ vecmath = "1.0" [dev-dependencies] alloc_counter = "0.0.4" +criterion = "0.5.1" env_logger = "0.10" iai = "0.1.1" rand = "0.8" @@ -62,3 +63,4 @@ cpal = ["dep:cpal"] cubeb = ["dep:cubeb"] cpal-jack = ["cpal", "cpal/jack"] cpal-asio = ["cpal", "cpal/asio"] +iai = [] diff --git a/benches/my_benchmark.rs b/benches/my_benchmark.rs index 7d866fe7..a5185a40 100644 --- a/benches/my_benchmark.rs +++ b/benches/my_benchmark.rs @@ -1,5 +1,9 @@ +#[cfg(feature = "iai")] use iai::black_box; +#[cfg(not(feature = "iai"))] +use criterion::{black_box, criterion_group, criterion_main, Criterion}; + use web_audio_api::context::BaseAudioContext; use web_audio_api::context::OfflineAudioContext; use web_audio_api::node::{AudioNode, AudioScheduledSourceNode, PanningModelType}; @@ -217,6 +221,7 @@ pub fn bench_hrtf_panners() { assert_eq!(ctx.start_rendering_sync().length(), SAMPLES); } +#[cfg(feature = "iai")] iai::main!( bench_ctor, bench_sine, @@ -231,3 +236,84 @@ iai::main!( bench_analyser_node, bench_hrtf_panners, ); + +#[cfg(not(feature = "iai"))] +fn criterion_ctor(c: &mut Criterion) { + c.bench_function("bench_ctor", |b| b.iter(|| bench_ctor())); +} +#[cfg(not(feature = "iai"))] +fn criterion_sine(c: &mut Criterion) { + c.bench_function("bench_sine", |b| b.iter(|| bench_sine())); +} +#[cfg(not(feature = "iai"))] +fn criterion_sine_gain(c: &mut Criterion) { + c.bench_function("bench_sine_gain", |b| b.iter(|| bench_sine_gain())); +} +#[cfg(not(feature = "iai"))] +fn criterion_sine_gain_delay(c: &mut Criterion) { + c.bench_function("bench_sine_gain_delay", |b| { + b.iter(|| bench_sine_gain_delay()) + }); +} +#[cfg(not(feature = "iai"))] +fn criterion_buffer_src(c: &mut Criterion) { + c.bench_function("bench_buffer_src", |b| b.iter(|| bench_buffer_src())); +} +#[cfg(not(feature = "iai"))] +fn criterion_buffer_src_delay(c: &mut Criterion) { + c.bench_function("bench_buffer_src_delay", |b| { + b.iter(|| bench_buffer_src_delay()) + }); +} +#[cfg(not(feature = "iai"))] +fn criterion_buffer_src_iir(c: &mut Criterion) { + c.bench_function("bench_buffer_src_iir", |b| { + b.iter(|| bench_buffer_src_iir()) + }); +} +#[cfg(not(feature = "iai"))] +fn criterion_buffer_src_biquad(c: &mut Criterion) { + c.bench_function("bench_buffer_src_biquad", |b| { + b.iter(|| bench_buffer_src_biquad()) + }); +} +#[cfg(not(feature = "iai"))] +fn criterion_stereo_positional(c: &mut Criterion) { + c.bench_function("bench_stereo_positional", |b| { + b.iter(|| bench_stereo_positional()) + }); +} +#[cfg(not(feature = "iai"))] +fn criterion_stereo_panning_automation(c: &mut Criterion) { + c.bench_function("bench_stereo_panning_automation", |b| { + b.iter(|| bench_stereo_panning_automation()) + }); +} +#[cfg(not(feature = "iai"))] +fn criterion_analyser_node(c: &mut Criterion) { + c.bench_function("bench_analyser_node", |b| b.iter(|| bench_analyser_node())); +} +#[cfg(not(feature = "iai"))] +fn criterion_hrtf_panners(c: &mut Criterion) { + c.bench_function("bench_hrtf_panners", |b| b.iter(|| bench_hrtf_panners())); +} + +#[cfg(not(feature = "iai"))] +criterion_group!( + benches, + criterion_ctor, + criterion_sine, + criterion_sine_gain, + criterion_sine_gain_delay, + criterion_buffer_src, + criterion_buffer_src_delay, + criterion_buffer_src_iir, + criterion_buffer_src_biquad, + criterion_stereo_positional, + criterion_stereo_panning_automation, + criterion_analyser_node, + criterion_hrtf_panners +); + +#[cfg(not(feature = "iai"))] +criterion_main!(benches); From caae9615a8e060c4e82a1aa81b6d3f870dc4e9d8 Mon Sep 17 00:00:00 2001 From: Otto Date: Thu, 2 Nov 2023 09:08:56 +0100 Subject: [PATCH 2/6] Make sure we are not benching IO in the criterion suite --- benches/my_benchmark.rs | 59 ++++++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/benches/my_benchmark.rs b/benches/my_benchmark.rs index a5185a40..a1afac31 100644 --- a/benches/my_benchmark.rs +++ b/benches/my_benchmark.rs @@ -7,16 +7,44 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use web_audio_api::context::BaseAudioContext; use web_audio_api::context::OfflineAudioContext; use web_audio_api::node::{AudioNode, AudioScheduledSourceNode, PanningModelType}; +use web_audio_api::AudioBuffer; + +use std::fs::File; +use std::sync::OnceLock; const SAMPLE_RATE: f32 = 48000.; const DURATION: usize = 10; const SAMPLES: usize = SAMPLE_RATE as usize * DURATION; +/// Load an audio buffer and cache the result +/// +/// We don't want to measure the IO and decoding in most of our benchmarks, so by using this static +/// instance we avoid this in the criterion benchmarks because the file is already loaded in the +/// warmup phase. +fn get_audio_buffer(ctx: &OfflineAudioContext) -> AudioBuffer { + static BUFFER: OnceLock = OnceLock::new(); + BUFFER + .get_or_init(|| { + println!("decoding now"); + let file = File::open("samples/think-stereo-48000.wav").unwrap(); + ctx.decode_audio_data_sync(file).unwrap() + }) + .clone() +} + pub fn bench_ctor() { let ctx = OfflineAudioContext::new(2, black_box(SAMPLES), SAMPLE_RATE); assert_eq!(ctx.start_rendering_sync().length(), SAMPLES); } +// This benchmark only makes sense in `iai`, because subsequent runs use the cached audiobuffer. +// However we would like to run this test here so the cache is filled for the subsequent benches. +pub fn bench_audio_buffer_decode() { + let ctx = OfflineAudioContext::new(2, black_box(SAMPLES), SAMPLE_RATE); + let buffer = get_audio_buffer(&ctx); + assert_eq!(buffer.length(), 101129); +} + pub fn bench_sine() { let ctx = OfflineAudioContext::new(2, black_box(SAMPLES), SAMPLE_RATE); let mut osc = ctx.create_oscillator(); @@ -60,9 +88,7 @@ pub fn bench_sine_gain_delay() { pub fn bench_buffer_src() { let ctx = OfflineAudioContext::new(2, black_box(SAMPLES), SAMPLE_RATE); - - let file = std::fs::File::open("samples/think-stereo-48000.wav").unwrap(); - let buffer = ctx.decode_audio_data_sync(file).unwrap(); + let buffer = get_audio_buffer(&ctx); let mut src = ctx.create_buffer_source(); src.connect(&ctx.destination()); @@ -74,9 +100,7 @@ pub fn bench_buffer_src() { pub fn bench_buffer_src_delay() { let ctx = OfflineAudioContext::new(2, black_box(SAMPLES), SAMPLE_RATE); - - let file = std::fs::File::open("samples/think-stereo-48000.wav").unwrap(); - let buffer = ctx.decode_audio_data_sync(file).unwrap(); + let buffer = get_audio_buffer(&ctx); let delay = ctx.create_delay(0.3); delay.delay_time().set_value(0.2); @@ -93,8 +117,7 @@ pub fn bench_buffer_src_delay() { pub fn bench_buffer_src_iir() { let ctx = OfflineAudioContext::new(2, black_box(SAMPLES), SAMPLE_RATE); - let file = std::fs::File::open("samples/think-stereo-48000.wav").unwrap(); - let buffer = ctx.decode_audio_data_sync(file).unwrap(); + let buffer = get_audio_buffer(&ctx); // these values correspond to a lowpass filter at 200Hz (calculated from biquad) let feedforward = vec![ @@ -120,8 +143,7 @@ pub fn bench_buffer_src_iir() { pub fn bench_buffer_src_biquad() { let ctx = OfflineAudioContext::new(2, black_box(SAMPLES), SAMPLE_RATE); - let file = std::fs::File::open("samples/think-stereo-48000.wav").unwrap(); - let buffer = ctx.decode_audio_data_sync(file).unwrap(); + let buffer = get_audio_buffer(&ctx); // Create an biquad filter node (defaults to low pass) let biquad = ctx.create_biquad_filter(); @@ -139,8 +161,7 @@ pub fn bench_buffer_src_biquad() { pub fn bench_stereo_positional() { let ctx = OfflineAudioContext::new(2, black_box(SAMPLES), SAMPLE_RATE); - let file = std::fs::File::open("samples/think-stereo-48000.wav").unwrap(); - let buffer = ctx.decode_audio_data_sync(file).unwrap(); + let buffer = get_audio_buffer(&ctx); // Create static panner node let panner = ctx.create_panner(); @@ -163,8 +184,7 @@ pub fn bench_stereo_positional() { pub fn bench_stereo_panning_automation() { let ctx = OfflineAudioContext::new(2, black_box(SAMPLES), SAMPLE_RATE); - let file = std::fs::File::open("samples/think-stereo-48000.wav").unwrap(); - let buffer = ctx.decode_audio_data_sync(file).unwrap(); + let buffer = get_audio_buffer(&ctx); let panner = ctx.create_stereo_panner(); panner.connect(&ctx.destination()); @@ -185,8 +205,7 @@ pub fn bench_stereo_panning_automation() { // benchmark this in deterministic way [citation needed]. pub fn bench_analyser_node() { let ctx = OfflineAudioContext::new(2, black_box(SAMPLES), SAMPLE_RATE); - let file = std::fs::File::open("samples/think-stereo-48000.wav").unwrap(); - let buffer = ctx.decode_audio_data_sync(file).unwrap(); + let buffer = get_audio_buffer(&ctx); let analyser = ctx.create_analyser(); analyser.connect(&ctx.destination()); @@ -224,6 +243,7 @@ pub fn bench_hrtf_panners() { #[cfg(feature = "iai")] iai::main!( bench_ctor, + bench_audio_buffer_decode, bench_sine, bench_sine_gain, bench_sine_gain_delay, @@ -242,6 +262,12 @@ fn criterion_ctor(c: &mut Criterion) { c.bench_function("bench_ctor", |b| b.iter(|| bench_ctor())); } #[cfg(not(feature = "iai"))] +fn criterion_audio_buffer_decode(c: &mut Criterion) { + c.bench_function("bench_audio_buffer_decode", |b| { + b.iter(|| bench_audio_buffer_decode()) + }); +} +#[cfg(not(feature = "iai"))] fn criterion_sine(c: &mut Criterion) { c.bench_function("bench_sine", |b| b.iter(|| bench_sine())); } @@ -302,6 +328,7 @@ fn criterion_hrtf_panners(c: &mut Criterion) { criterion_group!( benches, criterion_ctor, + criterion_audio_buffer_decode, criterion_sine, criterion_sine_gain, criterion_sine_gain_delay, From 4d179b0d92d509aae6b6457992bd69baa94935a2 Mon Sep 17 00:00:00 2001 From: Otto Date: Thu, 2 Nov 2023 09:11:31 +0100 Subject: [PATCH 3/6] Make HRTF benchmark less heavy by rendering less samples --- benches/my_benchmark.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/benches/my_benchmark.rs b/benches/my_benchmark.rs index a1afac31..7319057d 100644 --- a/benches/my_benchmark.rs +++ b/benches/my_benchmark.rs @@ -15,6 +15,7 @@ use std::sync::OnceLock; const SAMPLE_RATE: f32 = 48000.; const DURATION: usize = 10; const SAMPLES: usize = SAMPLE_RATE as usize * DURATION; +const SAMPLES_SHORT: usize = SAMPLE_RATE as usize; // only 1 second for heavy benchmarks /// Load an audio buffer and cache the result /// @@ -220,7 +221,7 @@ pub fn bench_analyser_node() { } pub fn bench_hrtf_panners() { - let ctx = OfflineAudioContext::new(2, black_box(SAMPLES), SAMPLE_RATE); + let ctx = OfflineAudioContext::new(2, black_box(SAMPLES_SHORT), SAMPLE_RATE); let mut panner1 = ctx.create_panner(); panner1.set_panning_model(PanningModelType::HRTF); @@ -237,7 +238,7 @@ pub fn bench_hrtf_panners() { osc.connect(&panner2); osc.start(); - assert_eq!(ctx.start_rendering_sync().length(), SAMPLES); + assert_eq!(ctx.start_rendering_sync().length(), SAMPLES_SHORT); } #[cfg(feature = "iai")] From fa7af30de077fbe40679af3fd1865c03d14bde0a Mon Sep 17 00:00:00 2001 From: Otto Date: Thu, 2 Nov 2023 09:14:23 +0100 Subject: [PATCH 4/6] Remove debugging statement --- benches/my_benchmark.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/benches/my_benchmark.rs b/benches/my_benchmark.rs index 7319057d..fa94be59 100644 --- a/benches/my_benchmark.rs +++ b/benches/my_benchmark.rs @@ -26,7 +26,6 @@ fn get_audio_buffer(ctx: &OfflineAudioContext) -> AudioBuffer { static BUFFER: OnceLock = OnceLock::new(); BUFFER .get_or_init(|| { - println!("decoding now"); let file = File::open("samples/think-stereo-48000.wav").unwrap(); ctx.decode_audio_data_sync(file).unwrap() }) From 2a399760765c0d1b838b94d040629a56a7590cb8 Mon Sep 17 00:00:00 2001 From: Otto Date: Thu, 2 Nov 2023 09:18:11 +0100 Subject: [PATCH 5/6] Clippy fixes --- benches/my_benchmark.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/benches/my_benchmark.rs b/benches/my_benchmark.rs index fa94be59..351d942e 100644 --- a/benches/my_benchmark.rs +++ b/benches/my_benchmark.rs @@ -259,69 +259,69 @@ iai::main!( #[cfg(not(feature = "iai"))] fn criterion_ctor(c: &mut Criterion) { - c.bench_function("bench_ctor", |b| b.iter(|| bench_ctor())); + c.bench_function("bench_ctor", |b| b.iter(bench_ctor)); } #[cfg(not(feature = "iai"))] fn criterion_audio_buffer_decode(c: &mut Criterion) { c.bench_function("bench_audio_buffer_decode", |b| { - b.iter(|| bench_audio_buffer_decode()) + b.iter(bench_audio_buffer_decode) }); } #[cfg(not(feature = "iai"))] fn criterion_sine(c: &mut Criterion) { - c.bench_function("bench_sine", |b| b.iter(|| bench_sine())); + c.bench_function("bench_sine", |b| b.iter(bench_sine)); } #[cfg(not(feature = "iai"))] fn criterion_sine_gain(c: &mut Criterion) { - c.bench_function("bench_sine_gain", |b| b.iter(|| bench_sine_gain())); + c.bench_function("bench_sine_gain", |b| b.iter(bench_sine_gain)); } #[cfg(not(feature = "iai"))] fn criterion_sine_gain_delay(c: &mut Criterion) { c.bench_function("bench_sine_gain_delay", |b| { - b.iter(|| bench_sine_gain_delay()) + b.iter(bench_sine_gain_delay) }); } #[cfg(not(feature = "iai"))] fn criterion_buffer_src(c: &mut Criterion) { - c.bench_function("bench_buffer_src", |b| b.iter(|| bench_buffer_src())); + c.bench_function("bench_buffer_src", |b| b.iter(bench_buffer_src)); } #[cfg(not(feature = "iai"))] fn criterion_buffer_src_delay(c: &mut Criterion) { c.bench_function("bench_buffer_src_delay", |b| { - b.iter(|| bench_buffer_src_delay()) + b.iter(bench_buffer_src_delay) }); } #[cfg(not(feature = "iai"))] fn criterion_buffer_src_iir(c: &mut Criterion) { c.bench_function("bench_buffer_src_iir", |b| { - b.iter(|| bench_buffer_src_iir()) + b.iter(bench_buffer_src_iir) }); } #[cfg(not(feature = "iai"))] fn criterion_buffer_src_biquad(c: &mut Criterion) { c.bench_function("bench_buffer_src_biquad", |b| { - b.iter(|| bench_buffer_src_biquad()) + b.iter(bench_buffer_src_biquad) }); } #[cfg(not(feature = "iai"))] fn criterion_stereo_positional(c: &mut Criterion) { c.bench_function("bench_stereo_positional", |b| { - b.iter(|| bench_stereo_positional()) + b.iter(bench_stereo_positional) }); } #[cfg(not(feature = "iai"))] fn criterion_stereo_panning_automation(c: &mut Criterion) { c.bench_function("bench_stereo_panning_automation", |b| { - b.iter(|| bench_stereo_panning_automation()) + b.iter(bench_stereo_panning_automation) }); } #[cfg(not(feature = "iai"))] fn criterion_analyser_node(c: &mut Criterion) { - c.bench_function("bench_analyser_node", |b| b.iter(|| bench_analyser_node())); + c.bench_function("bench_analyser_node", |b| b.iter(bench_analyser_node)); } #[cfg(not(feature = "iai"))] fn criterion_hrtf_panners(c: &mut Criterion) { - c.bench_function("bench_hrtf_panners", |b| b.iter(|| bench_hrtf_panners())); + c.bench_function("bench_hrtf_panners", |b| b.iter(bench_hrtf_panners)); } #[cfg(not(feature = "iai"))] From 08cd6aebfd65b8fe81664d8ca063bc29d07c616f Mon Sep 17 00:00:00 2001 From: Otto Date: Thu, 2 Nov 2023 09:18:32 +0100 Subject: [PATCH 6/6] cargo fmt --- benches/my_benchmark.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/benches/my_benchmark.rs b/benches/my_benchmark.rs index 351d942e..82a11d31 100644 --- a/benches/my_benchmark.rs +++ b/benches/my_benchmark.rs @@ -277,9 +277,7 @@ fn criterion_sine_gain(c: &mut Criterion) { } #[cfg(not(feature = "iai"))] fn criterion_sine_gain_delay(c: &mut Criterion) { - c.bench_function("bench_sine_gain_delay", |b| { - b.iter(bench_sine_gain_delay) - }); + c.bench_function("bench_sine_gain_delay", |b| b.iter(bench_sine_gain_delay)); } #[cfg(not(feature = "iai"))] fn criterion_buffer_src(c: &mut Criterion) { @@ -287,15 +285,11 @@ fn criterion_buffer_src(c: &mut Criterion) { } #[cfg(not(feature = "iai"))] fn criterion_buffer_src_delay(c: &mut Criterion) { - c.bench_function("bench_buffer_src_delay", |b| { - b.iter(bench_buffer_src_delay) - }); + c.bench_function("bench_buffer_src_delay", |b| b.iter(bench_buffer_src_delay)); } #[cfg(not(feature = "iai"))] fn criterion_buffer_src_iir(c: &mut Criterion) { - c.bench_function("bench_buffer_src_iir", |b| { - b.iter(bench_buffer_src_iir) - }); + c.bench_function("bench_buffer_src_iir", |b| b.iter(bench_buffer_src_iir)); } #[cfg(not(feature = "iai"))] fn criterion_buffer_src_biquad(c: &mut Criterion) {