diff --git a/README.md b/README.md index d5a0bf4..eb83e81 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Currently supported sources: ## Metadata format -The benchmark name in the `cargo` benchmarks can be provided as a `,`-separated string where each item represents a key-value pair with the format `key=value`, e.g. `category=ML-KEM,keySize=44,name=PK Validation,platform=neon,api=unpacked`. +The benchmark name in the `cargo` benchmarks can be provided as a `,`-separated string where each item represents a key-value pair with the format `key=value`, e.g. `category=ML-KEM,keySize=44,name=PK Validation,platform=neon,api=unpacked`. Additionally, one of these separators may be a `/`, where the left-hand side represents the keys configured for a Criterion group, and the right-hand side represents the keys for a benchmark function on that group, e.g. `category=ML-KEM,keySize=44,name=PK Validation/platform=neon,api=unpacked`. Aside from this separator, no other `/` characters should be included in the name. Alternatively, the benchmark name can be provided as a simple string, which will then be set as the value of the `name` key. This will only happen if there are no `key=value` pairs found in the benchmark name. diff --git a/scripts/prepare-release.sh b/scripts/prepare-release.sh index f5e5cc2..bfbd51c 100755 --- a/scripts/prepare-release.sh +++ b/scripts/prepare-release.sh @@ -50,7 +50,7 @@ cp -R node_modules .release/node_modules git checkout "$version" git pull -git rm -rf node_modules +git rm -rf --ignore-unmatch node_modules rm -rf node_modules # remove node_modules/.cache rm -rf dist diff --git a/src/extract.ts b/src/extract.ts index ffd1b27..3eea51e 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -14,8 +14,14 @@ export interface BenchmarkResult { } function addNameMetadataToResult(result: BenchmarkResult, nameString: string) { - // split by separator - const keyValuePairs = nameString.split(','); + // replace first '/' + // only one is allowed, as a Criterion group/function separator + const nameStringProcessed = nameString.replace('/', ','); + + if (nameStringProcessed.includes('/')) { + throw new Error("Only one '/' is allowed as a group/function separator"); + } + const keyValuePairs = nameStringProcessed.split(','); let foundAtLeastOne = false; diff --git a/test/data/extract/criterion_invalid.txt b/test/data/extract/criterion_invalid.txt new file mode 100644 index 0000000..d9d3562 --- /dev/null +++ b/test/data/extract/criterion_invalid.txt @@ -0,0 +1,14 @@ +WARNING: HTML report generation will become a non-default optional feature in Criterion.rs 0.4.0. +This feature is being moved to cargo-criterion (https://github.com/bheisler/cargo-criterion) and will be optional in a future version of Criterion.rs. To silence this warning, either switch to cargo-criterion or enable the 'html_reports' feature in your Cargo.toml. + +Gnuplot not found, using plotters backend +test category=ML-KEM,keySize=1024,name=PK Validation ... bench: 329 ns/iter (+/- 4) + +test category=ML-KEM,keySize=512/name=PK Validation/platform=neon,api=unpacked ... bench: 3268 ns/iter (+/- 47) + +test category=ML-KEM,keySize=768,name=PK Validation/platform=neon,api=unpacked ... bench: 12314 ns/iter (+/- 123) + + +Warning: Unable to complete 100 samples in 5.0s. You may wish to increase target time to 8.5s, enable flat sampling, or reduce sample count to 50. +test category=ML-KEM,keySize=512,name=PK Validation,platform=neon,api=unpacked (external random) ... bench: 1672496 ns/iter (+/- 10166) + diff --git a/test/data/extract/criterion_output_with_metadata_format.txt b/test/data/extract/criterion_output_with_metadata_format.txt index b314ec7..ce6bbae 100644 --- a/test/data/extract/criterion_output_with_metadata_format.txt +++ b/test/data/extract/criterion_output_with_metadata_format.txt @@ -4,9 +4,9 @@ This feature is being moved to cargo-criterion (https://github.com/bheisler/carg Gnuplot not found, using plotters backend test category=ML-KEM,keySize=1024,name=PK Validation ... bench: 329 ns/iter (+/- 4) -test category=ML-KEM,keySize=512,name=PK Validation,platform=neon,api=unpacked ... bench: 3268 ns/iter (+/- 47) +test category=ML-KEM,keySize=512,name=PK Validation/platform=neon,api=unpacked ... bench: 3268 ns/iter (+/- 47) -test category=ML-KEM,keySize=768,name=PK Validation,platform=neon,api=unpacked ... bench: 12314 ns/iter (+/- 123) +test category=ML-KEM,keySize=768,name=PK Validation/platform=neon,api=unpacked ... bench: 12314 ns/iter (+/- 123) Warning: Unable to complete 100 samples in 5.0s. You may wish to increase target time to 8.5s, enable flat sampling, or reduce sample count to 50. diff --git a/test/extract.spec.ts b/test/extract.spec.ts index 80046ca..e85092e 100644 --- a/test/extract.spec.ts +++ b/test/extract.spec.ts @@ -82,6 +82,15 @@ describe('extractData()', function () { } as Config; await A.rejects(extractData(config), /^Error: No benchmark result was found in /); }); + + it('raises an error when more than one / separator', async function () { + const config = { + os: 'ubuntu-latest', + tool: 'cargo', + outputFilePath: path.join(__dirname, 'data', 'extract', 'criterion_invalid.txt'), + } as Config; + await A.rejects(extractData(config), /^Error: Only one '\/' is allowed as a group\/function separator/); + }); }); describe('localWriteBenchmark()', function () {