diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 7887156..2b557c5 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -19,24 +19,24 @@ jobs: - uses: ./ # local action in repo with: tool: 'cargo' - platform: 'default' + os: 'ubuntu-latest' output-file-path: ./test/data/extract/cargo_output.txt data-out-path: output.txt - uses: ./ # local action in repo with: tool: 'cargo' - platform: 'default' + os: 'ubuntu-latest' output-file-path: ./test/data/extract/cargo_output2.txt data-out-path: output.txt - uses: ./ # local action in repo with: tool: 'cargo' - platform: 'default' + os: 'ubuntu-latest' output-file-path: ./test/data/extract/cargo_output3.txt data-out-path: output.txt - uses: ./ # local action in repo with: tool: 'cargo' - platform: 'default' + os: 'ubuntu-latest' output-file-path: ./test/data/extract/criterion_output.txt data-out-path: output.txt diff --git a/.github/workflows/minimal.yml b/.github/workflows/minimal.yml index 5fe3060..0099b65 100644 --- a/.github/workflows/minimal.yml +++ b/.github/workflows/minimal.yml @@ -16,6 +16,6 @@ jobs: - uses: ./ # local action in repo with: tool: 'cargo' - platform: 'default' + os: 'ubuntu-latest' output-file-path: ./test/data/extract/cargo_output.txt data-out-path: output.txt diff --git a/README.md b/README.md index dfcc1df..d91a0d7 100644 --- a/README.md +++ b/README.md @@ -9,21 +9,29 @@ Currently supported sources: ## Fields - `name` (required): name of the benchmark - `tool` (required): tool used to get benchmark output. One of `["cargo"]` -- `platform` (required): a string describing the platform +- `os` (required): a string describing the os - `output-file-path` (required): a path to a file containing the output of the benchmark tool - `data-out-path` (required): the path where the output of the action should be written +## Metadata format + +The benchmark name in the `cargo` benchmarks can be provided as a `/`-separated string with the format `category/key size/name/platform/api`. The key size should be an integer. Some fields in this string can be left blank. Any unspecified or invalid fields will be parsed to `undefined`. + ## Output data format The output will be written to `data-out-path` in a standardized JSON format: ```json [ - { - "name": "My Custom Smaller Is Better Benchmark - Memory Used", - "unit": "Megabytes", - "platform": "ubuntu-latest", - "value": 100, - "range": "3", - } + { + "api": "unpacked", + "category": "ML-KEM", + "keySize": 768, + "name": "PK Validation", + "os": "ubuntu-latest", + "platform": "neon", + "range": "± 123", + "unit": "ns/iter", + "value": 12314, + }, ] ``` diff --git a/action-types.yml b/action-types.yml index de67f57..769eaa2 100644 --- a/action-types.yml +++ b/action-types.yml @@ -5,7 +5,7 @@ inputs: type: enum allowed-values: - cargo - platform: + os: type: string output-file-path: type: string diff --git a/action.yml b/action.yml index 3b348ef..2a95b3f 100644 --- a/action.yml +++ b/action.yml @@ -13,8 +13,8 @@ inputs: tool: description: 'Tool to use to get benchmark output. One of "cargo"...' required: true - platform: - description: 'A string describing the platform' + os: + description: 'A string describing the os' required: true output-file-path: description: 'A path to file which contains the benchmark output' diff --git a/src/config.ts b/src/config.ts index 54f0ee7..474888f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -6,7 +6,7 @@ import * as path from 'path'; export type ToolType = typeof VALID_TOOLS[number]; export interface Config { name: string; - platform: string; + os: string; tool: ToolType; outputFilePath: string; dataOutPath: string; @@ -71,11 +71,11 @@ function validateName(name: string) { } throw new Error('Name must not be empty'); } -function validatePlatform(platform: string) { - if (platform) { +function validateOs(os: string) { + if (os) { return; } - throw new Error('Platform must not be empty'); + throw new Error('Os must not be empty'); } export async function configFromJobInput(): Promise { @@ -83,10 +83,10 @@ export async function configFromJobInput(): Promise { let outputFilePath: string = core.getInput('output-file-path'); let dataOutPath: string = core.getInput('data-out-path'); const name: string = core.getInput('name'); - const platform: string = core.getInput('platform'); + const os: string = core.getInput('os'); validateName(name); - validatePlatform(platform); + validateOs(os); validateToolType(tool); outputFilePath = await validateOutputFilePath(outputFilePath); dataOutPath = await validateDataOutPath(dataOutPath); @@ -96,6 +96,6 @@ export async function configFromJobInput(): Promise { tool, outputFilePath, dataOutPath, - platform, + os, }; } diff --git a/src/extract.ts b/src/extract.ts index fcda046..d163699 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -3,12 +3,61 @@ import { promises as fs } from 'fs'; import { Config } from './config'; export interface BenchmarkResult { - name: string; value: number; range?: string; unit: string; extra?: string; - platform: string; + os: string; + + // from NameMetadata + category: string | undefined; + keySize: number | undefined; + name: string; + platform: string | undefined; + api: string | undefined; +} + +interface NameMetadata { + category: string | undefined; + keySize: number | undefined; + name: string; + platform: string | undefined; + api: string | undefined; +} +function extractMetadataFromName(name_string: string): NameMetadata { + // split by separator + const values = name_string.split('/'); + + // if only one arg provided, just return name + if (values.length === 1) { + const name = name_string; + return { name, keySize: undefined, category: undefined, platform: undefined, api: undefined }; + } + + // extract by position + const category = values[0] === '' ? undefined : values[0]; + + // If keySize not a number, use `undefined` + const keySizeParsed = parseInt(values[1]); + + const keySize = isNaN(keySizeParsed) ? undefined : keySizeParsed; + + // if name is not defined, keep entire name_string as name + let name = values[2]; + if (name === undefined || name === '') { + name = name_string; + } + + const platform = values[3] === '' ? undefined : values[3]; + const api = values[4] === '' ? undefined : values[4]; + + return { + category, + keySize, + name, + platform, + api, + }; } function extractCargoResult(config: Config, output: string): BenchmarkResult[] { @@ -23,17 +72,24 @@ function extractCargoResult(config: Config, output: string): BenchmarkResult[] { continue; } - const name = m[1].trim(); + const name_string = m[1].trim(); const value = parseFloat(m[2].replace(reComma, '')); const unit = m[3].trim(); const range = m[4].replace(reComma, ''); + // TODO: error handling + const { category, keySize, name, platform, api } = extractMetadataFromName(name_string); + ret.push({ - name, value, range: `± ${range}`, unit: unit, - platform: config.platform, + os: config.os, + category, + keySize, + name, + platform, + api, }); } diff --git a/test/__snapshots__/extract.spec.ts.snap b/test/__snapshots__/extract.spec.ts.snap index d403a75..5707f18 100644 --- a/test/__snapshots__/extract.spec.ts.snap +++ b/test/__snapshots__/extract.spec.ts.snap @@ -3,15 +3,23 @@ exports[`extractData() extracts benchmark output from cargo - cargo_output.txt 1`] = ` [ { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "bench_fib_10", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 24", "unit": "ns/iter", "value": 135, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "bench_fib_20", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 755", "unit": "ns/iter", "value": 18149, @@ -22,36 +30,56 @@ exports[`extractData() extracts benchmark output from cargo - cargo_output.txt 1 exports[`extractData() extracts benchmark output from cargo - cargo_output_units.txt 1`] = ` [ { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "cmov", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 14", "unit": "cycles/iter", "value": 2835, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "cmov2", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 19", "unit": "cycles/iter", "value": 2845, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "mov", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 17", "unit": "cycles/iter", "value": 1508, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "upload", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 420", "unit": "MS/s", "value": 1911, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "download", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 69", "unit": "MS/s", "value": 9001, @@ -62,29 +90,45 @@ exports[`extractData() extracts benchmark output from cargo - cargo_output_units exports[`extractData() extracts benchmark output from cargo - cargo_output2.txt 1`] = ` [ { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "bench_engine_new", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 70126", "unit": "ns/iter", "value": 211834, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "bench_engine_new_raw", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 18", "unit": "ns/iter", "value": 197, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "bench_engine_new_raw_core", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 31", "unit": "ns/iter", "value": 196, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "bench_engine_register_fn", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 82", "unit": "ns/iter", "value": 493, @@ -95,15 +139,23 @@ exports[`extractData() extracts benchmark output from cargo - cargo_output2.txt exports[`extractData() extracts benchmark output from cargo - cargo_output3.txt 1`] = ` [ { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "bench_fib_10", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 2.21", "unit": "ns/iter", "value": 148.7, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "bench_fib_20", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 440.25", "unit": "ns/iter", "value": 18794.12, @@ -114,29 +166,94 @@ exports[`extractData() extracts benchmark output from cargo - cargo_output3.txt exports[`extractData() extracts benchmark output from cargo - criterion_output.txt 1`] = ` [ { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "Create Realm", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 4", "unit": "ns/iter", "value": 329, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "Symbols (Execution)", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 47", "unit": "ns/iter", "value": 3268, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "For loop (Execution)", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 123", "unit": "ns/iter", "value": 12314, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "Fibonacci (Execution)", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, + "range": "± 10166", + "unit": "ns/iter", + "value": 1672496, + }, +] +`; + +exports[`extractData() extracts benchmark output from cargo - criterion_output_with_metadata_format.txt 1`] = ` +[ + { + "api": undefined, + "category": "ML-KEM", + "keySize": 1024, + "name": "PK Validation", + "os": "ubuntu-latest", + "platform": undefined, + "range": "± 4", + "unit": "ns/iter", + "value": 329, + }, + { + "api": "unpacked", + "category": "ML-KEM", + "keySize": 512, + "name": "PK Validation", + "os": "ubuntu-latest", + "platform": "neon", + "range": "± 47", + "unit": "ns/iter", + "value": 3268, + }, + { + "api": "unpacked", + "category": "ML-KEM", + "keySize": 768, + "name": "PK Validation", + "os": "ubuntu-latest", + "platform": "neon", + "range": "± 123", + "unit": "ns/iter", + "value": 12314, + }, + { + "api": "unpacked (external random)", + "category": "ML-KEM", + "keySize": 512, + "name": "PK Validation", + "os": "ubuntu-latest", + "platform": "neon", "range": "± 10166", "unit": "ns/iter", "value": 1672496, @@ -147,15 +264,23 @@ exports[`extractData() extracts benchmark output from cargo - criterion_output.t exports[`localWriteBenchmark() extracts benchmark output from cargo - cargo_output.txt 1`] = ` [ { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "bench_fib_10", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 24", "unit": "ns/iter", "value": 135, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "bench_fib_20", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 755", "unit": "ns/iter", "value": 18149, @@ -166,36 +291,56 @@ exports[`localWriteBenchmark() extracts benchmark output from cargo - cargo_outp exports[`localWriteBenchmark() extracts benchmark output from cargo - cargo_output_units.txt 1`] = ` [ { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "cmov", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 14", "unit": "cycles/iter", "value": 2835, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "cmov2", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 19", "unit": "cycles/iter", "value": 2845, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "mov", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 17", "unit": "cycles/iter", "value": 1508, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "upload", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 420", "unit": "MS/s", "value": 1911, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "download", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 69", "unit": "MS/s", "value": 9001, @@ -206,29 +351,45 @@ exports[`localWriteBenchmark() extracts benchmark output from cargo - cargo_outp exports[`localWriteBenchmark() extracts benchmark output from cargo - cargo_output2.txt 1`] = ` [ { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "bench_engine_new", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 70126", "unit": "ns/iter", "value": 211834, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "bench_engine_new_raw", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 18", "unit": "ns/iter", "value": 197, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "bench_engine_new_raw_core", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 31", "unit": "ns/iter", "value": 196, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "bench_engine_register_fn", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 82", "unit": "ns/iter", "value": 493, @@ -239,15 +400,23 @@ exports[`localWriteBenchmark() extracts benchmark output from cargo - cargo_outp exports[`localWriteBenchmark() extracts benchmark output from cargo - cargo_output3.txt 1`] = ` [ { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "bench_fib_10", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 2.21", "unit": "ns/iter", "value": 148.7, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "bench_fib_20", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 440.25", "unit": "ns/iter", "value": 18794.12, @@ -258,29 +427,45 @@ exports[`localWriteBenchmark() extracts benchmark output from cargo - cargo_outp exports[`localWriteBenchmark() extracts benchmark output from cargo - criterion_output.txt 1`] = ` [ { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "Create Realm", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 4", "unit": "ns/iter", "value": 329, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "Symbols (Execution)", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 47", "unit": "ns/iter", "value": 3268, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "For loop (Execution)", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 123", "unit": "ns/iter", "value": 12314, }, { + "api": undefined, + "category": undefined, + "keySize": undefined, "name": "Fibonacci (Execution)", - "platform": "ubuntu-latest", + "os": "ubuntu-latest", + "platform": undefined, "range": "± 10166", "unit": "ns/iter", "value": 1672496, diff --git a/test/config.spec.ts b/test/config.spec.ts index 8621f08..7a677db 100644 --- a/test/config.spec.ts +++ b/test/config.spec.ts @@ -34,7 +34,7 @@ describe('configFromJobInput()', function () { tool: 'cargo', 'output-file-path': 'out.txt', 'data-out-path': 'test.txt', // TODO - platform: 'any', + os: 'any', }; const validationTests: Array<{ @@ -72,13 +72,13 @@ describe('configFromJobInput()', function () { interface ExpectedResult { name: string; tool: string; - platform: string; + os: string; } const defaultExpected: ExpectedResult = { name: 'Benchmark', tool: 'cargo', - platform: 'platform', + os: 'os', }; const returnedConfigTests: Array<{ diff --git a/test/data/extract/criterion_output_with_metadata_format.txt b/test/data/extract/criterion_output_with_metadata_format.txt new file mode 100644 index 0000000..dc900ca --- /dev/null +++ b/test/data/extract/criterion_output_with_metadata_format.txt @@ -0,0 +1,14 @@ +WARNING: HTML report generation will become a non-default optional feature in Criterion.rs 0.4.0. +This feature is being moved to cargo-criterion (https://github.com/bheisler/cargo-criterion) and will be optional in a future version of Criterion.rs. To silence this warning, either switch to cargo-criterion or enable the 'html_reports' feature in your Cargo.toml. + +Gnuplot not found, using plotters backend +test ML-KEM/1024/PK Validation// ... bench: 329 ns/iter (+/- 4) + +test ML-KEM/512/PK Validation/neon/unpacked ... bench: 3268 ns/iter (+/- 47) + +test ML-KEM/768/PK Validation/neon/unpacked ... bench: 12314 ns/iter (+/- 123) + + +Warning: Unable to complete 100 samples in 5.0s. You may wish to increase target time to 8.5s, enable flat sampling, or reduce sample count to 50. +test ML-KEM/512/PK Validation/neon/unpacked (external random) ... bench: 1672496 ns/iter (+/- 10166) + diff --git a/test/extract.spec.ts b/test/extract.spec.ts index c5fe36e..7798581 100644 --- a/test/extract.spec.ts +++ b/test/extract.spec.ts @@ -28,6 +28,10 @@ describe('extractData()', function () { tool: 'cargo', file: 'criterion_output.txt', }, + { + tool: 'cargo', + file: 'criterion_output_with_metadata_format.txt', + }, ]; it.each(normalCases)(`extracts benchmark output from $tool - $file`, async function (test) { @@ -36,7 +40,7 @@ describe('extractData()', function () { }); const outputFilePath = path.join(__dirname, 'data', 'extract', test.file); const config = { - platform: 'ubuntu-latest', + os: 'ubuntu-latest', tool: test.tool, outputFilePath, } as Config; @@ -49,7 +53,7 @@ describe('extractData()', function () { it('raises an error on unexpected tool', async function () { const config = { - platform: 'ubuntu-latest', + os: 'ubuntu-latest', tool: 'foo' as any, outputFilePath: path.join(__dirname, 'data', 'extract', 'cargo_output_units.txt'), } as Config; @@ -58,7 +62,7 @@ describe('extractData()', function () { it('raises an error when output file is not readable', async function () { const config = { - platform: 'ubuntu-latest', + os: 'ubuntu-latest', tool: 'cargo', outputFilePath: 'path/does/not/exist.txt', } as Config; @@ -67,7 +71,7 @@ describe('extractData()', function () { it('raises an error when no output found', async function () { const config = { - platform: 'ubuntu-latest', + os: 'ubuntu-latest', tool: 'cargo', outputFilePath: path.join(__dirname, 'data', 'extract', 'invalid.txt'), } as Config; @@ -109,7 +113,7 @@ describe('localWriteBenchmark()', function () { const outputFilePath = path.join(__dirname, 'data', 'extract', test.file); const dataOutPath = 'test.txt'; // TODO: tempfile const config = { - platform: 'ubuntu-latest', + os: 'ubuntu-latest', tool: test.tool, outputFilePath, dataOutPath, @@ -119,7 +123,7 @@ describe('localWriteBenchmark()', function () { expect(benches).toMatchSnapshot(); // write out - localWriteBenchmark(benches, config); + await localWriteBenchmark(benches, config); jest.useRealTimers(); });