Skip to content

Use new metadata format #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Apr 3, 2025
Merged
8 changes: 4 additions & 4 deletions .github/workflows/build-and-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,24 +19,24 @@ jobs:
- uses: ./ # local action in repo
with:
tool: 'cargo'
platform: 'default'
os: 'ubuntu-latest'
output-file-path: ./test/data/extract/cargo_output.txt
data-out-path: output.txt
- uses: ./ # local action in repo
with:
tool: 'cargo'
platform: 'default'
os: 'ubuntu-latest'
output-file-path: ./test/data/extract/cargo_output2.txt
data-out-path: output.txt
- uses: ./ # local action in repo
with:
tool: 'cargo'
platform: 'default'
os: 'ubuntu-latest'
output-file-path: ./test/data/extract/cargo_output3.txt
data-out-path: output.txt
- uses: ./ # local action in repo
with:
tool: 'cargo'
platform: 'default'
os: 'ubuntu-latest'
output-file-path: ./test/data/extract/criterion_output.txt
data-out-path: output.txt
2 changes: 1 addition & 1 deletion .github/workflows/minimal.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,6 @@ jobs:
- uses: ./ # local action in repo
with:
tool: 'cargo'
platform: 'default'
os: 'ubuntu-latest'
output-file-path: ./test/data/extract/cargo_output.txt
data-out-path: output.txt
24 changes: 16 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,29 @@ Currently supported sources:
## Fields
- `name` (required): name of the benchmark
- `tool` (required): tool used to get benchmark output. One of `["cargo"]`
- `platform` (required): a string describing the platform
- `os` (required): a string describing the os
- `output-file-path` (required): a path to a file containing the output of the benchmark tool
- `data-out-path` (required): the path where the output of the action should be written

## Metadata format

The benchmark name in the `cargo` benchmarks can be provided as a `/`-separated string with the format `category/key size/name/platform/api`. The key size should be an integer. Some fields in this string can be left blank. Any unspecified or invalid fields will be parsed to `undefined`.

## Output data format

The output will be written to `data-out-path` in a standardized JSON format:
```json
[
{
"name": "My Custom Smaller Is Better Benchmark - Memory Used",
"unit": "Megabytes",
"platform": "ubuntu-latest",
"value": 100,
"range": "3",
}
{
"api": "unpacked",
"category": "ML-KEM",
"keySize": 768,
"name": "PK Validation",
"os": "ubuntu-latest",
"platform": "neon",
"range": "± 123",
"unit": "ns/iter",
"value": 12314,
},
]
```
2 changes: 1 addition & 1 deletion action-types.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ inputs:
type: enum
allowed-values:
- cargo
platform:
os:
type: string
output-file-path:
type: string
Expand Down
4 changes: 2 additions & 2 deletions action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ inputs:
tool:
description: 'Tool to use to get benchmark output. One of "cargo"...'
required: true
platform:
description: 'A string describing the platform'
os:
description: 'A string describing the os'
required: true
output-file-path:
description: 'A path to file which contains the benchmark output'
Expand Down
14 changes: 7 additions & 7 deletions src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import * as path from 'path';
export type ToolType = typeof VALID_TOOLS[number];
export interface Config {
name: string;
platform: string;
os: string;
tool: ToolType;
outputFilePath: string;
dataOutPath: string;
Expand Down Expand Up @@ -71,22 +71,22 @@ function validateName(name: string) {
}
throw new Error('Name must not be empty');
}
function validatePlatform(platform: string) {
if (platform) {
function validateOs(os: string) {
if (os) {
return;
}
throw new Error('Platform must not be empty');
throw new Error('Os must not be empty');
}

export async function configFromJobInput(): Promise<Config> {
const tool: string = core.getInput('tool');
let outputFilePath: string = core.getInput('output-file-path');
let dataOutPath: string = core.getInput('data-out-path');
const name: string = core.getInput('name');
const platform: string = core.getInput('platform');
const os: string = core.getInput('os');

validateName(name);
validatePlatform(platform);
validateOs(os);
validateToolType(tool);
outputFilePath = await validateOutputFilePath(outputFilePath);
dataOutPath = await validateDataOutPath(dataOutPath);
Expand All @@ -96,6 +96,6 @@ export async function configFromJobInput(): Promise<Config> {
tool,
outputFilePath,
dataOutPath,
platform,
os,
};
}
66 changes: 61 additions & 5 deletions src/extract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,61 @@ import { promises as fs } from 'fs';
import { Config } from './config';

export interface BenchmarkResult {
name: string;
value: number;
range?: string;
unit: string;
extra?: string;
platform: string;
os: string;

// from NameMetadata
category: string | undefined;
keySize: number | undefined;
name: string;
platform: string | undefined;
api: string | undefined;
}

interface NameMetadata {
category: string | undefined;
keySize: number | undefined;
name: string;
platform: string | undefined;
api: string | undefined;
}
function extractMetadataFromName(name_string: string): NameMetadata {
// split by separator
const values = name_string.split('/');

// if only one arg provided, just return name
if (values.length === 1) {
const name = name_string;
return { name, keySize: undefined, category: undefined, platform: undefined, api: undefined };
}

// extract by position
const category = values[0] === '' ? undefined : values[0];

// If keySize not a number, use `undefined`
const keySizeParsed = parseInt(values[1]);

const keySize = isNaN(keySizeParsed) ? undefined : keySizeParsed;

// if name is not defined, keep entire name_string as name
let name = values[2];
if (name === undefined || name === '') {
name = name_string;
}

const platform = values[3] === '' ? undefined : values[3];
const api = values[4] === '' ? undefined : values[4];

return {
category,
keySize,
name,
platform,
api,
};
}

function extractCargoResult(config: Config, output: string): BenchmarkResult[] {
Expand All @@ -23,17 +72,24 @@ function extractCargoResult(config: Config, output: string): BenchmarkResult[] {
continue;
}

const name = m[1].trim();
const name_string = m[1].trim();
const value = parseFloat(m[2].replace(reComma, ''));
const unit = m[3].trim();
const range = m[4].replace(reComma, '');

// TODO: error handling
const { category, keySize, name, platform, api } = extractMetadataFromName(name_string);

ret.push({
name,
value,
range: `± ${range}`,
unit: unit,
platform: config.platform,
os: config.os,
category,
keySize,
name,
platform,
api,
});
}

Expand Down
Loading