Skip to content

Commit 74f06e4

Browse files
authored
test results CLI integration (#491)
* feat: integrate test results into toolbox * fix(test-results): add missing windows binary * fix(cd): add test-results to release * toil(test-results): remove unused lint rule
1 parent 7f5ee8a commit 74f06e4

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

76 files changed

+10617
-13
lines changed

.semaphore/release.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,5 +24,10 @@ blocks:
2424
- artifact pull workflow bin/darwin/amd64/sem-context -d sem-context/bin/darwin/amd64/sem-context
2525
- artifact pull workflow bin/darwin/arm64/sem-context -d sem-context/bin/darwin/arm64/sem-context
2626
- artifact pull workflow bin/windows/sem-context.exe -d sem-context/bin/windows/sem-context.exe
27+
- artifact pull workflow bin/linux/amd64/test-results -d test-results/bin/linux/amd64/test-results
28+
- artifact pull workflow bin/linux/arm64/test-results -d test-results/bin/linux/arm64/test-results
29+
- artifact pull workflow bin/darwin/amd64/test-results -d test-results/bin/darwin/amd64/test-results
30+
- artifact pull workflow bin/darwin/arm64/test-results -d test-results/bin/darwin/arm64/test-results
31+
- artifact pull workflow bin/windows/test-results.exe -d test-results/bin/windows/test-results.exe
2732
- bash release/create.sh -a
2833
- bash release/upload.sh

.semaphore/semaphore.yml

Lines changed: 97 additions & 1 deletion
Large diffs are not rendered by default.

release/create.sh

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ set -euo pipefail
44

55
ARTIFACT_CLI_VERSION="v0.6.5"
66
SPC_CLI_VERSION="v1.12.1"
7-
TEST_RESULTS_CLI_VERSION="v0.7.0"
87
WHEN_CLI_VERSION="v1.2.1"
98
# we include multiple when binaries for all suported Erlang versions
109
# and configure the correct one based on Erlang version in the VM where toolbox is installed
@@ -14,7 +13,6 @@ WHEN_BINARY_VERSION_3="when_otp_26"
1413

1514
ARTIFACT_CLI_URL="https://github.com/semaphoreci/artifact/releases/download/$ARTIFACT_CLI_VERSION"
1615
SPC_CLI_URL="https://github.com/semaphoreci/spc/releases/download/$SPC_CLI_VERSION"
17-
TEST_RESULTS_CLI_URL="https://github.com/semaphoreci/test-results/releases/download/$TEST_RESULTS_CLI_VERSION"
1816
WHEN_CLI_URL="https://github.com/renderedtext/when/releases/download/$WHEN_CLI_VERSION"
1917

2018
download_when_cli() {
@@ -114,6 +112,7 @@ hosted::create_initial_content() {
114112
docker-compose.yml
115113
cache-cli
116114
sem-context
115+
test-results
117116
install-self-hosted-toolbox
118117
install-self-hosted-toolbox.ps1
119118
Checkout.psm1
@@ -180,11 +179,6 @@ self_hosted::pack() {
180179
include_external_darwin_binary $ARTIFACT_CLI_URL "artifact" /tmp/self-hosted-Darwin "x86_64"
181180
include_external_darwin_binary $ARTIFACT_CLI_URL "artifact" /tmp/self-hosted-Darwin-arm "arm64"
182181
include_external_windows_binary $ARTIFACT_CLI_URL "artifact" /tmp/self-hosted-Windows
183-
include_external_linux_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/self-hosted-Linux "x86_64"
184-
include_external_linux_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/self-hosted-Linux-arm "arm64"
185-
include_external_darwin_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/self-hosted-Darwin "x86_64"
186-
include_external_darwin_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/self-hosted-Darwin-arm "arm64"
187-
include_external_windows_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/self-hosted-Windows
188182
include_external_linux_binary $SPC_CLI_URL "spc" /tmp/self-hosted-Linux "x86_64"
189183
include_external_linux_binary $SPC_CLI_URL "spc" /tmp/self-hosted-Linux-arm "arm64"
190184
include_external_darwin_binary $SPC_CLI_URL "spc" /tmp/self-hosted-Darwin "x86_64"
@@ -207,6 +201,12 @@ self_hosted::pack() {
207201
cp ~/$SEMAPHORE_GIT_DIR/sem-context/bin/linux/arm64/sem-context /tmp/self-hosted-Linux-arm/toolbox/
208202
cp ~/$SEMAPHORE_GIT_DIR/sem-context/bin/darwin/amd64/sem-context /tmp/self-hosted-Darwin/toolbox/
209203
cp ~/$SEMAPHORE_GIT_DIR/sem-context/bin/darwin/arm64/sem-context /tmp/self-hosted-Darwin-arm/toolbox/
204+
205+
cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/linux/amd64/test-results /tmp/self-hosted-Linux/toolbox/
206+
cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/linux/arm64/test-results /tmp/self-hosted-Linux-arm/toolbox/
207+
cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/darwin/amd64/test-results /tmp/self-hosted-Darwin/toolbox/
208+
cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/darwin/arm64/test-results /tmp/self-hosted-Darwin-arm/toolbox/
209+
cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/windows/test-results.exe /tmp/self-hosted-Windows/toolbox/
210210
}
211211

212212
hosted::pack() {
@@ -215,10 +215,6 @@ hosted::pack() {
215215
include_external_linux_binary $ARTIFACT_CLI_URL "artifact" /tmp/Linux-arm "arm64"
216216
include_external_darwin_binary $ARTIFACT_CLI_URL "artifact" /tmp/Darwin "x86_64"
217217
include_external_darwin_binary $ARTIFACT_CLI_URL "artifact" /tmp/Darwin-arm "arm64"
218-
include_external_linux_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/Linux "x86_64"
219-
include_external_linux_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/Linux-arm "arm64"
220-
include_external_darwin_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/Darwin "x86_64"
221-
include_external_darwin_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/Darwin-arm "arm64"
222218
include_external_linux_binary $SPC_CLI_URL "spc" /tmp/Linux "x86_64"
223219
include_external_linux_binary $SPC_CLI_URL "spc" /tmp/Linux-arm "arm64"
224220
cp ~/$SEMAPHORE_GIT_DIR/cache-cli/bin/linux/amd64/cache /tmp/Linux/toolbox/cache
@@ -229,6 +225,10 @@ hosted::pack() {
229225
cp ~/$SEMAPHORE_GIT_DIR/sem-context/bin/linux/arm64/sem-context /tmp/Linux-arm/toolbox/sem-context
230226
cp ~/$SEMAPHORE_GIT_DIR/sem-context/bin/darwin/amd64/sem-context /tmp/Darwin/toolbox/sem-context
231227
cp ~/$SEMAPHORE_GIT_DIR/sem-context/bin/darwin/arm64/sem-context /tmp/Darwin-arm/toolbox/sem-context
228+
cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/linux/amd64/test-results /tmp/Linux/toolbox/test-results
229+
cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/linux/arm64/test-results /tmp/Linux-arm/toolbox/test-results
230+
cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/darwin/amd64/test-results /tmp/Darwin/toolbox/test-results
231+
cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/darwin/arm64/test-results /tmp/Darwin-arm/toolbox/test-results
232232
cp /tmp/when-cli/$WHEN_BINARY_VERSION_1 /tmp/Linux/toolbox/$WHEN_BINARY_VERSION_1
233233
cp /tmp/when-cli/$WHEN_BINARY_VERSION_2 /tmp/Linux/toolbox/$WHEN_BINARY_VERSION_2
234234
cp /tmp/when-cli/$WHEN_BINARY_VERSION_3 /tmp/Linux/toolbox/$WHEN_BINARY_VERSION_3

release/install_in_tests.sh

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ prefix_cmd rm -rf ~/.toolbox
1616
prefix_cmd rm -f $(which artifact)
1717
prefix_cmd rm -f $(which spc)
1818
prefix_cmd rm -f $(which when)
19-
prefix_cmd rm -f $(which test-results)
2019
prefix_cmd rm -f $(which enetwork)
2120
cd ~
2221
arch=""

test-results/.gitignore

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Binaries for programs and plugins
2+
*.exe
3+
*.exe~
4+
*.dll
5+
*.so
6+
*.dylib
7+
8+
# Test binary, built with `go test -c`
9+
*.test
10+
11+
# Output of the go coverage tool, specifically when used with LiteIDE
12+
*.out
13+
*.lcov
14+
COVERAGE.md
15+
16+
# Dependency directories (remove the comment below to include it)
17+
# vendor/
18+
.vscode
19+
.devcontainer
20+
bin
21+
samples
22+
dist/
23+
test-results
24+
junit-report.xml

test-results/Dockerfile.dev

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
FROM golang:1.23
2+
3+
WORKDIR /app
4+
5+
COPY go.* ./
6+
7+
RUN go install gotest.tools/gotestsum@latest
8+
RUN go install golang.org/x/lint/golint@latest
9+
RUN go install github.com/mgechev/[email protected]

test-results/Makefile

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
PHONY: run
2+
3+
run:
4+
go run main.go $(arg)
5+
6+
regen:
7+
go run main.go compile --no-compress priv/parsers/generic/in.xml priv/parsers/generic/out.json
8+
go run main.go compile --no-compress priv/parsers/rspec/in.xml priv/parsers/rspec/out.json
9+
go run main.go compile --no-compress priv/parsers/exunit/in.xml priv/parsers/exunit/out.json
10+
go run main.go compile --no-compress priv/parsers/golang/in.xml priv/parsers/golang/out.json
11+
go run main.go compile --no-compress -p phpunit priv/parsers/phpunit/in.xml priv/parsers/phpunit/out.json
12+
go run main.go compile --no-compress -p embedded priv/parsers/embedded/in.xml priv/parsers/embedded/out.json
13+
go run main.go compile --no-compress priv/merging priv/merging/out.json
14+
15+
test.setup:
16+
docker compose build
17+
docker compose run cli go get ./...
18+
19+
lint:
20+
docker compose run --rm cli revive -formatter friendly -config lint.toml ./...
21+
22+
test:
23+
docker compose run --rm cli gotestsum --format short-verbose --junitfile junit-report.xml --packages="./..."
24+
25+
test.watch:
26+
gotestsum --watch ./...
27+
28+
test.cover:
29+
go install github.com/jandelgado/gcov2lcov@latest
30+
go install github.com/securego/gosec/v2/cmd/[email protected]
31+
go test -coverprofile=c.out ./...
32+
gcov2lcov -infile=c.out -outfile=coverage.lcov
33+
rm c.out
34+
scripts/lcov-to-md.sh
35+
36+
SECURITY_TOOLBOX_BRANCH ?= master
37+
SECURITY_TOOLBOX_TMP_DIR ?= /tmp/security-toolbox
38+
39+
check.prepare:
40+
rm -rf $(SECURITY_TOOLBOX_TMP_DIR)
41+
git clone [email protected]:renderedtext/security-toolbox.git $(SECURITY_TOOLBOX_TMP_DIR) && (cd $(SECURITY_TOOLBOX_TMP_DIR) && git checkout $(SECURITY_TOOLBOX_BRANCH) && cd -)
42+
43+
check.static: check.prepare
44+
docker run -it -v $$(pwd):/app \
45+
-v $(SECURITY_TOOLBOX_TMP_DIR):$(SECURITY_TOOLBOX_TMP_DIR) \
46+
registry.semaphoreci.com/ruby:2.7 \
47+
bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/code --language go -d'
48+
49+
check.deps: check.prepare
50+
docker run -it -v $$(pwd):/app \
51+
-v $(SECURITY_TOOLBOX_TMP_DIR):$(SECURITY_TOOLBOX_TMP_DIR) \
52+
-e TRIVY_DB_REPOSITORY -e TRIVY_JAVA_DB_REPOSITORY \
53+
registry.semaphoreci.com/ruby:2.7 \
54+
bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/dependencies --language go -d'
55+
56+
build.darwin:
57+
CGO_ENABLED=0 GOOS=darwin GOARCH=$(ARCH) go build -o bin/darwin/$(ARCH)/test-results main.go
58+
59+
build.linux:
60+
CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) go build -o bin/linux/$(ARCH)/test-results main.go
61+
62+
build.windows:
63+
CGO_ENABLED=0 GOOS=windows go build -o bin/windows/test-results.exe main.go

test-results/README.md

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
# Test results [![SemaphoreCI](https://semaphore.semaphoreci.com/badges/test-results-cli.svg)](https://semaphore.semaphoreci.com/projects/test-results-cli)
2+
3+
Semaphore collects XML test reports and uses them to provide insight into your pipelines.
4+
5+
With test reports, you enable your team to get an effective and consistent view of your CI/CD test suite across different test frameworks and stages in a CI/CD workflow. You get a clear failure report for each executed pipeline. Failures are extracted and highlighted, while the rest of the suite is available for analysis.
6+
7+
The test-results command-line interface (CLI) is a tool that helps you compile and process [JUnit XML](https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd) files. The output of the test results CLI is a report in JSON format.
8+
9+
This CLI is distributed as a part of the [Semaphore toolbox](https://github.com/semaphoreci/toolbox), and it is available in all Semaphore jobs.
10+
11+
The main purpose of the CLI is to:
12+
13+
- compile and publish JUnit XML files into a JSON report
14+
- merge multiple JSON reports into a single summary report
15+
16+
> [!NOTE]
17+
>
18+
> Starting from version `0.7.0`, test reports are compressed using gzip to optimize storage and handling. To maintain backward compatibility with existing systems and processes, the compressed files will continue to use the `.json` file extension. This approach ensures seamless integration with tools and workflows that expect files in this format.
19+
>
20+
> However, please be aware that while these files retain the `.json` extension, they are in a compressed format and will need to be decompressed using gzip-compatible tools before they can be read as standard JSON.
21+
>
22+
> For users who prefer to work with uncompressed reports or for systems that require non-compressed files, we've introduced a `--no-compress` option. This can be used to generate and upload test reports in the traditional, uncompressed JSON format.
23+
24+
## Compiling and publishing JUnit XML files
25+
26+
Given your JUnit XML report is named `results.xml` you can run the following command to generate a report:
27+
28+
```bash
29+
test-results publish results.xml
30+
```
31+
32+
The above command parses the content of the `results.xml` file, and publishes the results to Semaphore.
33+
34+
While parsing the content, the CLI tries to find the best parser for your result type. The following test runners have a dedicated parser:
35+
36+
- exunit
37+
- golang
38+
- mocha
39+
- rspec
40+
- phpunit
41+
42+
If a dedicated parser is not found, the CLI will parse the file using a generic parser. The generic parser uses [JUnit XML Schema](https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd) definition to extract data from the report.
43+
44+
The parser can be selected manually by using the `--parser` option.
45+
46+
```bash
47+
test-results publish --parser exunit results.xml
48+
```
49+
50+
The name of the generated report is based on the selected parser. If you want to overwrite this you can use the `--name` option:
51+
52+
```bash
53+
test-results publish --name "Unit Tests" results.xml
54+
```
55+
56+
The generated tests in a report will sometimes contain a prefix in the name. For example `Elixir.MyTest`. If you want to remove the `Elixir` prefix from the test names you can use `--suite-prefix` option:
57+
58+
```bash
59+
test-results publish --suite-prefix "Elixir." results.xml
60+
```
61+
62+
## Multiple reports from one job
63+
64+
If your job generates multiple reports: `integration.xml`, `unit.xml` you can use this command to merge and publish them
65+
66+
```bash
67+
test-results publish integration.xml unit.xml
68+
```
69+
70+
In addition, each report is published separately to artifact storage as a `junit-<index>.xml`. `<index>` is a number starting from 0 that corresponds to the order of the report passed to the command line.
71+
72+
## Merging multiple JSON reports into a single summary report
73+
74+
If you have multiple jobs in your pipeline that generate test results, you can merge them into a single report with the following command
75+
76+
```bash
77+
test-results gen-pipeline-report
78+
```
79+
80+
The above command assumes you are running it in a semaphore pipeline. As it uses `SEMAPHORE_PIPELINE_ID` environment variable to identify the pipeline and fetch the job level reports.
81+
82+
## Where are test reports stored?
83+
84+
The test results CLI uses the [Semaphore Artifact Storage](https://docs.semaphoreci.com/essentials/artifacts/) to store the test reports:
85+
86+
- the `test-results publish` command stores the report in the `test-results/junit.json` file on a job level
87+
- the `test-results gen-pipeline-report` command stores the report in the `test-results/${SEMAPHORE_PIPELINE_ID}.json` file on a workflow level
88+
89+
Expiration date of the artifacts can be controlled via [Retention Policies](https://docs.semaphoreci.com/essentials/artifacts/#artifact-retention-policies).
90+
91+
## Skip uploading raw JUnit XML files
92+
93+
By default, `test-results publish` will upload the raw JUnit XML file alongside the JSON report to the artifact storage. This can be disabled with the `--no-raw` option:
94+
95+
```bash
96+
test-results publish --no-raw results.xml
97+
```
98+
99+
## Overwrite existing reports
100+
101+
By default `test-results publish` and `test-results gen-pipeline-report` will fail if the report is already present in the artifact storage. This behaviour can be disabled with the `--force` option:
102+
103+
```bash
104+
# Publish the report
105+
test-results publish results.xml
106+
107+
#...
108+
109+
# other-results will overwrite results
110+
test-results publish --force other-results.xml
111+
```
112+
113+
## Using the CLI on a local machine
114+
115+
Latest CLI binaries are available [here](https://github.com/semaphoreci/test-results/releases/latest).

test-results/cmd/combine.go

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
package cmd
2+
3+
/*
4+
Copyright © 2021 NAME HERE <EMAIL ADDRESS>
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
19+
import (
20+
"github.com/semaphoreci/toolbox/test-results/pkg/cli"
21+
"github.com/semaphoreci/toolbox/test-results/pkg/logger"
22+
"github.com/semaphoreci/toolbox/test-results/pkg/parser"
23+
"github.com/spf13/cobra"
24+
)
25+
26+
// combineCmd represents the combine command
27+
var combineCmd = &cobra.Command{
28+
Use: "combine <json-file-path>... <json-file>]",
29+
Short: "combines multiples json summary files into one",
30+
Long: `Combines multiples json summary files into one"`,
31+
Args: cobra.MinimumNArgs(2),
32+
RunE: func(cmd *cobra.Command, args []string) error {
33+
inputs := args[:len(args)-1]
34+
output := args[len(args)-1]
35+
36+
err := cli.SetLogLevel(cmd)
37+
if err != nil {
38+
return err
39+
}
40+
41+
skipCompression, err := cmd.Flags().GetBool("no-compress")
42+
if err != nil {
43+
return err
44+
}
45+
46+
paths, err := cli.LoadFiles(inputs, ".json")
47+
if err != nil {
48+
return err
49+
}
50+
51+
result := parser.NewResult()
52+
for _, path := range paths {
53+
inFile, err := cli.CheckFile(path)
54+
if err != nil {
55+
logger.Error(err.Error())
56+
return err
57+
}
58+
59+
newResult, err := cli.Load(inFile)
60+
61+
if err != nil {
62+
logger.Error(err.Error())
63+
return err
64+
}
65+
result.Combine(*newResult)
66+
}
67+
68+
err = cli.DecorateResults(&result, cmd)
69+
if err != nil {
70+
logger.Error("Decorating results failed with error: %v", err)
71+
return err
72+
}
73+
74+
jsonData, err := cli.Marshal(result)
75+
if err != nil {
76+
return err
77+
}
78+
79+
_, err = cli.WriteToFilePath(jsonData, output, !skipCompression)
80+
if err != nil {
81+
return err
82+
}
83+
return nil
84+
},
85+
}
86+
87+
func init() {
88+
combineCmd.Flags().Int32P("trim-output-to", "s", 0, "trim stdout to N characters, defaults to 0(unlimited)")
89+
combineCmd.Flags().BoolP("omit-output-for-passed", "o", false, "omit stdout if test passed, defaults to false")
90+
rootCmd.AddCommand(combineCmd)
91+
}

0 commit comments

Comments
 (0)