test results CLI integration (#491)

* feat: integrate test results into toolbox

* fix(test-results): add missing windows binary

* fix(cd): add test-results to release

* toil(test-results): remove unused lint rule

Author: skipi

.semaphore/release.yml

@@ -24,5 +24,10 @@ blocks:
             - artifact pull workflow bin/darwin/amd64/sem-context -d sem-context/bin/darwin/amd64/sem-context
             - artifact pull workflow bin/darwin/arm64/sem-context -d sem-context/bin/darwin/arm64/sem-context
             - artifact pull workflow bin/windows/sem-context.exe -d sem-context/bin/windows/sem-context.exe
+            - artifact pull workflow bin/linux/amd64/test-results -d test-results/bin/linux/amd64/test-results
+            - artifact pull workflow bin/linux/arm64/test-results -d test-results/bin/linux/arm64/test-results
+            - artifact pull workflow bin/darwin/amd64/test-results -d test-results/bin/darwin/amd64/test-results
+            - artifact pull workflow bin/darwin/arm64/test-results -d test-results/bin/darwin/arm64/test-results
+            - artifact pull workflow bin/windows/test-results.exe -d test-results/bin/windows/test-results.exe
             - bash release/create.sh -a
             - bash release/upload.sh

.semaphore/semaphore.yml

@@ -4,7 +4,6 @@ set -euo pipefail
 
 ARTIFACT_CLI_VERSION="v0.6.5"
 SPC_CLI_VERSION="v1.12.1"
-TEST_RESULTS_CLI_VERSION="v0.7.0"
 WHEN_CLI_VERSION="v1.2.1"
 # we include multiple when binaries for all suported Erlang versions
 # and configure the correct one based on Erlang version in the VM where toolbox is installed
@@ -14,7 +13,6 @@ WHEN_BINARY_VERSION_3="when_otp_26"
 
 ARTIFACT_CLI_URL="https://github.com/semaphoreci/artifact/releases/download/$ARTIFACT_CLI_VERSION"
 SPC_CLI_URL="https://github.com/semaphoreci/spc/releases/download/$SPC_CLI_VERSION"
-TEST_RESULTS_CLI_URL="https://github.com/semaphoreci/test-results/releases/download/$TEST_RESULTS_CLI_VERSION"
 WHEN_CLI_URL="https://github.com/renderedtext/when/releases/download/$WHEN_CLI_VERSION"
 
 download_when_cli() {
@@ -114,6 +112,7 @@ hosted::create_initial_content() {
     docker-compose.yml
     cache-cli
     sem-context
+    test-results
     install-self-hosted-toolbox
     install-self-hosted-toolbox.ps1
     Checkout.psm1
@@ -180,11 +179,6 @@ self_hosted::pack() {
   include_external_darwin_binary $ARTIFACT_CLI_URL "artifact" /tmp/self-hosted-Darwin "x86_64"
   include_external_darwin_binary $ARTIFACT_CLI_URL "artifact" /tmp/self-hosted-Darwin-arm "arm64"
   include_external_windows_binary $ARTIFACT_CLI_URL "artifact" /tmp/self-hosted-Windows
-  include_external_linux_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/self-hosted-Linux "x86_64"
-  include_external_linux_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/self-hosted-Linux-arm "arm64"
-  include_external_darwin_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/self-hosted-Darwin "x86_64"
-  include_external_darwin_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/self-hosted-Darwin-arm "arm64"
-  include_external_windows_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/self-hosted-Windows
   include_external_linux_binary $SPC_CLI_URL "spc" /tmp/self-hosted-Linux "x86_64"
   include_external_linux_binary $SPC_CLI_URL "spc" /tmp/self-hosted-Linux-arm "arm64"
   include_external_darwin_binary $SPC_CLI_URL "spc" /tmp/self-hosted-Darwin "x86_64"
@@ -207,6 +201,12 @@ self_hosted::pack() {
   cp ~/$SEMAPHORE_GIT_DIR/sem-context/bin/linux/arm64/sem-context /tmp/self-hosted-Linux-arm/toolbox/
   cp ~/$SEMAPHORE_GIT_DIR/sem-context/bin/darwin/amd64/sem-context /tmp/self-hosted-Darwin/toolbox/
   cp ~/$SEMAPHORE_GIT_DIR/sem-context/bin/darwin/arm64/sem-context /tmp/self-hosted-Darwin-arm/toolbox/
+
+  cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/linux/amd64/test-results /tmp/self-hosted-Linux/toolbox/
+  cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/linux/arm64/test-results /tmp/self-hosted-Linux-arm/toolbox/
+  cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/darwin/amd64/test-results /tmp/self-hosted-Darwin/toolbox/
+  cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/darwin/arm64/test-results /tmp/self-hosted-Darwin-arm/toolbox/
+  cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/windows/test-results.exe /tmp/self-hosted-Windows/toolbox/
 }
 
 hosted::pack() {
@@ -215,10 +215,6 @@ hosted::pack() {
   include_external_linux_binary $ARTIFACT_CLI_URL "artifact" /tmp/Linux-arm "arm64"
   include_external_darwin_binary $ARTIFACT_CLI_URL "artifact" /tmp/Darwin "x86_64"
   include_external_darwin_binary $ARTIFACT_CLI_URL "artifact" /tmp/Darwin-arm "arm64"
-  include_external_linux_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/Linux "x86_64"
-  include_external_linux_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/Linux-arm "arm64"
-  include_external_darwin_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/Darwin "x86_64"
-  include_external_darwin_binary $TEST_RESULTS_CLI_URL "test-results" /tmp/Darwin-arm "arm64"
   include_external_linux_binary $SPC_CLI_URL "spc" /tmp/Linux "x86_64"
   include_external_linux_binary $SPC_CLI_URL "spc" /tmp/Linux-arm "arm64"
   cp ~/$SEMAPHORE_GIT_DIR/cache-cli/bin/linux/amd64/cache /tmp/Linux/toolbox/cache
@@ -229,6 +225,10 @@ hosted::pack() {
   cp ~/$SEMAPHORE_GIT_DIR/sem-context/bin/linux/arm64/sem-context /tmp/Linux-arm/toolbox/sem-context
   cp ~/$SEMAPHORE_GIT_DIR/sem-context/bin/darwin/amd64/sem-context /tmp/Darwin/toolbox/sem-context
   cp ~/$SEMAPHORE_GIT_DIR/sem-context/bin/darwin/arm64/sem-context /tmp/Darwin-arm/toolbox/sem-context
+  cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/linux/amd64/test-results /tmp/Linux/toolbox/test-results
+  cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/linux/arm64/test-results /tmp/Linux-arm/toolbox/test-results
+  cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/darwin/amd64/test-results /tmp/Darwin/toolbox/test-results
+  cp ~/$SEMAPHORE_GIT_DIR/test-results/bin/darwin/arm64/test-results /tmp/Darwin-arm/toolbox/test-results
   cp /tmp/when-cli/$WHEN_BINARY_VERSION_1 /tmp/Linux/toolbox/$WHEN_BINARY_VERSION_1
   cp /tmp/when-cli/$WHEN_BINARY_VERSION_2 /tmp/Linux/toolbox/$WHEN_BINARY_VERSION_2
   cp /tmp/when-cli/$WHEN_BINARY_VERSION_3 /tmp/Linux/toolbox/$WHEN_BINARY_VERSION_3

release/create.sh

@@ -16,7 +16,6 @@ prefix_cmd rm -rf ~/.toolbox
 prefix_cmd rm -f $(which artifact)
 prefix_cmd rm -f $(which spc)
 prefix_cmd rm -f $(which when)
-prefix_cmd rm -f $(which test-results)
 prefix_cmd rm -f $(which enetwork)
 cd ~
 arch=""

release/install_in_tests.sh

@@ -0,0 +1,24 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+*.lcov
+COVERAGE.md
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+.vscode
+.devcontainer
+bin
+samples
+dist/
+test-results
+junit-report.xml

test-results/.gitignore

@@ -0,0 +1,9 @@
+FROM golang:1.23
+
+WORKDIR /app
+
+COPY go.* ./
+
+RUN go install gotest.tools/gotestsum@latest
+RUN go install golang.org/x/lint/golint@latest
+RUN go install github.com/mgechev/[email protected]

test-results/Dockerfile.dev

@@ -0,0 +1,63 @@
+PHONY: run
+
+run:
+	go run main.go $(arg)
+
+regen:
+	go run main.go compile --no-compress priv/parsers/generic/in.xml priv/parsers/generic/out.json
+	go run main.go compile --no-compress priv/parsers/rspec/in.xml priv/parsers/rspec/out.json
+	go run main.go compile --no-compress priv/parsers/exunit/in.xml priv/parsers/exunit/out.json
+	go run main.go compile --no-compress priv/parsers/golang/in.xml priv/parsers/golang/out.json
+	go run main.go compile --no-compress -p phpunit priv/parsers/phpunit/in.xml priv/parsers/phpunit/out.json
+	go run main.go compile --no-compress -p embedded priv/parsers/embedded/in.xml priv/parsers/embedded/out.json
+	go run main.go compile --no-compress priv/merging priv/merging/out.json
+
+test.setup:
+	docker compose build
+	docker compose run cli go get ./...
+
+lint:
+	docker compose run --rm cli revive -formatter friendly -config lint.toml ./...
+
+test:
+	docker compose run --rm cli gotestsum --format short-verbose --junitfile junit-report.xml --packages="./..."
+
+test.watch:
+	gotestsum --watch ./...
+
+test.cover:
+	go install github.com/jandelgado/gcov2lcov@latest
+	go install github.com/securego/gosec/v2/cmd/[email protected]
+	go test -coverprofile=c.out ./...
+	gcov2lcov -infile=c.out -outfile=coverage.lcov
+	rm c.out
+	scripts/lcov-to-md.sh
+
+SECURITY_TOOLBOX_BRANCH ?= master
+SECURITY_TOOLBOX_TMP_DIR ?= /tmp/security-toolbox
+
+check.prepare:
+	rm -rf $(SECURITY_TOOLBOX_TMP_DIR)
+	git clone [email protected]:renderedtext/security-toolbox.git $(SECURITY_TOOLBOX_TMP_DIR) && (cd $(SECURITY_TOOLBOX_TMP_DIR) && git checkout $(SECURITY_TOOLBOX_BRANCH) && cd -)
+
+check.static: check.prepare
+	docker run -it -v $$(pwd):/app \
+		-v $(SECURITY_TOOLBOX_TMP_DIR):$(SECURITY_TOOLBOX_TMP_DIR) \
+		registry.semaphoreci.com/ruby:2.7 \
+		bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/code --language go -d'
+
+check.deps: check.prepare
+	docker run -it -v $$(pwd):/app \
+		-v $(SECURITY_TOOLBOX_TMP_DIR):$(SECURITY_TOOLBOX_TMP_DIR) \
+		-e TRIVY_DB_REPOSITORY -e TRIVY_JAVA_DB_REPOSITORY \
+		registry.semaphoreci.com/ruby:2.7 \
+		bash -c 'cd /app && $(SECURITY_TOOLBOX_TMP_DIR)/dependencies --language go -d'
+
+build.darwin:
+	CGO_ENABLED=0 GOOS=darwin GOARCH=$(ARCH) go build -o bin/darwin/$(ARCH)/test-results main.go
+
+build.linux:
+	CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) go build -o bin/linux/$(ARCH)/test-results main.go
+
+build.windows:
+	CGO_ENABLED=0 GOOS=windows go build -o bin/windows/test-results.exe main.go

test-results/Makefile

@@ -0,0 +1,115 @@
+# Test results [![SemaphoreCI](https://semaphore.semaphoreci.com/badges/test-results-cli.svg)](https://semaphore.semaphoreci.com/projects/test-results-cli)
+
+Semaphore collects XML test reports and uses them to provide insight into your pipelines.
+
+With test reports, you enable your team to get an effective and consistent view of your CI/CD test suite across different test frameworks and stages in a CI/CD workflow. You get a clear failure report for each executed pipeline. Failures are extracted and highlighted, while the rest of the suite is available for analysis.
+
+The test-results command-line interface (CLI) is a tool that helps you compile and process [JUnit XML](https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd) files. The output of the test results CLI is a report in JSON format.
+
+This CLI is distributed as a part of the [Semaphore toolbox](https://github.com/semaphoreci/toolbox), and it is available in all Semaphore jobs.
+
+The main purpose of the CLI is to:
+
+- compile and publish JUnit XML files into a JSON report
+- merge multiple JSON reports into a single summary report
+
+> [!NOTE]
+>
+> Starting from version `0.7.0`, test reports are compressed using gzip to optimize storage and handling. To maintain backward compatibility with existing systems and processes, the compressed files will continue to use the `.json` file extension. This approach ensures seamless integration with tools and workflows that expect files in this format.
+>
+> However, please be aware that while these files retain the `.json` extension, they are in a compressed format and will need to be decompressed using gzip-compatible tools before they can be read as standard JSON.
+>
+> For users who prefer to work with uncompressed reports or for systems that require non-compressed files, we've introduced a `--no-compress` option. This can be used to generate and upload test reports in the traditional, uncompressed JSON format.
+
+## Compiling and publishing JUnit XML files
+
+Given your JUnit XML report is named `results.xml` you can run the following command to generate a report:
+
+```bash
+test-results publish results.xml
+```
+
+The above command parses the content of the `results.xml` file, and publishes the results to Semaphore.
+
+While parsing the content, the CLI tries to find the best parser for your result type. The following test runners have a dedicated parser:
+
+- exunit
+- golang
+- mocha
+- rspec
+- phpunit
+
+If a dedicated parser is not found, the CLI will parse the file using a generic parser. The generic parser uses [JUnit XML Schema](https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd) definition to extract data from the report.
+
+The parser can be selected manually by using the `--parser` option.
+
+```bash
+test-results publish --parser exunit results.xml
+```
+
+The name of the generated report is based on the selected parser. If you want to overwrite this you can use the `--name` option:
+
+```bash
+test-results publish --name "Unit Tests" results.xml
+```
+
+The generated tests in a report will sometimes contain a prefix in the name. For example `Elixir.MyTest`. If you want to remove the `Elixir` prefix from the test names you can use `--suite-prefix` option:
+
+```bash
+test-results publish --suite-prefix "Elixir." results.xml
+```
+
+## Multiple reports from one job
+
+If your job generates multiple reports: `integration.xml`, `unit.xml` you can use this command to merge and publish them
+
+```bash
+test-results publish integration.xml unit.xml
+```
+
+In addition, each report is published separately to artifact storage as a `junit-<index>.xml`. `<index>` is a number starting from 0 that corresponds to the order of the report passed to the command line.
+
+## Merging multiple JSON reports into a single summary report
+
+If you have multiple jobs in your pipeline that generate test results, you can merge them into a single report with the following command
+
+```bash
+test-results gen-pipeline-report
+```
+
+The above command assumes you are running it in a semaphore pipeline. As it uses `SEMAPHORE_PIPELINE_ID` environment variable to identify the pipeline and fetch the job level reports.
+
+## Where are test reports stored?
+
+The test results CLI uses the [Semaphore Artifact Storage](https://docs.semaphoreci.com/essentials/artifacts/) to store the test reports:
+
+- the `test-results publish` command stores the report in the `test-results/junit.json` file on a job level
+- the `test-results gen-pipeline-report` command stores the report in the `test-results/${SEMAPHORE_PIPELINE_ID}.json` file on a workflow level
+
+Expiration date of the artifacts can be controlled via [Retention Policies](https://docs.semaphoreci.com/essentials/artifacts/#artifact-retention-policies).
+
+## Skip uploading raw JUnit XML files
+
+By default, `test-results publish` will upload the raw JUnit XML file alongside the JSON report to the artifact storage. This can be disabled with the `--no-raw` option:
+
+```bash
+test-results publish --no-raw results.xml
+```
+
+## Overwrite existing reports
+
+By default `test-results publish` and `test-results gen-pipeline-report` will fail if the report is already present in the artifact storage. This behaviour can be disabled with the `--force` option:
+
+```bash
+# Publish the report
+test-results publish results.xml
+
+#...
+
+# other-results will overwrite results
+test-results publish --force other-results.xml
+```
+
+## Using the CLI on a local machine
+
+Latest CLI binaries are available [here](https://github.com/semaphoreci/test-results/releases/latest).

test-results/README.md

@@ -0,0 +1,91 @@
+package cmd
+
+/*
+Copyright © 2021 NAME HERE <EMAIL ADDRESS>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import (
+	"github.com/semaphoreci/toolbox/test-results/pkg/cli"
+	"github.com/semaphoreci/toolbox/test-results/pkg/logger"
+	"github.com/semaphoreci/toolbox/test-results/pkg/parser"
+	"github.com/spf13/cobra"
+)
+
+// combineCmd represents the combine command
+var combineCmd = &cobra.Command{
+	Use:   "combine <json-file-path>... <json-file>]",
+	Short: "combines multiples json summary files into one",
+	Long:  `Combines multiples json summary files into one"`,
+	Args:  cobra.MinimumNArgs(2),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		inputs := args[:len(args)-1]
+		output := args[len(args)-1]
+
+		err := cli.SetLogLevel(cmd)
+		if err != nil {
+			return err
+		}
+
+		skipCompression, err := cmd.Flags().GetBool("no-compress")
+		if err != nil {
+			return err
+		}
+
+		paths, err := cli.LoadFiles(inputs, ".json")
+		if err != nil {
+			return err
+		}
+
+		result := parser.NewResult()
+		for _, path := range paths {
+			inFile, err := cli.CheckFile(path)
+			if err != nil {
+				logger.Error(err.Error())
+				return err
+			}
+
+			newResult, err := cli.Load(inFile)
+
+			if err != nil {
+				logger.Error(err.Error())
+				return err
+			}
+			result.Combine(*newResult)
+		}
+
+		err = cli.DecorateResults(&result, cmd)
+		if err != nil {
+			logger.Error("Decorating results failed with error: %v", err)
+			return err
+		}
+
+		jsonData, err := cli.Marshal(result)
+		if err != nil {
+			return err
+		}
+
+		_, err = cli.WriteToFilePath(jsonData, output, !skipCompression)
+		if err != nil {
+			return err
+		}
+		return nil
+	},
+}
+
+func init() {
+	combineCmd.Flags().Int32P("trim-output-to", "s", 0, "trim stdout to N characters, defaults to 0(unlimited)")
+	combineCmd.Flags().BoolP("omit-output-for-passed", "o", false, "omit stdout if test passed, defaults to false")
+	rootCmd.AddCommand(combineCmd)
+}