diff --git a/.gitignore b/.gitignore index bd77fc9f..2ee95b1e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ Gopkg.toml _artifacts vendor +godog.iml diff --git a/_examples/dupsteps/README.md b/_examples/dupsteps/README.md new file mode 100644 index 00000000..0b770322 --- /dev/null +++ b/_examples/dupsteps/README.md @@ -0,0 +1,47 @@ +# Duplicate Steps + +## Problem Statement + +In a testing pipeline in which many `godog` Scenario test cases for a single Feature are run +within a single `godog.TestSuite` (in order to produce a single report of results), it quickly +became apparent that Cucumber's requirement for a global one-to-one pairing between the Pattern +used to match the text of a Step to the Function implementing it is problematic. + +In the illustrative example provided (see the [demo](./demo) and associated [features](./features) folders), +Steps with matching text (e.g., `I fixed it`) appear in two Scenarios; but, each calls +for a _different_ implementation, specific to its Scenario. Cucumber's requirement (as +mentioned above) would force a single implementation of this Step across both Scenarios. +To accommodate this requirement, either the Gherkin (i.e., the _given_ business language) +would have to change, or coding best practices (e.g., Single Responsibility Principle, +Separation of Concerns, Modularity, Encapsulation, Cohesion, etc.) would have to give. + +Running the tests for the two Scenarios _separately_ (e.g., using a separate `godog.TestSuite`) +could "solve" the problem, as matching the common Step text to its scenario-specific Function +would then be unambiguous within the Scenario-specific testing run context. However, a hard requirement +within our build pipeline requires a single "cucumber report" file to be produced as evidence +of the success or failure of _all_ required test Scenarios. Because `godog` produces a +_separate_ report for each invocation of `godog.TestSuite`, _something had to change._ + +## Problem Detection + +A ["step checker" tool](cmd/stepchecker/README.md) was created to facilitate early detection +of the problem situation described above. Using this tool while modifying or adding tests +for new scenarios given by the business was proven useful as part of a "shift left" testing +strategy. + +## Proposed Solution + +A ["solution" was proposed](solution/README.md) of using a simple "report combiner" in conjunction +with establishment of separate, standard `go` tests. + +The main idea in the proposed "solution" _is to use separate_ `godog.TestSuite` instances +to partition execution of the test Scenarios, then collect and combine their output into +the single "cucumber report" required by our build pipeline. + +## Notes + +- See [PR-636](https://github.com/cucumber/godog/pull/636) dealing with a related issue: when `godog` chooses an + incorrect Step Function when more than one matches the text + from a Scenario being tested (i.e., an "ambiguous" Step, such as illustrated + by the `I fixed it` Step in the nested `demo` folder). + diff --git a/_examples/dupsteps/cmd/stepchecker/README.md b/_examples/dupsteps/cmd/stepchecker/README.md new file mode 100644 index 00000000..c282dde5 --- /dev/null +++ b/_examples/dupsteps/cmd/stepchecker/README.md @@ -0,0 +1,38 @@ +# Step Checker + +You can use this step checker tool to help detect missing, duplicate or ambiguous steps +implemented in a `godog` test suite. Early detection of the presence of these prior to +submitting a PR or running a Jenkins pipeline with new or modified step implementations +can potentially save time & help prevent the appearance of unexpected false positives +or negative test results, as described in this example's Problem Statement (see outer +README file). + +## Invocation + +Example: + +```shell +$ cd ~/repos/godog/_examples/dupsteps +$ go run cmd/stepchecker/main.go -- demo/*.go features/*.feature +Found 5 feature step(s): +1. "I can continue on my way" +2. "I accidentally poured concrete down my drain and clogged the sewer line" +3. "I fixed it" + - 2 matching godog step(s) found: + from: features/cloggedDrain.feature:7 + from: features/flatTire.feature:7 + to: demo/dupsteps_test.go:93:11 + to: demo/dupsteps_test.go:125:11 +4. "I can once again use my sink" +5. "I ran over a nail and got a flat tire" + +Found 5 godog step(s): +1. "^I can continue on my way$" +2. "^I accidentally poured concrete down my drain and clogged the sewer line$" +3. "^I fixed it$" +4. "^I can once again use my sink$" +5. "^I ran over a nail and got a flat tire$" + +2024/10/10 20:18:57 1 issue(s) found +exit status 1 +``` diff --git a/_examples/dupsteps/cmd/stepchecker/go.mod b/_examples/dupsteps/cmd/stepchecker/go.mod new file mode 100644 index 00000000..afde37ea --- /dev/null +++ b/_examples/dupsteps/cmd/stepchecker/go.mod @@ -0,0 +1,3 @@ +module github.com/cucumber/godog/_examples/dupsteps/cmd/stepchecker + +go 1.23.1 diff --git a/_examples/dupsteps/cmd/stepchecker/main.go b/_examples/dupsteps/cmd/stepchecker/main.go new file mode 100644 index 00000000..5f1744da --- /dev/null +++ b/_examples/dupsteps/cmd/stepchecker/main.go @@ -0,0 +1,295 @@ +package main + +import ( + "bufio" + "fmt" + "go/ast" + "go/parser" + "go/token" + "log" + "os" + "regexp" + "strconv" + "strings" +) + +// +// See accompanying README file(s); +// also https://github.com/cucumber/godog/pull/642 +// + +func main() { + if len(os.Args) < 3 { + log.Printf("Usage: main.go [go-file(s)] [feature-file(s)]\n") + + os.Exit(RC_USER) + } + + // Structures into which to collect step patterns found in Go and feature files + godogSteps := make(map[string]*StepMatch) + featureSteps := make(map[string]*StepMatch) + + // collect input files (must have at least one of each kind (e.g., *.go, *.feature) + for _, filePath := range os.Args[1:] { + if strings.HasSuffix(filePath, ".go") { + if err := collectGoSteps(filePath, godogSteps); err != nil { + fmt.Printf("error collecting `go` steps: %s\n", err) + + os.Exit(RC_ISSUES) + } + } + + if strings.HasSuffix(filePath, ".feature") { + if err := collectFeatureSteps(filePath, featureSteps); err != nil { + fmt.Printf("error collecting `feature` steps: %s\n", err) + + os.Exit(RC_ISSUES) + } + } + } + + if len(godogSteps) == 0 { + log.Printf("no godog step definition(s) found") + + os.Exit(RC_USER) + } + + if len(featureSteps) == 0 { + log.Printf("no feature step invocation(s) found") + + os.Exit(RC_USER) + } + + // Match steps between Go and feature files + matchSteps(godogSteps, featureSteps) + + var issuesFound int + + // Report on unexpected (i.e., lack of, duplicate or ambiguous) mapping from feature steps to go steps + fmt.Printf("Found %d feature step(s):\n", len(featureSteps)) + + var fsIdx int + + for text, step := range featureSteps { + fsIdx++ + + fmt.Printf("%d. %q\n", fsIdx, text) + + if len(step.matchedWith) != 1 { + issuesFound++ + + fmt.Printf(" - %d matching godog step(s) found:\n", len(step.matchedWith)) + + for _, match := range step.source { + fmt.Printf(" from: %s\n", match) + } + + for _, match := range step.matchedWith { + fmt.Printf(" to: %s\n", match) + } + } + } + + fmt.Println() + + // Report on lack of mapping from go steps to feature steps + fmt.Printf("Found %d godog step(s):\n", len(godogSteps)) + + var gdsIdx int + + for text, step := range godogSteps { + gdsIdx++ + + fmt.Printf("%d. %q\n", gdsIdx, text) + + if len(step.matchedWith) == 0 { + issuesFound++ + + fmt.Printf(" - No matching feature step(s) found:\n") + + for _, match := range step.source { + fmt.Printf(" from: %s\n", match) + } + } + } + + fmt.Println() + + if issuesFound != 0 { + log.Printf("%d issue(s) found\n", issuesFound) + os.Exit(RC_ISSUES) + } +} + +func collectGoSteps(goFilePath string, steps map[string]*StepMatch) error { + fset := token.NewFileSet() + + node, errParse := parser.ParseFile(fset, goFilePath, nil, parser.ParseComments) + if errParse != nil { + return fmt.Errorf("error parsing Go file %s: %w", goFilePath, errParse) + } + + stepDefPattern := regexp.MustCompile(`^godog.ScenarioContext.(Given|When|Then|And|Step)$`) + + var errInspect error + + ast.Inspect(node, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + + methodID := extractMethodID(call) + if methodID == "" { + return true + } + + if !stepDefPattern.MatchString(methodID) { + return true + } + + if len(call.Args) == 0 { + log.Printf("WARNING: ignoring call to step function with no arguments: %s\n", methodID) + + return true + } + + lit, ok := call.Args[0].(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + log.Printf("WARNING: ignoring unexpected step function invocation at %s\n", fset.Position(call.Pos())) + + return true + } + + pattern, errQ := strconv.Unquote(lit.Value) + + if errQ != nil { + errInspect = errQ + return false + } + + sm, found := steps[pattern] + if !found { + sm = &StepMatch{} + steps[pattern] = sm + } + + sm.source = append(sm.source, sourceRef(fset.Position(lit.ValuePos).String())) + + return true + }) + + if errInspect != nil { + return fmt.Errorf("error encountered while inspecting %q: %w", goFilePath, errInspect) + } + + return nil +} + +func collectFeatureSteps(featureFilePath string, steps map[string]*StepMatch) error { + file, errOpen := os.Open(featureFilePath) + if errOpen != nil { + return errOpen + } + + defer func() { _ = file.Close() }() + + scanner := bufio.NewScanner(file) + sp := regexp.MustCompile(`^\s*(Given|When|Then|And) (.+)\s*$`) + + for lineNo := 1; scanner.Scan(); lineNo++ { + + line := scanner.Text() + matches := sp.FindStringSubmatch(line) + if matches == nil { + continue + } + + if len(matches) != 3 { + return fmt.Errorf("unexpected number of matches at %s:%d: %d for %q\n", featureFilePath, lineNo, len(matches), line) + } + + stepText := matches[2] + + sm, found := steps[stepText] + if !found { + sm = &StepMatch{} + steps[stepText] = sm + } + + sm.source = append(sm.source, sourceRef(fmt.Sprintf("%s:%d", featureFilePath, lineNo))) + } + + return nil +} + +func matchSteps(godogSteps, featureSteps map[string]*StepMatch) { + // for each step definition found in go + for pattern, godogStep := range godogSteps { + matcher, errComp := regexp.Compile(pattern) + if errComp != nil { + log.Printf("error compiling regex for pattern '%s': %v\n", pattern, errComp) + + continue + } + + // record matches between feature steps and go steps + for featureText, featureStep := range featureSteps { + if matcher.MatchString(featureText) { + featureStep.matchedWith = append(featureStep.matchedWith, godogStep.source...) + godogStep.matchedWith = append(godogStep.matchedWith, featureStep.source...) + } + } + } +} + +func extractMethodID(call *ast.CallExpr) string { + // Ensure the function is a method call + fnSelExpr, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return "" + } + + // Ensure the method is being called on an identifier + lhsID, ok := fnSelExpr.X.(*ast.Ident) + if !ok || lhsID.Obj == nil { + return "" + } + + // Ensure the identifier represents a field declaration + lhsField, ok := lhsID.Obj.Decl.(*ast.Field) + if !ok { + return "" + } + + // Ensure the field type is a pointer to another type + lhsStarExpr, ok := lhsField.Type.(*ast.StarExpr) + if !ok { + return "" + } + + // Ensure the pointer type is a package or struct + lhsSelExpr, ok := lhsStarExpr.X.(*ast.SelectorExpr) + if !ok { + return "" + } + + // Ensure the receiver type is an identifier (e.g., the package or struct name) + lhsLhsID, ok := lhsSelExpr.X.(*ast.Ident) + if !ok { + return "" + } + + // return a method call identifier sufficient to identify those of interest + return fmt.Sprintf("%s.%s.%s", lhsLhsID.Name, lhsSelExpr.Sel.Name, fnSelExpr.Sel.Name) +} + +type sourceRef string + +type StepMatch struct { + source []sourceRef // location of the source(s) for this step + matchedWith []sourceRef // location of match(es) to this step +} + +const RC_ISSUES = 1 +const RC_USER = 2 diff --git a/_examples/dupsteps/demo/dupsteps_test.go b/_examples/dupsteps/demo/dupsteps_test.go new file mode 100644 index 00000000..e9a45e92 --- /dev/null +++ b/_examples/dupsteps/demo/dupsteps_test.go @@ -0,0 +1,189 @@ +package demo + +import ( + "bytes" + "flag" + "fmt" + "io" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/cucumber/godog" +) + +// +// The tests "pass" by demonstrating the "problem statement" discussed in this `dupsteps` +// example; i.e., they are expected to "fail" when the problem is fixed, or can be fixed +// by using a `godog` configuration option / enhancement which becomes available. +// +// What's being demonstrated is how godog's use of a global list of steps defined across +// all configured scenarios allows for indeterminate results, based upon the order in +// which the steps are loaded. The first "matching" step (text) will be used when +// evaluating a step for a given scenario, regardless of which scenario it was defined +// or intended to participate in. +// + +// TestFlatTireFirst demonstrates that loading the 'flatTire' step implementations first +// causes the 'cloggedDrain' test to fail +func TestFlatTireFirst(t *testing.T) { + demonstrateProblemCase(t, + func(ctx *godog.ScenarioContext) { + (&flatTire{}).addFlatTireSteps(ctx) + (&cloggedDrain{}).addCloggedDrainSteps(ctx) + }, + "the drain is still clogged", + ) +} + +// TestCloggedDrainFirst demonstrates that loading the 'cloggedDrain' step implementations first +// causes the 'flatTire' test to fail +func TestCloggedDrainFirst(t *testing.T) { + demonstrateProblemCase(t, + func(ctx *godog.ScenarioContext) { + (&cloggedDrain{}).addCloggedDrainSteps(ctx) + (&flatTire{}).addFlatTireSteps(ctx) + }, + "tire was not fixed", + ) +} + +// demonstrateProblemCase sets up the test suite using 'caseInitializer' and demonstrates the expected error result. +func demonstrateProblemCase(t *testing.T, caseInitializer func(ctx *godog.ScenarioContext), expectedError string) { + + var sawExpectedError bool + + opts := defaultOpts + opts.Format = "pretty" + opts.Output = &prettyOutputListener{ + wrapped: os.Stdout, + callback: func(s string) { + if strings.Contains(s, expectedError) { + sawExpectedError = true + fmt.Println("====>") + } + }, + } + + suite := godog.TestSuite{ + Name: t.Name(), + ScenarioInitializer: caseInitializer, + Options: &opts, + } + + rc := suite.Run() + + // (demonstration of) expected error + assert.NotZero(t, rc) + + // demonstrate that the expected error message was seen in the godog output + assert.True(t, sawExpectedError) +} + +// Implementation of the steps for the "Clogged Drain" scenario + +type cloggedDrain struct { + drainIsClogged bool +} + +func (cd *cloggedDrain) addCloggedDrainSteps(ctx *godog.ScenarioContext) { + ctx.Given(`^I accidentally poured concrete down my drain and clogged the sewer line$`, cd.clogSewerLine) + ctx.Then(`^I fixed it$`, cd.iFixedIt) + ctx.Then(`^I can once again use my sink$`, cd.useTheSink) +} + +func (cd *cloggedDrain) clogSewerLine() error { + cd.drainIsClogged = true + + return nil +} + +func (cd *cloggedDrain) iFixedIt() error { + cd.drainIsClogged = false + + return nil +} + +func (cd *cloggedDrain) useTheSink() error { + if cd.drainIsClogged { + return fmt.Errorf("the drain is still clogged") + } + + return nil +} + +// Implementation of the steps for the "Flat Tire" scenario + +type flatTire struct { + tireIsFlat bool +} + +func (ft *flatTire) addFlatTireSteps(ctx *godog.ScenarioContext) { + ctx.Given(`^I ran over a nail and got a flat tire$`, ft.gotFlatTire) + ctx.Then(`^I fixed it$`, ft.iFixedIt) + ctx.Then(`^I can continue on my way$`, ft.continueOnMyWay) +} + +func (ft *flatTire) gotFlatTire() error { + ft.tireIsFlat = true + + return nil +} + +func (ft *flatTire) iFixedIt() error { + ft.tireIsFlat = false + + return nil +} + +func (ft *flatTire) continueOnMyWay() error { + if ft.tireIsFlat { + return fmt.Errorf("tire was not fixed") + } + + return nil +} + +// standard godog global environment initialization sequence... + +var defaultOpts = godog.Options{ + Strict: true, + Paths: []string{"../features"}, +} + +func init() { + godog.BindFlags("godog.", flag.CommandLine, &defaultOpts) +} + +// a godog "pretty" output listener used to detect the expected godog error + +type prettyOutputListener struct { + wrapped io.Writer + callback func(string) + buf []byte +} + +func (lw *prettyOutputListener) Write(p []byte) (n int, err error) { + lw.buf = append(lw.buf, p...) + + for { + idx := bytes.IndexByte(lw.buf, '\n') + if idx == -1 { + break + } + + line := string(lw.buf[:idx]) + + lw.callback(line) + + if _, err := lw.wrapped.Write(lw.buf[:idx+1]); err != nil { + return len(p), err + } + + lw.buf = lw.buf[idx+1:] + } + + return len(p), nil +} diff --git a/_examples/dupsteps/demo/go.mod b/_examples/dupsteps/demo/go.mod new file mode 100644 index 00000000..2e5e1a37 --- /dev/null +++ b/_examples/dupsteps/demo/go.mod @@ -0,0 +1,21 @@ +module github.com/cucumber/godog/_examples/dupsteps/tests + +go 1.23.1 + +require ( + github.com/cucumber/godog v0.14.1 + github.com/stretchr/testify v1.9.0 +) + +require ( + github.com/cucumber/gherkin/go/v26 v26.2.0 // indirect + github.com/cucumber/messages/go/v21 v21.0.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/gofrs/uuid v4.3.1+incompatible // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-memdb v1.3.4 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/_examples/dupsteps/demo/go.sum b/_examples/dupsteps/demo/go.sum new file mode 100644 index 00000000..74013d4c --- /dev/null +++ b/_examples/dupsteps/demo/go.sum @@ -0,0 +1,52 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cucumber/gherkin/go/v26 v26.2.0 h1:EgIjePLWiPeslwIWmNQ3XHcypPsWAHoMCz/YEBKP4GI= +github.com/cucumber/gherkin/go/v26 v26.2.0/go.mod h1:t2GAPnB8maCT4lkHL99BDCVNzCh1d7dBhCLt150Nr/0= +github.com/cucumber/godog v0.14.1 h1:HGZhcOyyfaKclHjJ+r/q93iaTJZLKYW6Tv3HkmUE6+M= +github.com/cucumber/godog v0.14.1/go.mod h1:FX3rzIDybWABU4kuIXLZ/qtqEe1Ac5RdXmqvACJOces= +github.com/cucumber/messages/go/v21 v21.0.1 h1:wzA0LxwjlWQYZd32VTlAVDTkW6inOFmSM+RuOwHZiMI= +github.com/cucumber/messages/go/v21 v21.0.1/go.mod h1:zheH/2HS9JLVFukdrsPWoPdmUtmYQAQPLk7w5vWsk5s= +github.com/cucumber/messages/go/v22 v22.0.0/go.mod h1:aZipXTKc0JnjCsXrJnuZpWhtay93k7Rn3Dee7iyPJjs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.3.1+incompatible h1:0/KbAdpx3UXAx1kEOWHJeOkpbgRFGHVgv+CFIY7dBJI= +github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-memdb v1.3.4 h1:XSL3NR682X/cVk2IeV0d70N4DZ9ljI885xAEU8IoK3c= +github.com/hashicorp/go-memdb v1.3.4/go.mod h1:uBTr1oQbtuMgd1SSGoR8YV27eT3sBHbYiNm53bMpgSg= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/_examples/dupsteps/features/cloggedDrain.feature b/_examples/dupsteps/features/cloggedDrain.feature new file mode 100644 index 00000000..507a5c77 --- /dev/null +++ b/_examples/dupsteps/features/cloggedDrain.feature @@ -0,0 +1,8 @@ +@dupSteps +Feature: Dupsteps example features + + @cloggedDrain + Scenario: Clogged Drain + Given I accidentally poured concrete down my drain and clogged the sewer line + Then I fixed it + Then I can once again use my sink diff --git a/_examples/dupsteps/features/flatTire.feature b/_examples/dupsteps/features/flatTire.feature new file mode 100644 index 00000000..564588b0 --- /dev/null +++ b/_examples/dupsteps/features/flatTire.feature @@ -0,0 +1,8 @@ +@dupSteps +Feature: Dupsteps example features + + @flatTire + Scenario: Flat Tire + Given I ran over a nail and got a flat tire + Then I fixed it + Then I can continue on my way diff --git a/_examples/dupsteps/solution/README.md b/_examples/dupsteps/solution/README.md new file mode 100644 index 00000000..ffb99426 --- /dev/null +++ b/_examples/dupsteps/solution/README.md @@ -0,0 +1,14 @@ +# Proposed Solution + +In this folder is demonstrated a proposed solution to the "problem statement" +described in the parent `README` file related to the desire to encapsulate +Step implementations within Features or Scenarios, yet produce a single +report file as a result. + +## Overview + +The proposed solution leverages standard `go` test scaffolding to define and +run multiple `godog` tests (e.g., each using their own `godog.TestSuite`) +for selected Features or Scenarios, then combine the outputs produced into +a single report file, as required in our case. + diff --git a/_examples/dupsteps/solution/cukecombiner.go b/_examples/dupsteps/solution/cukecombiner.go new file mode 100644 index 00000000..35054dee --- /dev/null +++ b/_examples/dupsteps/solution/cukecombiner.go @@ -0,0 +1,31 @@ +package solution + +import ( + "encoding/json" + "fmt" + + "github.com/cucumber/godog/internal/formatters" +) + +// CombineCukeReports "knows" how to combine multiple "cucumber" reports into one +func CombineCukeReports(cukeReportOutputs [][]byte) ([]byte, error) { + var allCukeFeatureJSONs []formatters.CukeFeatureJSON + + for _, output := range cukeReportOutputs { + var cukeFeatureJSONS []formatters.CukeFeatureJSON + + err := json.Unmarshal(output, &cukeFeatureJSONS) + if err != nil { + return nil, fmt.Errorf("can't unmarshal cuke feature JSON: %w", err) + } + + allCukeFeatureJSONs = append(allCukeFeatureJSONs, cukeFeatureJSONS...) + } + + combinedCukeReport, err := json.MarshalIndent(allCukeFeatureJSONs, "", " ") + if err != nil { + return nil, fmt.Errorf("can't marshal combined cuke feature JSON: %w", err) + } + + return combinedCukeReport, nil +} diff --git a/_examples/dupsteps/solution/godoginit.go b/_examples/dupsteps/solution/godoginit.go new file mode 100644 index 00000000..5642692a --- /dev/null +++ b/_examples/dupsteps/solution/godoginit.go @@ -0,0 +1,18 @@ +package solution + +import ( + "flag" + + "github.com/cucumber/godog" +) + +func init() { + // allow user overrides of preferred godog defaults via command-line flags + godog.BindFlags("godog.", flag.CommandLine, &defaultOpts) +} + +// holds preferred godog defaults to be used by the test case(s) +var defaultOpts = godog.Options{ + Strict: true, + Format: "cucumber", +} diff --git a/_examples/dupsteps/solution/multiwriter.go b/_examples/dupsteps/solution/multiwriter.go new file mode 100644 index 00000000..e83db8e2 --- /dev/null +++ b/_examples/dupsteps/solution/multiwriter.go @@ -0,0 +1,28 @@ +package solution + +import "bytes" + +// MultiWriter utility used to collect output generated by the test case(s) +type MultiWriter struct { + bufs []*bytes.Buffer +} + +// NewWriter allocates and returns a new writer used to create output +func (w *MultiWriter) NewWriter() *bytes.Buffer { + buf := &bytes.Buffer{} + + w.bufs = append(w.bufs, buf) + + return buf +} + +// GetOutputs returns the output(s) written by any/all of the writers given out so far... +func (w *MultiWriter) GetOutputs() [][]byte { + outputs := make([][]byte, 0, len(w.bufs)) + + for _, buf := range w.bufs { + outputs = append(outputs, buf.Bytes()) + } + + return outputs +} diff --git a/_examples/dupsteps/solution/simple_test.go b/_examples/dupsteps/solution/simple_test.go new file mode 100644 index 00000000..e6158605 --- /dev/null +++ b/_examples/dupsteps/solution/simple_test.go @@ -0,0 +1,78 @@ +package solution + +import ( + "fmt" + "os" + "testing" + + "github.com/cucumber/godog" + "github.com/stretchr/testify/assert" +) + +// +// Demonstration of a unit testing approach for producing a single aggregated report from runs of separate test suites +// (e.g., `godog.TestSuite` instances), using standard `go` simple test cases. See associated README file. +// + +// the single global var needed to "collect" the output(s) produced by the test(s) +var mw = MultiWriter{} + +// TestMain runs the test case(s), then combines the outputs into a single report +func TestMain(m *testing.M) { + rc := m.Run() // runs the test case(s) + + // then invokes a "combiner" appropriate for the output(s) produced by the test case(s) + // NOTE: the "combiner" is formatter-specific; this one "knows" to combine "cucumber" reports + combinedReport, err := CombineCukeReports(mw.GetOutputs()) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "combiner error: %s\n", err) + } else { + // hmm, it'd be nice to have some CLI options to control this destination... + fmt.Println(string(combinedReport)) + } + + os.Exit(rc) +} + +// one or more test case(s), providing the desired level of step encapsulation + +func TestFlatTire(t *testing.T) { + opts := defaultOpts + + // test runs only selected features/scenarios + opts.Paths = []string{"../features/flatTire.feature"} + opts.Output = mw.NewWriter() + + gts := godog.TestSuite{ + Name: t.Name(), + ScenarioInitializer: func(ctx *godog.ScenarioContext) { + ctx.Step(`^I ran over a nail and got a flat tire$`, func() {}) + ctx.Step(`^I fixed it$`, func() {}) + ctx.Step(`^I can continue on my way$`, func() {}) + }, + Options: &opts, + } + + assert.Zero(t, gts.Run()) +} + +func TestCloggedDrain(t *testing.T) { + opts := defaultOpts + + // test runs only selected features/scenarios + opts.Paths = []string{"../features/cloggedDrain.feature"} + opts.Output = mw.NewWriter() + + gts := godog.TestSuite{ + Name: t.Name(), + ScenarioInitializer: func(ctx *godog.ScenarioContext) { + ctx.Step(`^I accidentally poured concrete down my drain and clogged the sewer line$`, func() {}) + ctx.Step(`^I fixed it$`, func() {}) + ctx.Step(`^I can once again use my sink$`, func() {}) + + }, + Options: &opts, + } + + assert.Zero(t, gts.Run()) +} diff --git a/_examples/dupsteps/solution/table_test.go b/_examples/dupsteps/solution/table_test.go new file mode 100644 index 00000000..839d20b8 --- /dev/null +++ b/_examples/dupsteps/solution/table_test.go @@ -0,0 +1,76 @@ +package solution + +import ( + "fmt" + "os" + "testing" + + "github.com/cucumber/godog" + "github.com/stretchr/testify/assert" +) + +// +// Demonstration of a unit testing approach for producing a single aggregated report from runs of separate test suites +// (e.g., `godog.TestSuite` instances), using standard `go` table-driven tests. See associated README file. +// + +// TestSeparateScenarios runs the case(s) defined in the table, then combines the outputs into a single report +func TestSeparateScenarios(t *testing.T) { + tests := []struct { + name string + paths []string + steps func(ctx *godog.ScenarioContext) + }{ + { + name: "flat tire", + paths: []string{"../features/flatTire.feature"}, + steps: func(ctx *godog.ScenarioContext) { + ctx.Step(`^I ran over a nail and got a flat tire$`, func() {}) + ctx.Step(`^I fixed it$`, func() {}) + ctx.Step(`^I can continue on my way$`, func() {}) + }, + }, + { + name: "clogged sink", + paths: []string{"../features/cloggedDrain.feature"}, + steps: func(ctx *godog.ScenarioContext) { + ctx.Step(`^I accidentally poured concrete down my drain and clogged the sewer line$`, func() {}) + ctx.Step(`^I fixed it$`, func() {}) + ctx.Step(`^I can once again use my sink$`, func() {}) + }, + }, + } + + outputCollector := MultiWriter{} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + opts := defaultOpts + + // test runs only selected features/scenarios + opts.Paths = test.paths + opts.Format = "cucumber" + opts.Output = outputCollector.NewWriter() + + gts := godog.TestSuite{ + Name: t.Name(), + ScenarioInitializer: test.steps, + Options: &opts, + } + + assert.Zero(t, gts.Run()) + }) + } + + // then invokes a "combiner" appropriate for the output(s) produced by the test case(s) + // NOTE: the "combiner" is formatter-specific; this one "knows" to combine "cucumber" reports + combinedReport, err := CombineCukeReports(outputCollector.GetOutputs()) + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "combiner error: %s\n", err) + return + } + + // route the combined output to where it should go... + // hmm, it'd be nice to have some CLI options to control this destination... + fmt.Println(string(combinedReport)) +}