diff --git a/__tests__/cli.test.ts b/__tests__/cli.test.ts index 560eb8c6..d0260b0f 100644 --- a/__tests__/cli.test.ts +++ b/__tests__/cli.test.ts @@ -34,7 +34,7 @@ import { safeNDJSONParse, } from '../src/helpers'; -describe('CLI', () => { +describe.only('CLI', () => { let server: Server; let serverParams: { url: string }; beforeAll(async () => { @@ -44,6 +44,7 @@ describe('CLI', () => { afterAll(async () => await server.close()); const FIXTURES_DIR = join(__dirname, 'fixtures'); + const MATRIX_FIXTURES_DIR = join(__dirname, 'fixtures/matrix'); // jest by default sets NODE_ENV to `test` const originalNodeEnv = process.env['NODE_ENV']; @@ -541,7 +542,7 @@ describe('CLI', () => { }); }); - describe('playwright options', () => { + describe.only('playwright options', () => { it('pass playwright options to runner', async () => { const cli = new CLIMock() .args([ @@ -574,12 +575,30 @@ describe('CLI', () => { }), ]) .run(); - await cli.waitFor('step/end'); - const output = cli.output(); + await cli.waitFor('step/end'); + const output = cli.output(); + expect(await cli.exitCode).toBe(1); + expect(JSON.parse(output).step).toMatchObject({ + status: 'failed', + }); + }); + + it.only('handles matrix', async () => { + const cli = new CLIMock() + .args([ + join(MATRIX_FIXTURES_DIR, 'example.journey.ts'), + '--reporter', + 'json', + '--config', + join(MATRIX_FIXTURES_DIR, 'synthetics.config.ts'), + ]) + .run(); expect(await cli.exitCode).toBe(1); - expect(JSON.parse(output).step).toMatchObject({ - status: 'failed', - }); + const endEvents = cli.buffer().filter(data => data.includes('journey/end')); + const badsslFailing = endEvents.find(event => event.includes('badssl failing')); + const badsslSucceeded = endEvents.find(event => event.includes('badssl passing')) + expect(JSON.parse(badsslFailing || '')?.journey?.status).toBe("failed"); + expect(JSON.parse(badsslSucceeded || '')?.journey?.status).toBe("succeeded"); }); }); }); diff --git a/__tests__/core/runner.test.ts b/__tests__/core/runner.test.ts index c069e5a5..f2287675 100644 --- a/__tests__/core/runner.test.ts +++ b/__tests__/core/runner.test.ts @@ -729,7 +729,7 @@ describe('runner', () => { type: 'browser', tags: [], locations: ['united_kingdom'], - privaateLocations: undefined, + privateLocations: undefined, schedule: 3, params: undefined, playwrightOptions: undefined, diff --git a/__tests__/fixtures/matrix/example.journey.ts b/__tests__/fixtures/matrix/example.journey.ts new file mode 100644 index 00000000..57ee0e34 --- /dev/null +++ b/__tests__/fixtures/matrix/example.journey.ts @@ -0,0 +1,33 @@ +/** + * MIT License + * + * Copyright (c) 2020-present, Elastic NV + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +import { journey, step } from '../../../src/index'; + +journey('matrix journey', ({ page, params }) => { + step('go to test page', async () => { + await page.goto(params.url, { waitUntil: 'networkidle' }); + await page.waitForSelector(`text=${params.assertion}`, { timeout: 1500 }); + }); +}); diff --git a/__tests__/fixtures/matrix/synthetics.config.ts b/__tests__/fixtures/matrix/synthetics.config.ts new file mode 100644 index 00000000..a4fb9473 --- /dev/null +++ b/__tests__/fixtures/matrix/synthetics.config.ts @@ -0,0 +1,31 @@ +import type { SyntheticsConfig } from '../../../src'; + +module.exports = () => { + const config: SyntheticsConfig = { + params: { + url: 'dev', + }, + matrix: { + adjustments: [{ + name: 'badssl failing', + params: { + url: 'https://expired.badssl.com/', + assertion: 'expired', + }, + playwrightOptions: { + ignoreHTTPSErrors: false, + } + }, { + name: 'badssl passing', + params: { + url: 'https://expired.badssl.com/', + assertion: 'expired', + }, + playwrightOptions: { + ignoreHTTPSErrors: true, + } + }] + } + }; + return config; +}; diff --git a/src/common_types.ts b/src/common_types.ts index ce3ddd2f..77270949 100644 --- a/src/common_types.ts +++ b/src/common_types.ts @@ -237,6 +237,7 @@ export type RunOptions = BaseArgs & { environment?: string; playwrightOptions?: PlaywrightOptions; networkConditions?: NetworkConditions; + matrix?: Matrix; reporter?: BuiltInReporterName | ReporterInstance; }; @@ -254,12 +255,18 @@ export type ProjectSettings = { space: string; }; +export type Matrix = { + // values: Record; + adjustments: Array<{ playwrightOptions?: PlaywrightOptions, name: string, params?: Record }>; +} + export type PlaywrightOptions = LaunchOptions & BrowserContextOptions; export type SyntheticsConfig = { params?: Params; playwrightOptions?: PlaywrightOptions; monitor?: MonitorConfig; project?: ProjectSettings; + matrix?: Matrix; }; /** Runner Payload types */ diff --git a/src/core/index.ts b/src/core/index.ts index ce806e6a..f41f5ce0 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -22,7 +22,6 @@ * THE SOFTWARE. * */ - import { Journey, JourneyCallback, JourneyOptions } from '../dsl'; import Runner from './runner'; import { VoidCallback, HooksCallback, Location } from '../common_types'; @@ -30,6 +29,16 @@ import { wrapFnWithLocation } from '../helpers'; import { log } from './logger'; import { MonitorConfig } from '../dsl/monitor'; + +/* TODO: Testing + * Local vs global matrix: Local matrix fully overwrites global matrix, rather than merging + * Adjustments: Duplicates in adjustments do not run extra journeys + * Regular params are combined with matrix params + * Project monitors: name and id are overwritten only for matrix monitors + * How does monitor params/playwright options interact with matrix overrides? + * Should it be global params -> monitor params -> matrix params + * Should it be global playwrightOptions -> monitor playWrightOptions -> matrix playwrightOptions + /** * Use a gloabl Runner which would be accessed by the runtime and * required to handle the local vs global invocation through CLI @@ -38,7 +47,6 @@ const SYNTHETICS_RUNNER = Symbol.for('SYNTHETICS_RUNNER'); if (!global[SYNTHETICS_RUNNER]) { global[SYNTHETICS_RUNNER] = new Runner(); } - export const runner: Runner = global[SYNTHETICS_RUNNER]; export const journey = wrapFnWithLocation( diff --git a/src/core/runner.ts b/src/core/runner.ts index d3dee99e..04f71b27 100644 --- a/src/core/runner.ts +++ b/src/core/runner.ts @@ -25,7 +25,7 @@ import { join } from 'path'; import { mkdir, rm, writeFile } from 'fs/promises'; -import { Journey } from '../dsl/journey'; +import { Journey, Suite } from '../dsl/journey'; import { Step } from '../dsl/step'; import { reporters, Reporter } from '../reporters'; import { @@ -35,6 +35,7 @@ import { runParallel, generateUniqueId, } from '../helpers'; +import { getCombinations } from '../matrix'; import { HooksCallback, Params, @@ -44,6 +45,7 @@ import { RunOptions, JourneyResult, StepResult, + Location, } from '../common_types'; import { PluginManager } from '../plugins'; import { PerformanceManager } from '../plugins'; @@ -68,6 +70,7 @@ export default class Runner { #reporter: Reporter; #currentJourney?: Journey = null; journeys: Journey[] = []; + suites: Map = new Map(); hooks: SuiteHooks = { beforeAll: [], afterAll: [] }; hookError: Error | undefined; monitor?: Monitor; @@ -138,10 +141,22 @@ export default class Runner { } addJourney(journey: Journey) { + const journeySuite = this.suites.get(journey.location); + if (journeySuite) { + journeySuite.addJourney(journey); + } else { + const suite = new Suite(journey.location); + suite.addJourney(journey); + this.addSuite(suite); + } this.journeys.push(journey); this.#currentJourney = journey; } + addSuite(suite: Suite) { + this.suites.set(suite.location, suite); + } + setReporter(options: RunOptions) { /** * Set up the corresponding reporter and fallback @@ -339,16 +354,19 @@ export default class Runner { async runJourney(journey: Journey, options: RunOptions) { const result: JourneyResult = { status: 'succeeded' }; - const context = await Runner.createContext(options); + const params = Object.freeze({...options.params, ...journey.params}); + const playwrightOptions = { ...options.playwrightOptions, ...journey.playwrightOptions }; + const journeyOptions = { ...options, params, playwrightOptions }; + const context = await Runner.createContext(journeyOptions); log(`Runner: start journey (${journey.name})`); try { this.registerJourney(journey, context); const hookArgs = { env: options.environment, - params: options.params, + params: params, }; await this.runBeforeHook(journey, hookArgs); - const stepResults = await this.runSteps(journey, context, options); + const stepResults = await this.runSteps(journey, context, journeyOptions); /** * Mark journey as failed if any intermediate step fails */ @@ -363,7 +381,7 @@ export default class Runner { result.status = 'failed'; result.error = e; } finally { - await this.endJourney(journey, { ...context, ...result }, options); + await this.endJourney(journey, { ...context, ...result }, journeyOptions); await Gatherer.dispose(context.driver); } log(`Runner: end journey (${journey.name})`); @@ -383,10 +401,13 @@ export default class Runner { } buildMonitors(options: RunOptions) { + /* Build out monitors according to matrix specs */ + this.parseMatrix(options); + /** * Update the global monitor configuration required for * setting defaults - */ + */ this.updateMonitor({ throttling: options.throttling, schedule: options.schedule, @@ -398,7 +419,12 @@ export default class Runner { const { match, tags } = options; const monitors: Monitor[] = []; - for (const journey of this.journeys) { + + const journeys = this.getAllJourneys(); + + for (const journey of journeys) { + const params = Object.freeze({ ...options.params, ...this.monitor.config?.params, ...journey.params }); + const playwrightOptions = { ...options.playwrightOptions, ...this.monitor.config?.playwrightOptions, ...journey.playwrightOptions } if (!journey.isMatch(match, tags)) { continue; } @@ -407,14 +433,72 @@ export default class Runner { * Execute dummy callback to get all monitor specific * configurations for the current journey */ - journey.callback({ params: options.params } as any); - journey.monitor.update(this.monitor?.config); + journey.callback({ params: params } as any); + journey.monitor.update({ + ...this.monitor?.config, + params: Object.keys(params).length ? params : undefined, + playwrightOptions + }); + + /* Only overwrite name and id values when using matrix */ + if (journey.matrix) { + journey.monitor.config.name = journey.name; + journey.monitor.config.id = journey.id; + journey.monitor.config.playwrightOptions = playwrightOptions; + } journey.monitor.validate(); monitors.push(journey.monitor); } return monitors; } + async parseMatrix(options: RunOptions) { + this.journeys.forEach(journey => { + const { matrix: globalMatrix } = options; + const { matrix: localMatrix } = journey; + // local journey matrix takes priority over global matrix + const matrix = localMatrix || globalMatrix; + + if (!matrix) { + return; + } + + if (!matrix.adjustments) { + throw new Error('Please specify adjustments for your testing matrix'); + } + + + if (matrix.adjustments.some(adjustment => !adjustment.name)) { + throw new Error('Please specify a name for each adjustment'); + } + + const suite = this.suites.get(journey.location); + suite.clearJourneys(); + + const combinations = getCombinations(matrix); + combinations.forEach(matrixParams => { + const j = journey.clone(); + const { playwrightOptions, name, params } = matrixParams; + if (playwrightOptions) { + j.playwrightOptions = { ...options.playwrightOptions, ...playwrightOptions } + } + j.name = `${j.name} - ${name}`; + j.id = `${j.id}-${name}`; + j.params = params; + j.matrix = matrix; + this.addJourney(j); + }); + }) + } + + getAllJourneys() { + const journeys = Array.from(this.suites.values()).reduce((acc, suite) => { + const suiteJourneys = suite.entries; + return [...acc, ...suiteJourneys]; + }, []); + return journeys; + } + async run(options: RunOptions) { const result: RunResult = {}; if (this.#active) { @@ -429,7 +513,12 @@ export default class Runner { }).catch(e => (this.hookError = e)); const { dryRun, match, tags } = options; - for (const journey of this.journeys) { + + this.parseMatrix(options); + + const journeys = this.getAllJourneys(); + + for (const journey of journeys) { /** * Used by heartbeat to gather all registered journeys */ diff --git a/src/dsl/journey.ts b/src/dsl/journey.ts index b9bb7200..6283d4e3 100644 --- a/src/dsl/journey.ts +++ b/src/dsl/journey.ts @@ -32,13 +32,14 @@ import { } from 'playwright-chromium'; import micromatch, { isMatch } from 'micromatch'; import { Step } from './step'; -import { VoidCallback, HooksCallback, Params, Location } from '../common_types'; +import { VoidCallback, HooksCallback, Params, Location, Matrix, PlaywrightOptions } from '../common_types'; import { Monitor, MonitorConfig } from './monitor'; export type JourneyOptions = { name: string; id?: string; tags?: string[]; + matrix?: Matrix; }; type HookType = 'before' | 'after'; @@ -56,23 +57,30 @@ export class Journey { name: string; id?: string; tags?: string[]; + matrix: Matrix; callback: JourneyCallback; location?: Location; steps: Step[] = []; hooks: Hooks = { before: [], after: [] }; monitor: Monitor; + params: Params = {}; + parent!: Suite; + playwrightOptions: PlaywrightOptions = {}; constructor( options: JourneyOptions, callback: JourneyCallback, - location?: Location + location?: Location, + params?: Params, ) { this.name = options.name; this.id = options.id || options.name; this.tags = options.tags; + this.matrix = options.matrix; this.callback = callback; this.location = location; this.updateMonitor({}); + this.params = params; } addStep(name: string, callback: VoidCallback, location?: Location) { @@ -120,4 +128,55 @@ export class Journey { const matchess = micromatch(this.tags || ['*'], pattern); return matchess.length > 0; } + + private _serialize(): any { + return { + options: { + name: this.name, + id: this.id, + tags: this.tags, + }, + callback: this.callback, + location: this.location, + steps: this.steps, + hooks: this.hooks, + monitor: this.monitor, + params: this.params, + }; + } + + clone(): Journey { + const data = this._serialize(); + const test = Journey._parse(data); + return test; + } + + static _parse(data: any): Journey { + const journey = new Journey(data.options, data.callback, data.location, data.params); + return journey; + } +} + +export class Suite { + location: Location; + private _entries: Journey[] = []; + + constructor( + location: Location, + ) { + this.location = location + } + + get entries() { + return this._entries; + } + + addJourney(journey: Journey) { + journey.parent = this; + this._entries.push(journey); + } + + clearJourneys() { + this._entries = []; + } } diff --git a/src/loader.ts b/src/loader.ts index ec378308..2a3ad617 100644 --- a/src/loader.ts +++ b/src/loader.ts @@ -71,6 +71,7 @@ export async function loadTestFiles(options: CliArgs, args: string[]) { loadInlineScript(source); return; } + /** * Handle piped files by reading the STDIN * ex: ls example/suites/*.js | npx @elastic/synthetics diff --git a/src/matrix.ts b/src/matrix.ts new file mode 100644 index 00000000..108ecbeb --- /dev/null +++ b/src/matrix.ts @@ -0,0 +1,99 @@ +/** + * MIT License + * + * Copyright (c) 2020-present, Elastic NV + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +// import { createHash } from 'crypto'; +import type { Matrix } from './common_types'; + +export const getCombinations = (matrix: Matrix): Matrix['adjustments'] => { + const { adjustments } = matrix; + + // no longer need this logic when removing values + + // const matrixKeys = Object.keys(matrix.values); + // const entries = Object.values(values); + + // let combinations = calculateCombinations(entries[0]); + // for (let i = 1; i < entries.length; i++) { + // combinations = calculateCombinations(combinations, entries[i]); + // } + + // const matrixParams = combinations.map(combination => { + // return getCombinationParams(matrixKeys, combination); + // }); + + // if (!adjustments) { + // return matrixParams; + // } + + // const currentCombinations = new Set(matrixParams.map(params => { + // const hash = createHash('sha256'); + // const paramHash = hash.update(JSON.stringify(params)).digest('base64'); + // return paramHash; + // })); + + // adjustments.forEach(adjustment => { + // const hash = createHash('sha256'); + // const adjustmentHash = hash.update(JSON.stringify(adjustment)).digest('base64'); + // if (!currentCombinations.has(adjustmentHash)) { + // matrixParams.push(adjustment); + // } + // }); + + return adjustments; +} + +export const calculateCombinations = (groupA: Array, groupB?: Array): Array => { + const results = []; + groupA.forEach(optionA => { + if (!groupB) { + results.push([optionA]); + return; + } + groupB.forEach(optionB => { + if (Array.isArray(optionA)) { + return results.push([...optionA, optionB]) + } else { + return results.push([optionA, optionB]) + } + }); + }); + return results; +} + +export const getCombinationName = (name: string, combinations: Record) => { + const values = Object.values(combinations); + return values.reduce((acc, combination) => { + const nameAdjustment = typeof combination === 'object' ? JSON.stringify(combination) : combination.toString(); + acc += ` - ${nameAdjustment.toString()}`; + return acc; + }, name).trim(); +} + +export const getCombinationParams = (keys: string[], values: unknown[]): Record => { + return keys.reduce>((acc, key, index) => { + acc[key] = values[index]; + return acc; + }, {}); +} diff --git a/src/options.ts b/src/options.ts index f9549437..f6241529 100644 --- a/src/options.ts +++ b/src/options.ts @@ -100,6 +100,11 @@ export function normalizeOptions(cliArgs: CliArgs): RunOptions { */ options.params = Object.freeze(merge(config.params, cliArgs.params || {})); + /** + * Grab matrix only from config and not cliArgs + */ + options.matrix = Object.freeze(config.matrix); + /** * Merge playwright options from CLI and Synthetics config * and prefer individual options over other option