add basic support for testing matrix

Author: dominiqueclarke

src/common_types.ts

@@ -237,6 +237,7 @@ export type RunOptions = BaseArgs & {
   environment?: string;
   playwrightOptions?: PlaywrightOptions;
   networkConditions?: NetworkConditions;
+  matrix?: Matrix;
   reporter?: BuiltInReporterName | ReporterInstance;
 };
 
@@ -254,12 +255,18 @@ export type ProjectSettings = {
   space: string;
 };
 
+export type Matrix = {
+  values: Record<string, unknown[]>;
+  adjustments?: Array<Record<string, unknown>>;
+}
+
 export type PlaywrightOptions = LaunchOptions & BrowserContextOptions;
 export type SyntheticsConfig = {
   params?: Params;
   playwrightOptions?: PlaywrightOptions;
   monitor?: MonitorConfig;
   project?: ProjectSettings;
+  matrix?: Matrix;
 };
 
 /** Runner Payload types */

src/core/index.ts

@@ -22,14 +22,21 @@
  * THE SOFTWARE.
  *
  */
-
 import { Journey, JourneyCallback, JourneyOptions } from '../dsl';
 import Runner from './runner';
 import { VoidCallback, HooksCallback, Location } from '../common_types';
+import { normalizeOptions } from '../options';
 import { wrapFnWithLocation } from '../helpers';
+import { getCombinations, getCombinationName } from '../matrix';
 import { log } from './logger';
 import { MonitorConfig } from '../dsl/monitor';
 
+
+/* TODO: Testing
+ * Local vs global matrix: Local matrix fully overwrites global matrix, rather than merging
+ * Adjustments: Duplicates in adjustments do not run extra journeys
+ * 
+
 /**
  * Use a gloabl Runner which would be accessed by the runtime and
  * required to handle the local vs global invocation through CLI
@@ -38,7 +45,6 @@ const SYNTHETICS_RUNNER = Symbol.for('SYNTHETICS_RUNNER');
 if (!global[SYNTHETICS_RUNNER]) {
   global[SYNTHETICS_RUNNER] = new Runner();
 }
-
 export const runner: Runner = global[SYNTHETICS_RUNNER];
 
 export const journey = wrapFnWithLocation(
@@ -51,9 +57,27 @@ export const journey = wrapFnWithLocation(
     if (typeof options === 'string') {
       options = { name: options, id: options };
     }
-    const j = new Journey(options, callback, location);
-    runner.addJourney(j);
-    return j;
+    const { matrix: globalMatrix } = normalizeOptions({});
+    const { matrix: journeyMatrix } = options;
+    if (!globalMatrix && !journeyMatrix) {
+      const j = new Journey(options, callback, location);
+      runner.addJourney(j);
+      return j;
+    }
+
+    // local journey matrix takes priority over global matrix
+    const matrix = journeyMatrix || globalMatrix;
+
+    if (!matrix.values) {
+      throw new Error('Please specify values for your testing matrix');
+    }
+
+    const combinations = getCombinations(matrix);
+    combinations.forEach(matrixParams => {
+      const name = getCombinationName((options as JourneyOptions)?.name, matrixParams);
+      const j = new Journey({...options as JourneyOptions, name}, callback, location, matrixParams);
+      runner.addJourney(j);  
+    })
   }
 );
 

src/core/runner.ts

@@ -339,16 +339,18 @@ export default class Runner {
 
   async runJourney(journey: Journey, options: RunOptions) {
     const result: JourneyResult = { status: 'succeeded' };
-    const context = await Runner.createContext(options);
+    const params = {...options.params, ...journey.params};
+    const journeyOptions = { ...options, params};
+    const context = await Runner.createContext(journeyOptions);
     log(`Runner: start journey (${journey.name})`);
     try {
       this.registerJourney(journey, context);
       const hookArgs = {
         env: options.environment,
-        params: options.params,
+        params: params,
       };
       await this.runBeforeHook(journey, hookArgs);
-      const stepResults = await this.runSteps(journey, context, options);
+      const stepResults = await this.runSteps(journey, context, journeyOptions);
       /**
        * Mark journey as failed if any intermediate step fails
        */
@@ -363,7 +365,7 @@ export default class Runner {
       result.status = 'failed';
       result.error = e;
     } finally {
-      await this.endJourney(journey, { ...context, ...result }, options);
+      await this.endJourney(journey, { ...context, ...result }, journeyOptions);
       await Gatherer.dispose(context.driver);
     }
     log(`Runner: end journey (${journey.name})`);
@@ -399,6 +401,7 @@ export default class Runner {
     const { match, tags } = options;
     const monitors: Monitor[] = [];
     for (const journey of this.journeys) {
+      const params = { ...this.monitor.config?.params, ...options.params, ...journey.params };
       if (!journey.isMatch(match, tags)) {
         continue;
       }
@@ -407,8 +410,8 @@ export default class Runner {
        * Execute dummy callback to get all monitor specific
        * configurations for the current journey
        */
-      journey.callback({ params: options.params } as any);
-      journey.monitor.update(this.monitor?.config);
+      journey.callback({ params: params } as any);
+      journey.monitor.update({ ...this.monitor?.config, params });
       journey.monitor.validate();
       monitors.push(journey.monitor);
     }

src/dsl/journey.ts

@@ -32,13 +32,14 @@ import {
 } from 'playwright-chromium';
 import micromatch, { isMatch } from 'micromatch';
 import { Step } from './step';
-import { VoidCallback, HooksCallback, Params, Location } from '../common_types';
+import { VoidCallback, HooksCallback, Params, Location, Matrix } from '../common_types';
 import { Monitor, MonitorConfig } from './monitor';
 
 export type JourneyOptions = {
   name: string;
   id?: string;
   tags?: string[];
+  matrix?: Matrix;
 };
 
 type HookType = 'before' | 'after';
@@ -61,18 +62,21 @@ export class Journey {
   steps: Step[] = [];
   hooks: Hooks = { before: [], after: [] };
   monitor: Monitor;
+  params: Params = {};
 
   constructor(
     options: JourneyOptions,
     callback: JourneyCallback,
-    location?: Location
+    location?: Location,
+    params?: Params,
   ) {
     this.name = options.name;
     this.id = options.id || options.name;
     this.tags = options.tags;
     this.callback = callback;
     this.location = location;
     this.updateMonitor({});
+    this.params = params;
   }
 
   addStep(name: string, callback: VoidCallback, location?: Location) {

src/matrix.ts

@@ -0,0 +1,96 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import { createHash } from 'crypto';
+import type { Matrix } from './common_types';
+
+export const getCombinations = (matrix: Matrix): Array<Record<string, unknown>> => {
+  const { values, adjustments } = matrix;
+  const matrixKeys = Object.keys(matrix.values);
+  const entries = Object.values(values);
+
+  let combinations = calculateCombinations(entries[0]);
+  for (let i = 1; i < entries.length; i++) {
+    combinations = calculateCombinations(combinations, entries[i]);
+  }
+
+  const matrixParams = combinations.map(combination => {
+    return getCombinationParams(matrixKeys, combination);
+  });
+
+  if (!adjustments) {
+    return matrixParams;
+  }
+
+  const currentCombinations = new Set(matrixParams.map(params => {
+    const hash = createHash('sha256');
+    const paramHash = hash.update(JSON.stringify(params)).digest('base64');
+    return paramHash;
+  }));
+
+  adjustments.forEach(adjustment => {
+    const hash = createHash('sha256');
+    const adjustmentHash = hash.update(JSON.stringify(adjustment)).digest('base64');
+    if (!currentCombinations.has(adjustmentHash)) {
+      matrixParams.push(adjustment);
+    }
+  });
+
+  return matrixParams;
+}
+
+export const calculateCombinations = (groupA: Array<unknown | unknown[]>, groupB?: Array<unknown>): Array<unknown[]> => {
+  const results = [];
+  groupA.forEach(optionA => {
+    if (!groupB) {
+      results.push([optionA]);
+      return;
+    }
+    groupB.forEach(optionB => {
+      if (Array.isArray(optionA)) {
+        return results.push([...optionA, optionB])
+      } else {
+        return results.push([optionA, optionB])
+      }
+    });
+  });
+  return results;
+}
+
+export const getCombinationName = (name: string, combinations: Record<string, unknown>) => {
+  const values = Object.values(combinations);
+  return values.reduce<string>((acc, combination) => {
+    const nameAdjustment = typeof combination === 'object' ? JSON.stringify(combination) : combination.toString();
+    acc += ` - ${nameAdjustment.toString()}`;
+    return acc;
+  }, name).trim();
+}
+
+export const getCombinationParams = (keys: string[], values: unknown[]): Record<string, unknown> => {
+  return keys.reduce<Record<string, unknown>>((acc, key, index) => {
+    acc[key] = values[index];
+    return acc;
+  }, {});
+}

src/options.ts

@@ -100,6 +100,11 @@ export function normalizeOptions(cliArgs: CliArgs): RunOptions {
    */
   options.params = Object.freeze(merge(config.params, cliArgs.params || {}));
 
+  /**
+   * Grab matrix only from config and not cliArgs
+   */
+  options.matrix = Object.freeze(config.matrix);
+
   /**
    * Merge playwright options from CLI and Synthetics config
    * and prefer individual options over other option