Parallel tests (#2968)

Author: holiber

azure-pipelines.yml

@@ -6,7 +6,41 @@ variables:
   CI: true
 
 jobs:
-  - job: Tests
+  - job: Test
+    pool:
+      name: 'Default'
+
+    workspace:
+      clean: resources
+
+    strategy:
+      parallel: 4
+
+    steps:
+      - powershell: echo "IP for RDP connections:" $(Invoke-RestMethod ipinfo.io/ip)
+        displayName: 'Get RDP address'
+
+      - powershell: Set-DisplayResolution -Width 1920 -Height 1080 -Force
+        displayName: 'Setup Screen Resolution'
+
+      - script: yarn install --frozen-lockfile --check-files 2>&1
+        displayName: 'Install Dependencies'
+
+      - script: node node_modules/electron/install.js
+        displayName: 'Install Electron'
+
+      - script: 'yarn ci:compile'
+        displayName: 'Compile Assets'
+
+      - script: 'yarn ci:tests'
+        displayName: 'Run Tests'
+        env:
+          SLOBS_TEST_USER_POOL_TOKEN: $(userPoolToken)
+          BROWSER_SOURCE_HARDWARE_ACCELERATION: 'OFF'
+          SLOBS_TEST_STREAM_KEY: $(twitchStreamKey)
+          SLOBS_TEST_RUN_CHUNK: "$(System.JobPositionInPhase)/$(System.TotalJobsInPhase)"
+
+  - job: Check_eslint_and_strict_nulls_regression
 
     pool:
       name: 'Default'
@@ -25,26 +59,13 @@ jobs:
     - script: yarn install --frozen-lockfile --check-files 2>&1
       displayName: 'Install Dependencies'
 
-    - script: node node_modules/electron/install.js
-      displayName: 'Install Electron'
-
     - script: 'yarn eslint'
       displayName: 'ESLint'
 
     - script: 'yarn compile:strictnulls'
       displayName: 'Check regressions in files with strictNullChecks'
 
-    - script: 'yarn ci:compile'
-      displayName: 'Compile Assets'
-
-    - script: 'yarn ci:tests'
-      displayName: 'Run Tests'
-      env:
-        SLOBS_TEST_USER_POOL_TOKEN: $(userPoolToken)
-        BROWSER_SOURCE_HARDWARE_ACCELERATION: 'OFF'
-        SLOBS_TEST_STREAM_KEY: $(twitchStreamKey)
-
-#  - job: Screenshots
+#  - job: Performance
 #
 #    pool:
 #      name: 'Default'
@@ -54,64 +75,26 @@ jobs:
 #
 #    steps:
 #
-#    - powershell: echo "IP for RDP connections:" $(Invoke-RestMethod ipinfo.io/ip)
-#      displayName: 'Get RDP address'
-#
-#    - powershell: Set-DisplayResolution -Width 1920 -Height 1080 -Force
-#      displayName: 'Setup Screen Resolution'
+#      - powershell: echo "IP for RDP connections:" $(Invoke-RestMethod ipinfo.io/ip)
+#        displayName: 'Get RDP address'
 #
-#    - script: yarn install --frozen-lockfile --check-files 2>&1
-#      displayName: 'Install Dependencies'
+#      - powershell: Set-DisplayResolution -Width 1920 -Height 1080 -Force
+#        displayName: 'Setup Screen Resolution'
 #
-#    - script: node node_modules/electron/install.js
-#      displayName: 'Install Electron'
+#      - script: yarn install --frozen-lockfile --check-files 2>&1
+#        displayName: 'Install Dependencies'
 #
-#    - script: 'yarn ci:screentests'
-#      displayName: 'Snapshot testing'
-#      env:
-#        BROWSER_SOURCE_HARDWARE_ACCELERATION: 'OFF'
-#        SLOBS_TEST_USER_POOL_TOKEN: $(userPoolToken)
-#        STREAMLABS_BOT_ID: $(streamlabsBotId)
-#        STREAMLABS_BOT_KEY: $(streamlabsBotKey)
-#        AWS_BUCKET: $(awsBucket)
-#        AWS_ACCESS_KEY: $(awsAccessKey)
-#        AWS_SECRET_KEY: $(awsSecretKey)
+#      - script: node node_modules/electron/install.js
+#        displayName: 'Install Electron'
 #
-#    - task: PublishPipelineArtifact@0
-#      inputs:
-#        artifactName: 'SnapshotsResults'
-#        targetPath: $(Build.SourcesDirectory)/test-dist/screentest
+#      - script: 'yarn ci:compile'
+#        displayName: 'Compile Assets'
 #
-  - job: Performance
-
-    pool:
-      name: 'Default'
-
-    workspace:
-      clean: resources
-
-    steps:
-
-      - powershell: echo "IP for RDP connections:" $(Invoke-RestMethod ipinfo.io/ip)
-        displayName: 'Get RDP address'
-
-      - powershell: Set-DisplayResolution -Width 1920 -Height 1080 -Force
-        displayName: 'Setup Screen Resolution'
-
-      - script: yarn install --frozen-lockfile --check-files 2>&1
-        displayName: 'Install Dependencies'
-
-      - script: node node_modules/electron/install.js
-        displayName: 'Install Electron'
-
-      - script: 'yarn ci:compile'
-        displayName: 'Compile Assets'
-
-      - script: 'yarn ci:performance'
-        displayName: 'Performance testing'
-        env:
-          BROWSER_SOURCE_HARDWARE_ACCELERATION: 'OFF'
-          SLOBS_TEST_USER_POOL_TOKEN: $(userPoolToken)
+#      - script: 'yarn ci:performance'
+#        displayName: 'Performance testing'
+#        env:
+#          BROWSER_SOURCE_HARDWARE_ACCELERATION: 'OFF'
+#          SLOBS_TEST_USER_POOL_TOKEN: $(userPoolToken)
 
   - job: macOS
 

test-job-template.yml

@@ -1,5 +1,6 @@
 /**
  * Tests runner script:
+ * - fetch average test timings from the DB
  * - run tests
  * - if some tests failed retry only these tests
  */
@@ -8,33 +9,37 @@ const { execSync } = require('child_process');
 const fs = require('fs');
 const rimraf = require('rimraf');
 const request = require('request');
+const fetch = require('node-fetch');
 
 const failedTestsFile = 'test-dist/failed-tests.json';
 const args = process.argv.slice(2);
 const TIMEOUT = 3; // timeout in minutes
 
-(function main() {
+(async function main() {
   try {
     rimraf.sync(failedTestsFile);
+    await createTestTimingsFile();
     execSync(`yarn test --timeout=${TIMEOUT}m ` + args.join(' '), { stdio: [0, 1, 2] });
   } catch (e) {
+    console.log(e);
     retryTests();
   }
 })();
 
-
 function retryTests() {
   log('retrying failed tests');
 
   if (!fs.existsSync(failedTestsFile)) {
     throw 'no tests to retry';
   }
 
-  const failedTests = JSON.parse(fs.readFileSync(failedTestsFile));
+  const failedTests = JSON.parse(fs.readFileSync(failedTestsFile, 'utf8'));
   const retryingArgs = failedTests.map(testName => `--match="${testName}"`);
   let retryingFailed = false;
   try {
-    execSync(`yarn test --timeout=${TIMEOUT}m ` + args.concat(retryingArgs).join(' '), { stdio: [0, 1, 2] });
+    execSync(`yarn test --timeout=${TIMEOUT}m ` + args.concat(retryingArgs).join(' '), {
+      stdio: [0, 1, 2],
+    });
     log('retrying succeed');
   } catch (e) {
     retryingFailed = true;
@@ -44,7 +49,6 @@ function retryTests() {
   sendFailedTestsToAnalytics(failedTests).then(() => {
     if (retryingFailed) failAndExit();
   });
-
 }
 
 function log(...args) {
@@ -58,7 +62,6 @@ function failAndExit() {
 function sendFailedTestsToAnalytics(failedTests) {
   log('Sending analytics..');
   return new Promise((resolve, reject) => {
-
     const options = {
       url: 'https://r2d2.streamlabs.com/slobs/data/ping',
       json: {
@@ -69,13 +72,13 @@ function sendFailedTestsToAnalytics(failedTests) {
             product: 'SLOBS',
             version: this.version,
             count: 1,
-            uuid: 'test environment'
-          }
-        ]
-      }
+            uuid: 'test environment',
+          },
+        ],
+      },
     };
 
-    const callback = (error) => {
+    const callback = error => {
       if (error) {
         console.error('Analytics has not been sent');
         resolve();
@@ -88,3 +91,38 @@ function sendFailedTestsToAnalytics(failedTests) {
     request(options, callback);
   });
 }
+
+/**
+ * Fetch average execution timings for tests from DB
+ * and save results to a file
+ */
+async function createTestTimingsFile() {
+  const utilsServerUrl = 'https://slobs-users-pool.herokuapp.com';
+  const token = process.env.SLOBS_TEST_USER_POOL_TOKEN;
+  const testTimingsFile = 'test-dist/test-timings.json';
+  rimraf.sync(testTimingsFile);
+
+  return new Promise((resolve, reject) => {
+    fetch(`${utilsServerUrl}/testStats`, {
+      method: 'get',
+      headers: {
+        Authorization: `Bearer ${token}`,
+        'Content-Type': 'application/json',
+      },
+    })
+      .then(res => {
+        if (res.status !== 200) {
+          reject('Unable to request the utility server', res);
+        } else {
+          res.json().then(data => {
+            if (!fs.existsSync('test-dist')) {
+              fs.mkdirSync('test-dist');
+            }
+            fs.writeFileSync(testTimingsFile, JSON.stringify(data));
+            resolve();
+          });
+        }
+      })
+      .catch(e => reject(`Utility server is not available ${e}`));
+  });
+}