feat: [OSM-1024] pnpm dep graph builder

Author: gemaxim

.circleci/config.yml

@@ -4,6 +4,12 @@ orbs:
   win: circleci/[email protected]
   prodsec: snyk/[email protected]
 
+filters_branches_ignore_master: &filters_branches_ignore_master
+  filters:
+    branches:
+      ignore:
+        - master
+
 defaults: &defaults
   parameters:
     node_version:
@@ -17,6 +23,9 @@ windows_defaults: &windows_defaults
   executor:
     name: win/default
 
+test_matrix: &test_matrix
+  node_version: ['12.22.12', '14.17.6', '16.13.2']
+
 commands:
   install_deps:
     description: Install dependencies
@@ -66,7 +75,7 @@ jobs:
           name: Run lint
           command: npm run lint
 
-  test-windows:
+  test-windows-jest:
     <<: *defaults
     <<: *windows_defaults
     steps:
@@ -80,9 +89,39 @@ jobs:
       - show_node_version
       - run:
           name: Run tests
-          command: npm test
+          command: npm run test:jest
 
-  test-unix:
+  test-windows-tap:
+    <<: *defaults
+    <<: *windows_defaults
+    steps:
+      - run: git config --global core.autocrlf false
+      - install_node_npm:
+          node_version: << parameters.node_version >>
+      - checkout
+      - attach_workspace:
+          at: ~/nodejs-lockfile-parser
+      - install_deps
+      - show_node_version
+      - run:
+          name: Run tests
+          command: npm run unit-test
+
+  test-unix-jest:
+    <<: *defaults
+    docker:
+      - image: cimg/node:<< parameters.node_version >>
+    steps:
+      - checkout
+      - attach_workspace:
+          at: ~/nodejs-lockfile-parser
+      - install_deps
+      - show_node_version
+      - run:
+          name: Run tests
+          command: npm run test:jest
+
+  test-unix-tap:
     <<: *defaults
     docker:
       - image: cimg/node:<< parameters.node_version >>
@@ -94,7 +133,7 @@ jobs:
       - show_node_version
       - run:
           name: Run tests
-          command: npm test
+          command: npm run unit-test
 
   release:
     <<: *defaults
@@ -121,70 +160,47 @@ workflows:
           name: Lint
           context: nodejs-install
           node_version: "16.13.2"
-          filters:
-            branches:
-              ignore:
-                - master
-      - test-windows:
-          name: Windows Tests for Node v16 support
-          context: nodejs-install
-          node_version: "16.13.2"
-          requires:
-            - Lint
-          filters:
-            branches:
-              ignore:
-                - master
-      - test-windows:
-          name: Windows Tests for Node v14 support
-          context: nodejs-install
-          node_version: "14.17.6"
-          requires:
-            - Lint
-          filters:
-            branches:
-              ignore:
-                - master
-      - test-unix:
-          name: Unix Tests for Node v16 support
+          <<: *filters_branches_ignore_master
+      - test-windows-jest:
+          matrix:
+            alias: test-windows-jest
+            parameters:
+              <<: *test_matrix
+          name: Windows Tests (Jest) for Node=<< matrix.node_version >> support
           context: nodejs-install
-          node_version: "16.13.2"
           requires:
             - Lint
-          filters:
-            branches:
-              ignore:
-                - master
-      - test-unix:
-          name: Unix Tests for Node v14 support
+          <<: *filters_branches_ignore_master
+      - test-windows-tap:
+          matrix:
+            alias: test-windows-tap
+            parameters:
+              <<: *test_matrix
+          name: Windows Tests (Tap) for Node=<< matrix.node_version >> support
           context: nodejs-install
-          node_version: "14.17.6"
           requires:
             - Lint
-          filters:
-            branches:
-              ignore:
-                - master
-      - test-windows:
-          name: Windows Tests for Node v12 support
+          <<: *filters_branches_ignore_master
+      - test-unix-jest:
+          matrix:
+            alias: test-unix-jest
+            parameters:
+              <<: *test_matrix
+          name: Unix Tests (Jest) for Node=<< matrix.node_version >> support
           context: nodejs-install
-          node_version: "12.22.12"
           requires:
             - Lint
-          filters:
-            branches:
-              ignore:
-                - master
-      - test-unix:
-          name: Unix Tests for Node v12 support
+          <<: *filters_branches_ignore_master
+      - test-unix-tap:
+          matrix:
+            alias: test-unix-tap
+            parameters:
+              <<: *test_matrix
+          name: Unix Tests (Tap) for Node=<< matrix.node_version >> support
           context: nodejs-install
-          node_version: "12.22.12"
           requires:
             - Lint
-          filters:
-            branches:
-              ignore:
-                - master
+          <<: *filters_branches_ignore_master
       - release:
           name: Release
           context: nodejs-app-release

.gitignore

@@ -9,3 +9,4 @@ dist
 .idea
 .nyc_output/
 .dccache
+coverage/

README.md

@@ -14,6 +14,7 @@ Dep graph generation supported for:
 
 - `package-lock.json` (at Versions 2 and 3)
 - `yarn.lock`
+- `pnpm-lock.yaml` (lockfileVersion 5.x or 6.x)
 
 Legacy dep tree supported for:
 

lib/dep-graph-builders/index.ts

@@ -13,6 +13,8 @@ import {
   extractPkgsFromYarnLockV2,
 } from './yarn-lock-v2';
 import { parseNpmLockV2Project } from './npm-lock-v2';
+import { parsePnpmProject } from './pnpm';
+import { parsePkgJson } from './util';
 
 export {
   parseNpmLockV2Project,
@@ -26,4 +28,6 @@ export {
   buildDepGraphYarnLockV2Simple,
   parseYarnLockV2Project,
   extractPkgsFromYarnLockV2,
+  parsePnpmProject,
+  parsePkgJson,
 };

lib/dep-graph-builders/pnpm/build-dep-graph-pnpm.ts

@@ -0,0 +1,147 @@
+import { DepGraphBuilder } from '@snyk/dep-graph';
+import { getTopLevelDeps } from '../util';
+import type {
+  Overrides,
+  PnpmProjectParseOptions,
+  PnpmWorkspaceArgs,
+} from '../types';
+import type { PackageJsonBase } from '../types';
+import { getPnpmChildNode } from './utils';
+import { eventLoopSpinner } from 'event-loop-spinner';
+import { PnpmLockfileParser } from './lockfile-parser/lockfile-parser';
+import { NormalisedPnpmPkgs, PnpmNode } from './types';
+
+export const buildDepGraphPnpm = async (
+  lockFileParser: PnpmLockfileParser,
+  pkgJson: PackageJsonBase,
+  options: PnpmProjectParseOptions,
+  workspaceArgs?: PnpmWorkspaceArgs,
+) => {
+  const { strictOutOfSync, includeOptionalDeps, pruneWithinTopLevelDeps } =
+    options;
+
+  const depGraphBuilder = new DepGraphBuilder(
+    { name: 'pnpm' },
+    { name: pkgJson.name, version: pkgJson.version },
+  );
+
+  const extractedPnpmPkgs: NormalisedPnpmPkgs =
+    lockFileParser.extractedPackages;
+
+  const topLevelDeps = getTopLevelDeps(pkgJson, options);
+
+  const extractedTopLevelDeps =
+    lockFileParser.extractTopLevelDependencies(options) || {};
+
+  for (const name of Object.keys(topLevelDeps)) {
+    topLevelDeps[name].version = extractedTopLevelDeps[name].version;
+  }
+
+  const rootNode: PnpmNode = {
+    id: 'root-node',
+    name: pkgJson.name,
+    version: pkgJson.version,
+    dependencies: topLevelDeps,
+    isDev: false,
+  };
+
+  await dfsVisit(
+    depGraphBuilder,
+    rootNode,
+    extractedPnpmPkgs,
+    strictOutOfSync,
+    includeOptionalDeps,
+    // we have rootWorkspaceOverrides if this is workspace pkg with overrides
+    // at root - therefore it should take precedent
+    // TODO: inspect if this is needed at all, seems like pnpm resolves everything in lockfile
+    workspaceArgs?.rootOverrides || pkgJson.pnpm?.overrides || {},
+    pruneWithinTopLevelDeps,
+    lockFileParser,
+  );
+
+  return depGraphBuilder.build();
+};
+
+/**
+ * Use DFS to add all nodes and edges to the depGraphBuilder and prune cyclic nodes.
+ * The visitedMap keep track of which nodes have already been discovered during traversal.
+ *  - If a node doesn't exist in the map, it means it hasn't been visited.
+ *  - If a node is already visited, simply connect the new node with this node.
+ */
+const dfsVisit = async (
+  depGraphBuilder: DepGraphBuilder,
+  node: PnpmNode,
+  extractedPnpmPkgs: NormalisedPnpmPkgs,
+  strictOutOfSync: boolean,
+  includeOptionalDeps: boolean,
+  overrides: Overrides,
+  pruneWithinTopLevel: boolean,
+  lockFileParser: PnpmLockfileParser,
+  visited?: Set<string>,
+): Promise<void> => {
+  for (const [name, depInfo] of Object.entries(node.dependencies || {})) {
+    if (eventLoopSpinner.isStarving()) {
+      await eventLoopSpinner.spin();
+    }
+
+    const localVisited = visited || new Set<string>();
+
+    const childNode: PnpmNode = getPnpmChildNode(
+      name,
+      depInfo,
+      extractedPnpmPkgs,
+      strictOutOfSync,
+      includeOptionalDeps,
+      lockFileParser,
+    );
+
+    if (localVisited.has(childNode.id)) {
+      if (pruneWithinTopLevel) {
+        const prunedId = `${childNode.id}:pruned`;
+        depGraphBuilder.addPkgNode(
+          { name: childNode.name, version: childNode.version },
+          prunedId,
+          {
+            labels: {
+              scope: childNode.isDev ? 'dev' : 'prod',
+              pruned: 'true',
+              ...(node.missingLockFileEntry && {
+                missingLockFileEntry: 'true',
+              }),
+            },
+          },
+        );
+        depGraphBuilder.connectDep(node.id, prunedId);
+      } else {
+        depGraphBuilder.connectDep(node.id, childNode.id);
+      }
+      continue;
+    }
+
+    depGraphBuilder.addPkgNode(
+      { name: childNode.name, version: childNode.version },
+      childNode.id,
+      {
+        labels: {
+          scope: childNode.isDev ? 'dev' : 'prod',
+          ...(node.missingLockFileEntry && {
+            missingLockFileEntry: 'true',
+          }),
+        },
+      },
+    );
+    depGraphBuilder.connectDep(node.id, childNode.id);
+    localVisited.add(childNode.id);
+    await dfsVisit(
+      depGraphBuilder,
+      childNode,
+      extractedPnpmPkgs,
+      strictOutOfSync,
+      includeOptionalDeps,
+      overrides,
+      pruneWithinTopLevel,
+      lockFileParser,
+      localVisited,
+    );
+  }
+};