Improve dev experience (#1854)

* WIP switch to yarn

* WIP switch to yarn

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* confirm node and TS versions on CI

* Fix

* WIP

* WIP

* WIP

* WIP

* WIP

* shrink the test matrix more

* fix

* fix

Author: cspotcode

.github/workflows/continuous-integration.yml

@@ -20,22 +20,25 @@ jobs:
       # checkout code
       - uses: actions/checkout@v2
       # install node
-      - name: Use Node.js 16
+      - name: Use Node.js 18
         uses: actions/setup-node@v1
         with:
-          node-version: 16
-      - run: |
-          npm config set cache "$( node -p "process.cwd()" )/temp/npm-cache"
+          node-version: 18
+      - name: Get cache directory
+        id: get-cache-directory
+        run: |
+          yarn config get cacheFolder
+          echo "::set-output name=path::$( yarn config get cacheFolder )"
       - name: Cache dependencies
         uses: actions/cache@v2
         with:
-          path: temp/npm-cache
-          key: npm-cache-packaging-${{ hashFiles('package-lock.json') }}
-          restore-keys: npm-cache-packaging-
+          path: ${{ steps.get-cache-directory.outputs.path }}
+          key: yarn-cache-packaging-${{ hashFiles('yarn.lock') }}
+          restore-keys: yarn-cache-packaging-
       # lint, build, test
-      - run: npm ci
-      - run: npm run lint
-      - run: npm run build-pack
+      - run: yarn install --immutable
+      - run: yarn lint
+      - run: yarn build
       - name: Upload package artifact
         uses: actions/upload-artifact@v1
         with:
@@ -46,20 +49,23 @@ jobs:
     needs: lint-build
     name: "Test: ${{ matrix.os }}, node ${{ matrix.node }}, TS ${{ matrix.typescript }}"
     runs-on: ${{ matrix.os }}-latest
+    env:
+      TEST_MATRIX_NODE_VERSION: ${{ matrix.node }}
+      TEST_MATRIX_TYPESCRIPT_VERSION: ${{ matrix.typescript }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu, windows]
         # Don't forget to add all new flavors to this list!
-        flavor: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
+        flavor: [2, 3, 4, 6, 8, 9, 10, 11]
         include:
           # Node 14.13.0
           # To test ESM builtin module resolution immediately before a node behavioral change: https://github.com/TypeStrong/ts-node/issues/1130
-          - flavor: 1
-            node: 14.13.0
-            nodeFlag: 14_13_0
-            typescript: latest
-            typescriptFlag: latest
+          # - flavor: 1
+          #   node: 14.13.0
+          #   nodeFlag: 14_13_0
+          #   typescript: latest
+          #   typescriptFlag: latest
           # Node 14
           - flavor: 2
             node: 14
@@ -79,23 +85,23 @@ jobs:
           # Node 16
           # Node 16.11.1
           # Earliest version that supports old ESM Loader Hooks API: https://github.com/TypeStrong/ts-node/pull/1522
-          - flavor: 5
-            node: 16.11.1
-            nodeFlag: 16_11_1
-            typescript: latest
-            typescriptFlag: latest
+          # - flavor: 5
+          #   node: 16.11.1
+          #   nodeFlag: 16_11_1
+          #   typescript: latest
+          #   typescriptFlag: latest
           - flavor: 6
             node: 16
             nodeFlag: 16
             typescript: latest
             typescriptFlag: latest
             downgradeNpm: true
-          - flavor: 7
-            node: 16
-            nodeFlag: 16
-            typescript: 4.0
-            typescriptFlag: 4_0
-            downgradeNpm: true
+          # - flavor: 7
+          #   node: 16
+          #   nodeFlag: 16
+          #   typescript: 4.0
+          #   typescriptFlag: 4_0
+          #   downgradeNpm: true
           - flavor: 8
             node: 16
             nodeFlag: 16
@@ -153,40 +159,40 @@ jobs:
           Invoke-WebRequest -Uri $url -OutFile $targetPath -UserAgent ([Microsoft.PowerShell.Commands.PSUserAgent]::Chrome)
           node --version
       # lint, build, test
-        # Downgrade from npm 7 to 6 because 7 still seems buggy to me
-      - if: ${{ matrix.downgradeNpm }}
-        run: npm install -g npm@6
-      - run: |
-          npm config set cache "$( node -p "process.cwd()" )/temp/npm-cache"
+      - name: Get cache directory
+        id: get-cache-directory
+        run: |
+          yarn config get cacheFolder
+          echo "::set-output name=path::$( yarn config get cacheFolder )"
       - name: Cache dependencies
         if: ${{ matrix.os != 'windows' }}
         uses: actions/cache@v2
         with:
-          path: temp/npm-cache
-          key: npm-cache-${{ matrix.os }}-${{ hashFiles('package-lock.json') }}
-          restore-keys: npm-cache-${{matrix.os }}-
-      - run: npm ci --ignore-scripts
-      - name: Upload npm logs
+          path: ${{ steps.get-cache-directory.outputs.path }}
+          key: yarn-cache-${{ matrix.os }}-${{ hashFiles('yarn.lock') }}
+          restore-keys: yarn-cache-${{matrix.os }}-
+      - run: yarn install --immutable --mode=skip-build
+      - name: Upload yarn logs
         if: ${{ failure() }}
         uses: actions/upload-artifact@v1
         with:
-          name: npm-logs
-          path: temp/npm-cache/_logs
-      - run: npm run build-tsc
+          name: yarn-logs
+          path: yarn-error.log
+      - run: yarn build-tsc
       - name: Download package artifact
         uses: actions/download-artifact@v1
         with:
           name: ts-node-packed.tgz
           path: tests/
-      - run: npm install typescript@${{ matrix.typescript }} --force
-      - run: npm run test-cov
-      - name: Upload npm logs
+      - run: yarn add -D typescript@${{ matrix.typescript }}
+      - run: yarn test-cov
+      - name: Upload yarn logs
         if: ${{ failure() }}
         uses: actions/upload-artifact@v1
         with:
-          name: npm-logs-${{ matrix.os }}-node-${{ matrix.nodeFlag }}-typescript-${{ matrix.typescriptFlag }}
-          path: temp/npm-cache/_logs
-      - run: npm run coverage-report
+          name: yarn-logs-${{ matrix.os }}-node-${{ matrix.nodeFlag }}-typescript-${{ matrix.typescriptFlag }}
+          path: yarn-error.log
+      - run: yarn coverage-report
         if: ${{ always() }}
       - name: Codecov
         if: ${{ always() }}

.gitignore

@@ -13,3 +13,6 @@ npm-debug.log
 /website/static/api
 /tsconfig.tsbuildinfo
 /temp
+/yarn-error.log
+/.yarn/install-state.gz
+/tests/.yarn/install-state.gz

.yarn/plugins/@yarnpkg/plugin-compat.cjs

@@ -0,0 +1,6 @@
+// No-op plugin to disable yarn's own plugin-compat, preventing any unexpected
+// modifications to the typescript package and others
+module.exports = {
+  name: `@yarnpkg/plugin-compat`,
+  factory: require => ({})
+};

.yarn/releases/yarn-3.2.2.cjs

@@ -0,0 +1,13 @@
+enableGlobalCache: true
+
+nmMode: hardlinks-global
+
+nodeLinker: node-modules
+
+plugins:
+  - path: .yarn/plugins/@yarnpkg/plugin-compat.cjs
+    spec: .yarn/plugins/@yarnpkg/plugin-compat.cjs
+
+yarnPath: .yarn/releases/yarn-3.2.2.cjs
+
+nmHoistingLimits: workspaces

.yarnrc.yml

@@ -1,17 +1,24 @@
 *This guide is best-effort and will be improved as necessary.*
 
+## Dev environment
+
+Tools I use:
+
+- yarn 3 for package management
+- volta (optional, node version management)
+
 ## Features, bugfixes, and other code
 
-We use npm scripts for building, testing, and linting.  Read the scripts to become familiar with our build process.  The big ones are:
+We use package.json scripts for building, testing, and linting.  Read the scripts to become familiar with our build process.  The big ones are:
 
 ```
-npm install
-npm run build
-npm run test-local
-npm run lint-fix
+yarn
+yarn build
+yarn test-local
+yarn fmt
 ```
 
-`npm prepare` is maintained so that anyone can install `ts-node` from git, which is useful for testing experimental branches and unreleased features.
+`npm prepare` / `yarn prepare` is maintained so that anyone can install `ts-node` from git, which is useful for testing experimental branches and unreleased features.
 
 Source lives in `src` and is compiled to `dist`.  Some shim files live outside of `src` so that they can be imported at
 certain paths.  For example, to allow users to import `ts-node/register`, we have `register/index.js` which is a shim to
@@ -21,20 +28,20 @@ compiled code in `dist`.
 
 ## Tests
 
-Test cases are declared in `src/test/*.spec.ts`, and test fixtures live in `./tests`.  They can be run with `npm run test-local`.
+Test cases are declared in `src/test/*.spec.ts`, and test fixtures live in `./tests`.  They can be run with `yarn test-local`.
 
 To run a subset of tests:
 
 ```
 # Use ava's --match flag to match the name of a test or suite
 # https://github.com/avajs/ava/blob/main/docs/05-command-line.md
 # Don't forget the * wildcards
-npm run test-local -- --match '*esm loader*'
+yarn test-local --match '*esm loader*'
 ```
 
 Tests are run with AVA, but using a custom wrapper API to enable some TS-friendly features and grouped test suites.
 
-The tests `npm pack` ts-node into a tarball and `npm install` it into `./tests/node_modules`.  This makes `./tests` a better testing environment
+The tests `yarn pack` ts-node into a tarball and `yarn install` it into `./tests/node_modules`.  This makes `./tests` a better testing environment
 because it more closely matches the end-user's environment.  Complex `require()` / `import` / `--require` / `--loader` invocations behave
 the way they would in a users's project.
 
@@ -120,7 +127,7 @@ We publish using `np`: https://npm.im/np
 
 ## APIExtractor
 
-`npm run api-extractor` will update an API report generated by [`api-extractor`](https://api-extractor.com/pages/overview/intro/) which may be useful
+`yarn api-extractor` will update an API report generated by [`api-extractor`](https://api-extractor.com/pages/overview/intro/) which may be useful
 when generating release notes to detect (breaking) changes in our API surface.
 
 I configured it for my own convenience; it is not a necessary part of our development process.

CONTRIBUTING.md

@@ -13,13 +13,13 @@ regenerate:
     const fs = require("fs");
     let acc = fs.readFileSync("justfile", "utf8").replace(/(# CUT\n)[\s\S]+/, "$1\n");
     for(const [key, value] of Object.entries(require("./package.json").scripts)) {
-      acc += `${key} *ARGS:\n  ${value.replace(/npm run /g, "just ").replace(/ --$/, "")} "$@"\n`;
+      acc += `${key} *ARGS:\n  ${value.replace(/(npm run |yarn )/g, "just ").replace(/ --$/, "")} "$@"\n`;
     }
     fs.writeFileSync("justfile", acc);
   '
 
 install:
-  npm install
+  yarn
 
 # EVERYTHING BELOW THIS LINE IS AUTO-GENERATED FROM PACKAGE.JSON
 # DO NOT MODIFY BY HAND
@@ -28,14 +28,14 @@ install:
 
 lint *ARGS:
   dprint check "$@"
-lint-fix *ARGS:
+fmt *ARGS:
   dprint fmt "$@"
 clean *ARGS:
-  rimraf temp dist tsconfig.schema.json tsconfig.schemastore-schema.json tsconfig.tsbuildinfo tests/ts-node-packed.tgz tests/tmp "$@"
+  rimraf temp dist tsconfig.schema.json tsconfig.schemastore-schema.json tsconfig.tsbuildinfo tests/ts-node-packed.tgz tests/node_modules tests/tmp "$@"
 rebuild *ARGS:
   just clean && just build "$@"
 build *ARGS:
-  just build-nopack && just build-pack "$@"
+  just build-nopack && just build-pack && just build-manifest "$@"
 build-nopack *ARGS:
   just build-tsc && just build-configSchema "$@"
 build-tsc *ARGS:
@@ -44,23 +44,23 @@ build-configSchema *ARGS:
   typescript-json-schema --topRef --refs --validationKeywords allOf --out tsconfig.schema.json tsconfig.build-schema.json TsConfigSchema && node --require ./register ./scripts/create-merged-schema "$@"
 build-pack *ARGS:
   node ./scripts/build-pack.js "$@"
+build-manifest *ARGS:
+  node ./scripts/build-manifest.mjs "$@"
 test-spec *ARGS:
   ava "$@"
 test-cov *ARGS:
   nyc ava "$@"
 test *ARGS:
   just build && just lint && just test-cov "$@"
 test-local *ARGS:
-  just lint-fix && just build-tsc && just build-pack && just test-spec "$@"
+  just fmt && just build-tsc && just build-pack && just test-spec "$@"
 pre-debug *ARGS:
   just build-tsc && just build-pack "$@"
 coverage-report *ARGS:
   nyc report --reporter=lcov "$@"
-prepare *ARGS:
+__prepare_template__ *ARGS:
   just clean && just build-nopack "$@"
+prepare *ARGS:
+  rimraf temp dist tsconfig.schema.json tsconfig.schemastore-schema.json tsconfig.tsbuildinfo tests/ts-node-packed.tgz tests/node_modules tests/tmp && tsc -b ./tsconfig.build-dist.json && typescript-json-schema --topRef --refs --validationKeywords allOf --out tsconfig.schema.json tsconfig.build-schema.json TsConfigSchema && node --require ./register ./scripts/create-merged-schema "$@"
 api-extractor *ARGS:
   api-extractor run --local --verbose "$@"
-esm-usage-example *ARGS:
-  just build-tsc && cd esm-usage-example && node --experimental-specifier-resolution node --loader ../esm.mjs ./index "$@"
-esm-usage-example2 *ARGS:
-  just build-tsc && cd tests && TS_NODE_PROJECT=./module-types/override-to-cjs/tsconfig.json node --loader ../esm.mjs ./module-types/override-to-cjs/test.cjs "$@"