Update esm loader hooks API (#1457)

* Initial commit

* Update hooks

* wip impl of load

* Expose old hooks for backward compat

* Some logging

* Add raw copy of default get format

* Adapt defaultGetFormat() from node source

* Fix defaultTransformSource

* Add missing newline

* Fix require

* Check node version to avoid deprecation warning

* Remove load from old hooks

* Add some comments

* Use versionGte

* Remove logging

* Refine comments

* Wording

* Use format hint if available

* One more comment

* Nitpicky changes to comments

* Update index.ts

* lint-fix

* attempt at downloading node nightly in tests

* fix

* fix

* Windows install of node nightly

* update version checks to be ready for node backporting

* Add guards for undefined source

* More error info

* Skip source transform for builtin and commonjs

* Update transpile-only.mjs

* Tweak `createEsmHooks` type

* fix test to accomodate new api

Co-authored-by: Andrew Bradley <[email protected]>

Author: jonaskello

.github/workflows/continuous-integration.yml

@@ -48,7 +48,7 @@ jobs:
       matrix:
         os: [ubuntu, windows]
         # Don't forget to add all new flavors to this list!
-        flavor: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+        flavor: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
         include:
           # Node 12.15
           # TODO Add comments about why we test 12.15; I think git blame says it's because of an ESM behavioral change that happened at 12.16
@@ -112,14 +112,43 @@ jobs:
             typescript: next
             typescriptFlag: next
             downgradeNpm: true
+          # Node nightly
+          - flavor: 11
+            node: nightly
+            nodeFlag: nightly
+            typescript: latest
+            typescriptFlag: latest
+            downgradeNpm: true
     steps:
       # checkout code
       - uses: actions/checkout@v2
       # install node
       - name: Use Node.js ${{ matrix.node }}
+        if: matrix.node != 'nightly'
         uses: actions/setup-node@v1
         with:
           node-version: ${{ matrix.node }}
+      - name: Use Node.js 16, will be subsequently overridden by download of nightly
+        if: matrix.node == 'nightly'
+        uses: actions/setup-node@v1
+        with:
+          node-version: 16
+      - name: Download Node.js nightly
+        if: matrix.node == 'nightly' && matrix.os == 'ubuntu'
+        run: |
+          export N_PREFIX=$(pwd)/n
+          npm install -g n
+          n nightly
+          sudo cp "${N_PREFIX}/bin/node" "$(which node)"
+          node --version
+      - name: Download Node.js nightly
+        if: matrix.node == 'nightly' && matrix.os == 'windows'
+        run: |
+          $version = (Invoke-WebRequest https://nodejs.org/download/nightly/index.json | ConvertFrom-json)[0].version
+          $url = "https://nodejs.org/download/nightly/$version/win-x64/node.exe"
+          $targetPath = (Get-Command node.exe).Source
+          Invoke-WebRequest -Uri $url -OutFile $targetPath
+          node --version
       # lint, build, test
         # Downgrade from npm 7 to 6 because 7 still seems buggy to me
       - if: ${{ matrix.downgradeNpm }}

dist-raw/node-esm-default-get-format.js

@@ -0,0 +1,83 @@
+// Copied from https://raw.githubusercontent.com/nodejs/node/v15.3.0/lib/internal/modules/esm/get_format.js
+// Then modified to suite our needs.
+// Formatting is intentionally bad to keep the diff as small as possible, to make it easier to merge
+// upstream changes and understand our modifications.
+
+'use strict';
+const {
+  RegExpPrototypeExec,
+  StringPrototypeStartsWith,
+} = require('./node-primordials');
+const { extname } = require('path');
+const { getOptionValue } = require('./node-options');
+
+const experimentalJsonModules = getOptionValue('--experimental-json-modules');
+const experimentalSpeciferResolution =
+  getOptionValue('--experimental-specifier-resolution');
+const experimentalWasmModules = getOptionValue('--experimental-wasm-modules');
+const { getPackageType } = require('./node-esm-resolve-implementation.js').createResolve({tsExtensions: [], jsExtensions: []});
+const { URL, fileURLToPath } = require('url');
+const { ERR_UNKNOWN_FILE_EXTENSION } = require('./node-errors').codes;
+
+const extensionFormatMap = {
+  '__proto__': null,
+  '.cjs': 'commonjs',
+  '.js': 'module',
+  '.mjs': 'module'
+};
+
+const legacyExtensionFormatMap = {
+  '__proto__': null,
+  '.cjs': 'commonjs',
+  '.js': 'commonjs',
+  '.json': 'commonjs',
+  '.mjs': 'module',
+  '.node': 'commonjs'
+};
+
+if (experimentalWasmModules)
+  extensionFormatMap['.wasm'] = legacyExtensionFormatMap['.wasm'] = 'wasm';
+
+if (experimentalJsonModules)
+  extensionFormatMap['.json'] = legacyExtensionFormatMap['.json'] = 'json';
+
+function defaultGetFormat(url, context, defaultGetFormatUnused) {
+  if (StringPrototypeStartsWith(url, 'node:')) {
+    return { format: 'builtin' };
+  }
+  const parsed = new URL(url);
+  if (parsed.protocol === 'data:') {
+    const [ , mime ] = RegExpPrototypeExec(
+      /^([^/]+\/[^;,]+)(?:[^,]*?)(;base64)?,/,
+      parsed.pathname,
+    ) || [ null, null, null ];
+    const format = ({
+      '__proto__': null,
+      'text/javascript': 'module',
+      'application/json': experimentalJsonModules ? 'json' : null,
+      'application/wasm': experimentalWasmModules ? 'wasm' : null
+    })[mime] || null;
+    return { format };
+  } else if (parsed.protocol === 'file:') {
+    const ext = extname(parsed.pathname);
+    let format;
+    if (ext === '.js') {
+      format = getPackageType(parsed.href) === 'module' ? 'module' : 'commonjs';
+    } else {
+      format = extensionFormatMap[ext];
+    }
+    if (!format) {
+      if (experimentalSpeciferResolution === 'node') {
+        process.emitWarning(
+          'The Node.js specifier resolution in ESM is experimental.',
+          'ExperimentalWarning');
+        format = legacyExtensionFormatMap[ext];
+      } else {
+        throw new ERR_UNKNOWN_FILE_EXTENSION(ext, fileURLToPath(url));
+      }
+    }
+    return { format: format || null };
+  }
+  return { format: null };
+}
+exports.defaultGetFormat = defaultGetFormat;

esm.mjs

@@ -6,6 +6,7 @@ const require = createRequire(fileURLToPath(import.meta.url));
 const esm = require('./dist/esm');
 export const {
   resolve,
+  load,
   getFormat,
   transformSource,
 } = esm.registerAndCreateEsmHooks();

esm/transpile-only.mjs

@@ -6,6 +6,7 @@ const require = createRequire(fileURLToPath(import.meta.url));
 const esm = require('../dist/esm');
 export const {
   resolve,
+  load,
   getFormat,
   transformSource,
 } = esm.registerAndCreateEsmHooks({ transpileOnly: true });

package.json

@@ -70,10 +70,9 @@
     "pre-debug": "npm run build-tsc && npm run build-pack",
     "coverage-report": "nyc report --reporter=lcov",
     "prepare": "npm run clean && npm run build-nopack",
-    "api-extractor": "api-extractor run --local --verbose"
-  },
-  "engines": {
-    "node": ">=12.0.0"
+    "api-extractor": "api-extractor run --local --verbose",
+    "esm-usage-example": "npm run build-tsc && cd esm-usage-example && node --experimental-specifier-resolution node --loader ../esm.mjs ./index",
+    "esm-usage-example2": "npm run build-tsc && cd tests && TS_NODE_PROJECT=./module-types/override-to-cjs/tsconfig.json node --loader ../esm.mjs ./module-types/override-to-cjs/test.cjs"
   },
   "repository": {
     "type": "git",

src/esm.ts

@@ -1,4 +1,10 @@
-import { getExtensions, register, RegisterOptions, Service } from './index';
+import {
+  register,
+  getExtensions,
+  RegisterOptions,
+  Service,
+  versionGteLt,
+} from './index';
 import {
   parse as parseUrl,
   format as formatUrl,
@@ -12,9 +18,24 @@ import { normalizeSlashes } from './util';
 const {
   createResolve,
 } = require('../dist-raw/node-esm-resolve-implementation');
+const { defaultGetFormat } = require('../dist-raw/node-esm-default-get-format');
 
 // Note: On Windows, URLs look like this: file:///D:/dev/@TypeStrong/ts-node-examples/foo.ts
 
+// NOTE ABOUT MULTIPLE EXPERIMENTAL LOADER APIS
+//
+// At the time of writing, this file implements 2x different loader APIs.
+// Node made a breaking change to the loader API in https://github.com/nodejs/node/pull/37468
+//
+// We check the node version number and export either the *old* or the *new* API.
+//
+// Today, we are implementing the *new* API on top of our implementation of the *old* API,
+// which relies on copy-pasted code from the *old* hooks implementation in node.
+//
+// In the future, we will likely invert this: we will copy-paste the *new* API implementation
+// from node, build our implementation of the *new* API on top of it, and implement the *old*
+// hooks API as a shim to the *new* API.
+
 /** @internal */
 export function registerAndCreateEsmHooks(opts?: RegisterOptions) {
   // Automatically performs registration just like `-r ts-node/register`
@@ -32,7 +53,24 @@ export function createEsmHooks(tsNodeService: Service) {
     preferTsExts: tsNodeService.options.preferTsExts,
   });
 
-  return { resolve, getFormat, transformSource };
+  // The hooks API changed in node version X so we need to check for backwards compatibility.
+  // TODO: When the new API is backported to v12, v14, v16, update these version checks accordingly.
+  const newHooksAPI =
+    versionGteLt(process.versions.node, '17.0.0') ||
+    versionGteLt(process.versions.node, '16.999.999', '17.0.0') ||
+    versionGteLt(process.versions.node, '14.999.999', '15.0.0') ||
+    versionGteLt(process.versions.node, '12.999.999', '13.0.0');
+
+  // Explicit return type to avoid TS's non-ideal inferred type
+  const hooksAPI: {
+    resolve: typeof resolve;
+    getFormat: typeof getFormat | undefined;
+    transformSource: typeof transformSource | undefined;
+    load: typeof load | undefined;
+  } = newHooksAPI
+    ? { resolve, load, getFormat: undefined, transformSource: undefined }
+    : { resolve, getFormat, transformSource, load: undefined };
+  return hooksAPI;
 
   function isFileUrlOrNodeStyleSpecifier(parsed: UrlWithStringQuery) {
     // We only understand file:// URLs, but in node, the specifier can be a node-style `./foo` or `foo`
@@ -76,6 +114,52 @@ export function createEsmHooks(tsNodeService: Service) {
     );
   }
 
+  // `load` from new loader hook API (See description at the top of this file)
+  async function load(
+    url: string,
+    context: { format: Format | null | undefined },
+    defaultLoad: typeof load
+  ): Promise<{ format: Format; source: string | Buffer | undefined }> {
+    // If we get a format hint from resolve() on the context then use it
+    // otherwise call the old getFormat() hook using node's old built-in defaultGetFormat() that ships with ts-node
+    const format =
+      context.format ??
+      (await getFormat(url, context, defaultGetFormat)).format;
+
+    let source = undefined;
+    if (format !== 'builtin' && format !== 'commonjs') {
+      // Call the new defaultLoad() to get the source
+      const { source: rawSource } = await defaultLoad(
+        url,
+        { format },
+        defaultLoad
+      );
+
+      if (rawSource === undefined || rawSource === null) {
+        throw new Error(
+          `Failed to load raw source: Format was '${format}' and url was '${url}''.`
+        );
+      }
+
+      // Emulate node's built-in old defaultTransformSource() so we can re-use the old transformSource() hook
+      const defaultTransformSource: typeof transformSource = async (
+        source,
+        _context,
+        _defaultTransformSource
+      ) => ({ source });
+
+      // Call the old hook
+      const { source: transformedSource } = await transformSource(
+        rawSource,
+        { url, format },
+        defaultTransformSource
+      );
+      source = transformedSource;
+    }
+
+    return { format, source };
+  }
+
   type Format = 'builtin' | 'commonjs' | 'dynamic' | 'json' | 'module' | 'wasm';
   async function getFormat(
     url: string,
@@ -129,6 +213,10 @@ export function createEsmHooks(tsNodeService: Service) {
     context: { url: string; format: Format },
     defaultTransformSource: typeof transformSource
   ): Promise<{ source: string | Buffer }> {
+    if (source === null || source === undefined) {
+      throw new Error('No source');
+    }
+
     const defer = () =>
       defaultTransformSource(source, context, defaultTransformSource);
 

src/index.ts

@@ -47,18 +47,31 @@ export type {
 const engineSupportsPackageTypeField =
   parseInt(process.versions.node.split('.')[0], 10) >= 12;
 
-function versionGte(version: string, requirement: string) {
-  const [major, minor, patch, extra] = version
-    .split(/[\.-]/)
-    .map((s) => parseInt(s, 10));
-  const [reqMajor, reqMinor, reqPatch] = requirement
-    .split('.')
-    .map((s) => parseInt(s, 10));
-  return (
-    major > reqMajor ||
-    (major === reqMajor &&
-      (minor > reqMinor || (minor === reqMinor && patch >= reqPatch)))
-  );
+/** @internal */
+export function versionGteLt(
+  version: string,
+  gteRequirement: string,
+  ltRequirement?: string
+) {
+  const [major, minor, patch, extra] = parse(version);
+  const [gteMajor, gteMinor, gtePatch] = parse(gteRequirement);
+  const isGte =
+    major > gteMajor ||
+    (major === gteMajor &&
+      (minor > gteMinor || (minor === gteMinor && patch >= gtePatch)));
+  let isLt = true;
+  if (ltRequirement) {
+    const [ltMajor, ltMinor, ltPatch] = parse(ltRequirement);
+    isLt =
+      major < ltMajor ||
+      (major === ltMajor &&
+        (minor < ltMinor || (minor === ltMinor && patch < ltPatch)));
+  }
+  return isGte && isLt;
+
+  function parse(requirement: string) {
+    return requirement.split(/[\.-]/).map((s) => parseInt(s, 10));
+  }
 }
 
 /**
@@ -570,7 +583,7 @@ export function create(rawOptions: CreateOptions = {}): Service {
     );
   }
   // Top-level await was added in TS 3.8
-  const tsVersionSupportsTla = versionGte(ts.version, '3.8.0');
+  const tsVersionSupportsTla = versionGteLt(ts.version, '3.8.0');
   if (options.experimentalReplAwait === true && !tsVersionSupportsTla) {
     throw new Error(
       'Experimental REPL await is not compatible with TypeScript versions older than 3.8'

tests/esm-custom-loader/loader.mjs

@@ -11,6 +11,6 @@ const tsNodeInstance = register({
   },
 });
 
-export const { resolve, getFormat, transformSource } = createEsmHooks(
+export const { resolve, getFormat, transformSource, load } = createEsmHooks(
   tsNodeInstance
 );