From 670d0ea56302fdd17f6486599f5de98f989d459f Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 7 Sep 2023 14:51:11 -0700 Subject: [PATCH 01/14] remove swc experimental options This removes support for keeping import assertions, which were broken in swc at some point, and unconditionally transpiled into import attributes. (Ie, `import/with` instead of `import/assert`.) No version of node supports import attributes with this syntax yet, so anyone using swc to import json in ESM is out of luck no matter what. And swc 1.3.83 broke the option that ts-node was using. The position of the swc project is that experimental features are not supported, and may change in patch versions without warning, making them unsafe to rely on (as evidenced here, and the reason why this behavior changed unexpectedly in the first place). Better to just not use experimental swc features, and let it remove import assertions rather than transpile them into something that node can't run. Fix: https://github.com/TypeStrong/ts-node/issues/2056 --- src/test/transpilers.spec.ts | 19 ------------------- src/transpilers/swc.ts | 3 --- 2 files changed, 22 deletions(-) diff --git a/src/test/transpilers.spec.ts b/src/test/transpilers.spec.ts index d11c368bf..185d92ce8 100644 --- a/src/test/transpilers.spec.ts +++ b/src/test/transpilers.spec.ts @@ -120,23 +120,4 @@ test.suite('swc', (test) => { ); }); }); - - test.suite('preserves import assertions for json imports', (test) => { - test.if(tsSupportsImportAssertions); - test( - 'basic json import', - compileMacro, - { module: 'esnext' }, - outdent` - import document from './document.json' assert {type: 'json'}; - document; - `, - outdent` - import document from './document.json' assert { - type: 'json' - }; - document; - ` - ); - }); }); diff --git a/src/transpilers/swc.ts b/src/transpilers/swc.ts index 16868cad2..9b438d59b 100644 --- a/src/transpilers/swc.ts +++ b/src/transpilers/swc.ts @@ -235,9 +235,6 @@ export function createSwcOptions( }, }, keepClassNames, - experimental: { - keepImportAssertions: true, - }, }, }; From afbd4b69fc039864c73fd5de60a33eb58fc8d699 Mon Sep 17 00:00:00 2001 From: isaacs Date: Fri, 5 May 2023 21:37:04 -0700 Subject: [PATCH 02/14] fix: Add globalPreload to ts-node/esm for node 20 As of node v20, loader hooks are executed in a separate isolated thread environment. As a result, they are unable to register the `require.extensions` hooks in a way that would (in other node versions) make both CJS and ESM work as expected. By adding a `globalPreload` method, which *does* execute in the main script environment (but with very limited capabilities), these hooks can be attached properly, and `--loader=ts-node/esm` will once again make both cjs and esm typescript programs work properly. --- esm.mjs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/esm.mjs b/esm.mjs index c09e49095..d5139c1aa 100644 --- a/esm.mjs +++ b/esm.mjs @@ -5,3 +5,13 @@ const require = createRequire(fileURLToPath(import.meta.url)); /** @type {import('./dist/esm')} */ const esm = require('./dist/esm'); export const { resolve, load, getFormat, transformSource } = esm.registerAndCreateEsmHooks(); + +// Affordance for node 20, where load() happens in an isolated thread +export const globalPreload = () => { + const self = fileURLToPath(import.meta.url); + return ` +const { createRequire } = getBuiltin('module'); +const require = createRequire(${JSON.stringify(self)}); +require('./dist/esm').registerAndCreateEsmHooks(); +`; +}; From 71e319cbbc9eecd7d1e693b9a500a92ff5ab7241 Mon Sep 17 00:00:00 2001 From: isaacs Date: Sat, 6 May 2023 21:20:42 -0700 Subject: [PATCH 03/14] loader: return empty globalPreload when not using off-thread loader --- esm.mjs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/esm.mjs b/esm.mjs index d5139c1aa..e2c41d3b6 100644 --- a/esm.mjs +++ b/esm.mjs @@ -1,5 +1,6 @@ import { fileURLToPath } from 'url'; import { createRequire } from 'module'; +import { versionGteLt } from './dist/util.js'; const require = createRequire(fileURLToPath(import.meta.url)); /** @type {import('./dist/esm')} */ @@ -7,7 +8,11 @@ const esm = require('./dist/esm'); export const { resolve, load, getFormat, transformSource } = esm.registerAndCreateEsmHooks(); // Affordance for node 20, where load() happens in an isolated thread +const offThreadLoader = versionGteLt(process.versions.node, '20.0.0'); export const globalPreload = () => { + if (!offThreadLoader) { + return ''; + } const self = fileURLToPath(import.meta.url); return ` const { createRequire } = getBuiltin('module'); From 8b4be1d465494b232185c4e170b9fa0c3f725674 Mon Sep 17 00:00:00 2001 From: isaacs Date: Mon, 22 May 2023 23:20:27 -0700 Subject: [PATCH 04/14] fix: ts-node --esm on Node v20 When running `ts-node --esm`, a child process is spawned with the `child-loader.mjs` loader, `dist/child/child-entrypoint.js` main, and `argv[2]` set to the base64 encoded compressed configuration payload. `child-loader.mjs` imports and re-exports the functions defined in `src/child/child-loader.ts`. These are initially set to empty loader hooks which call the next hook in line until they are defined by calling `lateBindHooks()`. `child-entrypoint.ts` reads the config payload from argv, and bootstraps the registration process, which then calls `lateBindHooks()`. Presumably, the reason for this hand-off is because `--loader` hooks do not have access to `process.argv`. Unfortunately, in node 20, they don't have access to anything else, either, so calling `lateBindHooks` is effectively a no-op; the `child-loader.ts` where the hooks end up getting bound is not the same one that is being used as the actual loader. To solve this, the following changes are added: 1. An `isLoaderThread` flag is added to the BootstrapState. If this flag is set, then no further processing is performed beyond binding the loader hooks. 2. `callInChild` adds the config payload to _both_ the argv and the loader URL as a query param. 3. In the `child-loader.mjs` loader, only on node v20 and higher, the config payload is read from `import.meta.url`, and `bootstrap` is called, setting the `isLoaderThread` flag. I'm not super enthusiastic about this implementation. It definitely feels like there's a refactoring opportunity to clean it up, as it adds some copypasta between child-entrypoint.ts and child-loader.mjs. A further improvement would be to remove the late-binding handoff complexity entirely, and _always_ pass the config payload on the loader URL rather than on process.argv. --- child-loader.mjs | 7 +++++-- esm.mjs | 17 +---------------- src/bin.ts | 9 +++++++-- src/child/child-loader.ts | 19 +++++++++++++++++++ src/child/spawn-child.ts | 8 ++++++-- src/esm.ts | 12 ++++++++++++ 6 files changed, 50 insertions(+), 22 deletions(-) diff --git a/child-loader.mjs b/child-loader.mjs index 6ef592ac5..4c5c22393 100644 --- a/child-loader.mjs +++ b/child-loader.mjs @@ -1,8 +1,11 @@ -import { fileURLToPath } from 'url'; import { createRequire } from 'module'; +import { fileURLToPath } from 'url'; + const require = createRequire(fileURLToPath(import.meta.url)); // TODO why use require() here? I think we can just `import` /** @type {import('./dist/child-loader')} */ const childLoader = require('./dist/child/child-loader'); -export const { resolve, load, getFormat, transformSource } = childLoader; +export const { resolve, load, getFormat, transformSource, bindFromLoaderThread } = childLoader; + +bindFromLoaderThread(import.meta.url); diff --git a/esm.mjs b/esm.mjs index e2c41d3b6..4a4b2e323 100644 --- a/esm.mjs +++ b/esm.mjs @@ -1,22 +1,7 @@ import { fileURLToPath } from 'url'; import { createRequire } from 'module'; -import { versionGteLt } from './dist/util.js'; const require = createRequire(fileURLToPath(import.meta.url)); /** @type {import('./dist/esm')} */ const esm = require('./dist/esm'); -export const { resolve, load, getFormat, transformSource } = esm.registerAndCreateEsmHooks(); - -// Affordance for node 20, where load() happens in an isolated thread -const offThreadLoader = versionGteLt(process.versions.node, '20.0.0'); -export const globalPreload = () => { - if (!offThreadLoader) { - return ''; - } - const self = fileURLToPath(import.meta.url); - return ` -const { createRequire } = getBuiltin('module'); -const require = createRequire(${JSON.stringify(self)}); -require('./dist/esm').registerAndCreateEsmHooks(); -`; -}; +export const { resolve, load, getFormat, transformSource, globalPreload } = esm.registerAndCreateEsmHooks(); diff --git a/src/bin.ts b/src/bin.ts index cbe34caf8..5015d6a74 100644 --- a/src/bin.ts +++ b/src/bin.ts @@ -71,6 +71,7 @@ export interface BootstrapState { parseArgvResult: ReturnType; phase2Result?: ReturnType; phase3Result?: ReturnType; + isLoaderThread?: boolean; } /** @internal */ @@ -441,7 +442,7 @@ function getEntryPointInfo(state: BootstrapState) { } function phase4(payload: BootstrapState) { - const { isInChildProcess, tsNodeScript } = payload; + const { isInChildProcess, tsNodeScript, isLoaderThread } = payload; const { version, showConfig, restArgs, code, print, argv } = payload.parseArgvResult; const { cwd } = payload.phase2Result!; const { preloadedConfig } = payload.phase3Result!; @@ -522,8 +523,12 @@ function phase4(payload: BootstrapState) { if (replStuff) replStuff.state.path = join(cwd, REPL_FILENAME(service.ts.version)); - if (isInChildProcess) + if (isInChildProcess) { (require('./child/child-loader') as typeof import('./child/child-loader')).lateBindHooks(createEsmHooks(service)); + // we should not do anything else at this point in the loader thread, + // let the entrypoint run the actual program. + if (isLoaderThread) return; + } // Bind REPL service to ts-node compiler service (chicken-and-egg problem) replStuff?.repl.setService(service); diff --git a/src/child/child-loader.ts b/src/child/child-loader.ts index 8d07c32cf..a68d1e3f0 100644 --- a/src/child/child-loader.ts +++ b/src/child/child-loader.ts @@ -1,5 +1,24 @@ import type { NodeLoaderHooksAPI1, NodeLoaderHooksAPI2 } from '..'; import { filterHooksByAPIVersion } from '../esm'; +import { URL } from 'url'; +import { bootstrap } from '../bin'; +import { versionGteLt } from '../util'; +import { argPrefix, decompress } from './argv-payload'; + +// On node v20, we cannot lateBind the hooks from outside the loader thread +// so it has to be done in the loader thread. +export function bindFromLoaderThread(loaderURL: string) { + // If we aren't in a loader thread, then skip this step. + if (!versionGteLt(process.versions.node, '20.0.0')) return; + + const url = new URL(loaderURL); + const base64Payload = url.searchParams.get(argPrefix); + if (!base64Payload) throw new Error('unexpected loader url'); + const state = decompress(base64Payload); + state.isInChildProcess = true; + state.isLoaderThread = true; + bootstrap(state); +} let hooks: NodeLoaderHooksAPI1 & NodeLoaderHooksAPI2; diff --git a/src/child/spawn-child.ts b/src/child/spawn-child.ts index 2859a61d9..1c270e7fe 100644 --- a/src/child/spawn-child.ts +++ b/src/child/spawn-child.ts @@ -10,6 +10,10 @@ import { argPrefix, compress } from './argv-payload'; * the child process. */ export function callInChild(state: BootstrapState) { + const loaderURL = pathToFileURL(require.resolve('../../child-loader.mjs')); + const compressedState = compress(state); + loaderURL.searchParams.set(argPrefix, compressedState); + const child = spawn( process.execPath, [ @@ -17,9 +21,9 @@ export function callInChild(state: BootstrapState) { require.resolve('./child-require.js'), '--loader', // Node on Windows doesn't like `c:\` absolute paths here; must be `file:///c:/` - pathToFileURL(require.resolve('../../child-loader.mjs')).toString(), + loaderURL.toString(), require.resolve('./child-entrypoint.js'), - `${argPrefix}${compress(state)}`, + `${argPrefix}${compressedState}`, ...state.parseArgvResult.restArgs, ], { diff --git a/src/esm.ts b/src/esm.ts index 9ca6f0dec..7596231f1 100644 --- a/src/esm.ts +++ b/src/esm.ts @@ -43,6 +43,7 @@ export namespace NodeLoaderHooksAPI1 { export interface NodeLoaderHooksAPI2 { resolve: NodeLoaderHooksAPI2.ResolveHook; load: NodeLoaderHooksAPI2.LoadHook; + globalPreload?: NodeLoaderHooksAPI2.GlobalPreload; } export namespace NodeLoaderHooksAPI2 { export type ResolveHook = ( @@ -74,6 +75,7 @@ export namespace NodeLoaderHooksAPI2 { export interface NodeImportAssertions { type?: 'json'; } + export type GlobalPreload = () => string; } export type NodeLoaderHooksFormat = 'builtin' | 'commonjs' | 'dynamic' | 'json' | 'module' | 'wasm'; @@ -111,14 +113,24 @@ export function createEsmHooks(tsNodeService: Service) { const nodeResolveImplementation = tsNodeService.getNodeEsmResolver(); const nodeGetFormatImplementation = tsNodeService.getNodeEsmGetFormat(); const extensions = tsNodeService.extensions; + const useLoaderThread = versionGteLt(process.versions.node, '20.0.0'); const hooksAPI = filterHooksByAPIVersion({ resolve, load, getFormat, transformSource, + globalPreload: useLoaderThread ? globalPreload : undefined, }); + function globalPreload() { + return ` + const { createRequire } = getBuiltin('module'); + const require = createRequire(${JSON.stringify(__filename)}); + require('./index').register(); + `; + } + function isFileUrlOrNodeStyleSpecifier(parsed: UrlWithStringQuery) { // We only understand file:// URLs, but in node, the specifier can be a node-style `./foo` or `foo` const { protocol } = parsed; From a7aa44079f6df58b4e7299ac993468d60c2e1ed6 Mon Sep 17 00:00:00 2001 From: Andrew Bradley Date: Sun, 23 Jul 2023 23:36:11 -0400 Subject: [PATCH 05/14] WIP fix tests for off-thread loader --- ava.config.cjs | 2 +- src/test/helpers/ctx-ts-node.ts | 2 + src/test/helpers/reset-node-environment.ts | 3 +- src/test/repl/repl.spec.ts | 4 +- src/test/resolver.spec.ts | 10 ++-- src/test/test-loader.d.ts | 8 --- src/test/test-loader.mjs | 19 ------ src/test/test-loader/README.md | 11 ++++ src/test/test-loader/client.ts | 26 ++++++++ src/test/test-loader/loader.mjs | 69 ++++++++++++++++++++++ 10 files changed, 119 insertions(+), 35 deletions(-) delete mode 100644 src/test/test-loader.d.ts delete mode 100644 src/test/test-loader.mjs create mode 100644 src/test/test-loader/README.md create mode 100644 src/test/test-loader/client.ts create mode 100644 src/test/test-loader/loader.mjs diff --git a/ava.config.cjs b/ava.config.cjs index 5532195d4..9879cd1d8 100644 --- a/ava.config.cjs +++ b/ava.config.cjs @@ -16,7 +16,7 @@ module.exports = { CONCURRENT_TESTS: '4' }, require: ['./src/test/remove-env-var-force-color.js'], - nodeArguments: ['--loader', './src/test/test-loader.mjs', '--no-warnings'], + nodeArguments: ['--loader', './src/test/test-loader/loader.mjs', '--no-warnings'], timeout: '300s', concurrency: 4, // We do chdir -- maybe other things -- that you can't do in worker_threads. diff --git a/src/test/helpers/ctx-ts-node.ts b/src/test/helpers/ctx-ts-node.ts index 7097f117f..c38b2af3b 100644 --- a/src/test/helpers/ctx-ts-node.ts +++ b/src/test/helpers/ctx-ts-node.ts @@ -11,8 +11,10 @@ import { testsDirRequire, tsNodeTypes } from './misc'; /** Pass to `test.context()` to get access to the ts-node API under test */ export async function ctxTsNode() { await installTsNode(); + const tsNodeSpecifier = testsDirRequire.resolve('ts-node'); const tsNodeUnderTest: typeof tsNodeTypes = testsDirRequire('ts-node'); return { + tsNodeSpecifier, tsNodeUnderTest, }; } diff --git a/src/test/helpers/reset-node-environment.ts b/src/test/helpers/reset-node-environment.ts index 5c8140406..f6442f9c2 100644 --- a/src/test/helpers/reset-node-environment.ts +++ b/src/test/helpers/reset-node-environment.ts @@ -1,4 +1,5 @@ import { has, mapValues, sortBy } from 'lodash'; +import * as testLoader from '../test-loader/client'; // Reset node environment // Useful because ts-node installation necessarily must mutate the node environment. @@ -44,7 +45,7 @@ export function resetNodeEnvironment() { resetObject(global, defaultGlobal, ['__coverage__']); // Reset our ESM hooks - process.__test_setloader__?.(undefined); + testLoader.clearLoader(); } function captureObjectState(object: any, avoidGetters: string[] = []) { diff --git a/src/test/repl/repl.spec.ts b/src/test/repl/repl.spec.ts index 99612ac40..31e30e4a6 100644 --- a/src/test/repl/repl.spec.ts +++ b/src/test/repl/repl.spec.ts @@ -222,8 +222,8 @@ test.suite('top level await', ({ contextEach }) => { }); test('should pass upstream test cases', async (t) => { - const { tsNodeUnderTest } = t.context; - await upstreamTopLevelAwaitTests({ TEST_DIR, tsNodeUnderTest }); + const { tsNodeUnderTest, tsNodeSpecifier } = t.context; + await upstreamTopLevelAwaitTests({ TEST_DIR, tsNodeUnderTest, tsNodeSpecifier }); }); }); diff --git a/src/test/resolver.spec.ts b/src/test/resolver.spec.ts index 15d788c46..be5fbac58 100755 --- a/src/test/resolver.spec.ts +++ b/src/test/resolver.spec.ts @@ -13,9 +13,10 @@ import * as semver from 'semver'; import { padStart } from 'lodash'; import _ = require('lodash'); import { pathToFileURL } from 'url'; -import type { RegisterOptions } from '..'; +import type { CreateOptions, RegisterOptions } from '..'; import * as fs from 'fs'; import * as Path from 'path'; +import * as testLoader from './test-loader/client'; /* * Each test case is a separate TS project, with a different permutation of @@ -660,10 +661,11 @@ async function execute(t: T, p: FsProject, entrypoints: Entrypoint[]) { // Install ts-node and try to import all the index-* files // - const service = t.context.tsNodeUnderTest.register({ + const options: CreateOptions = { projectSearchDir: p.cwd, - }); - process.__test_setloader__(t.context.tsNodeUnderTest.createEsmHooks(service)); + }; + t.context.tsNodeUnderTest.register(options); + await testLoader.setLoader(t.context.tsNodeSpecifier, options); for (const entrypoint of entrypoints) { t.log(`Importing ${join(p.cwd, entrypoint)}`); diff --git a/src/test/test-loader.d.ts b/src/test/test-loader.d.ts deleted file mode 100644 index 1d83a9976..000000000 --- a/src/test/test-loader.d.ts +++ /dev/null @@ -1,8 +0,0 @@ -export {}; -declare global { - namespace NodeJS { - interface Process { - __test_setloader__(hooks: any): void; - } - } -} diff --git a/src/test/test-loader.mjs b/src/test/test-loader.mjs deleted file mode 100644 index 7f5e2b71a..000000000 --- a/src/test/test-loader.mjs +++ /dev/null @@ -1,19 +0,0 @@ -// Grant ourselves the ability to install ESM loader behaviors in-process during tests -import semver from 'semver'; - -const newHooksAPI = semver.gte(process.versions.node, '16.12.0'); - -let hooks = undefined; -process.__test_setloader__ = function (_hooks) { - hooks = _hooks; -}; -function createHook(name) { - return function (a, b, c) { - const target = (hooks && hooks[name]) || c; - return target(...arguments); - }; -} -export const resolve = createHook('resolve'); -export const load = newHooksAPI ? createHook('load') : null; -export const getFormat = !newHooksAPI ? createHook('getFormat') : null; -export const transformSource = !newHooksAPI ? createHook('transformSource') : null; diff --git a/src/test/test-loader/README.md b/src/test/test-loader/README.md new file mode 100644 index 000000000..36363f3c0 --- /dev/null +++ b/src/test/test-loader/README.md @@ -0,0 +1,11 @@ +An injectable --loader used by tests. It allows tests to setup and teardown +different loader behaviors in-process. + +By default, it does nothing, as if we were not using a loader at all. But it +responds to specially-crafted `import`s, installing or uninstalling a loader at +runtime. + +See also `ava.config.cjs` + +The loader is implemented in `loader.mjs`, and functions to send it commands are +in `client.ts`. diff --git a/src/test/test-loader/client.ts b/src/test/test-loader/client.ts new file mode 100644 index 000000000..bca06384a --- /dev/null +++ b/src/test/test-loader/client.ts @@ -0,0 +1,26 @@ +import { URL } from 'url'; +import type { CreateOptions } from '../..'; + +// Duplicated in loader.mjs +const protocol = 'testloader://'; +const clearLoaderCmd = 'clearLoader'; +const setLoaderCmd = 'setLoader'; + +// Avoid ts compiler transforming import() into require(). +const doImport = new Function('specifier', 'return import(specifier)'); +let cacheBust = 0; +async function call(url: URL) { + url.searchParams.set('cacheBust', `${cacheBust++}`); + await doImport(url.toString()); +} + +export async function clearLoader() { + await call(new URL(`${protocol}${clearLoaderCmd}`)); +} + +export async function setLoader(specifier: string, options: CreateOptions | undefined) { + const url = new URL(`${protocol}${setLoaderCmd}`); + url.searchParams.append('specifier', specifier); + url.searchParams.append('options', JSON.stringify(options)); + await call(url); +} diff --git a/src/test/test-loader/loader.mjs b/src/test/test-loader/loader.mjs new file mode 100644 index 000000000..b2d39329d --- /dev/null +++ b/src/test/test-loader/loader.mjs @@ -0,0 +1,69 @@ +import semver from 'semver'; +import { URL } from 'url'; +import { createRequire } from 'module'; + +export const protocol = 'testloader://'; + +const newHooksAPI = semver.gte(process.versions.node, '16.12.0'); + +let hooks = undefined; + +const require = createRequire(import.meta.url); + +// +// Commands +// + +export const clearLoaderCmd = 'clearLoader'; +function clearLoader() { + hooks = undefined; +} + +export const setLoaderCmd = 'setLoader'; +function setLoader(specifier, options) { + const tsNode = require(specifier); + const service = tsNode.create(options); + hooks = tsNode.createEsmHooks(service); +} + +// +// Loader hooks +// + +function createHook(name) { + return function (a, b, c) { + const target = (hooks && hooks[name]) || c; + return target(...arguments); + }; +} + +export const empty = `${protocol}empty`; +const resolveEmpty = { url: empty, shortCircuit: true }; + +const _resolve = createHook('resolve'); +export function resolve(specifier, ...rest) { + if (specifier.startsWith(protocol)) { + const url = new URL(specifier); + switch (url.host) { + case setLoaderCmd: + const specifier = url.searchParams.get('specifier'); + const options = JSON.parse(url.searchParams.get('options')); + setLoader(specifier, options); + return resolveEmpty; + case clearLoaderCmd: + clearLoader(); + return resolveEmpty; + } + } + return _resolve(specifier, ...rest); +} + +const _loadHook = newHooksAPI ? createHook('load') : null; +function _load(url, ...rest) { + if (url === empty) return { format: 'module', source: '', shortCircuit: true }; + return _loadHook(url, ...rest); +} +export const load = newHooksAPI ? _load : null; + +export const getFormat = !newHooksAPI ? createHook('getFormat') : null; +export const transformSource = !newHooksAPI ? createHook('transformSource') : null; From 9346c54864c0dfc423fcb6e6b35cb248d451cd81 Mon Sep 17 00:00:00 2001 From: Andrew Bradley Date: Fri, 1 Sep 2023 12:47:27 -0400 Subject: [PATCH 06/14] Pushing my local changes --- package.json | 2 +- src/esm.ts | 8 ++++---- yarn.lock | 29 +++++++++++++++-------------- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/package.json b/package.json index c55deb400..be71f93a4 100644 --- a/package.json +++ b/package.json @@ -113,7 +113,7 @@ "homepage": "https://typestrong.org/ts-node", "devDependencies": { "@TypeStrong/fs-fixture-builder": "https://github.com/Typestrong/fs-fixture-builder.git#3099e53621daf99db971af29c96145dc115693cd", - "@cspotcode/ava-lib": "https://github.com/cspotcode/ava-lib#bbbed83f393342b51dc6caf2ddf775a3e89371d8", + "@cspotcode/ava-lib": "https://github.com/cspotcode/ava-lib#805aab17b2b89c388596b6dc2b4eece403c5fb87", "@cspotcode/expect-stream": "https://github.com/cspotcode/node-expect-stream#4e425ff1eef240003af8716291e80fbaf3e3ae8f", "@microsoft/api-extractor": "^7.19.4", "@swc/core": "1.3.32", diff --git a/src/esm.ts b/src/esm.ts index 7596231f1..c11775bfd 100644 --- a/src/esm.ts +++ b/src/esm.ts @@ -43,7 +43,7 @@ export namespace NodeLoaderHooksAPI1 { export interface NodeLoaderHooksAPI2 { resolve: NodeLoaderHooksAPI2.ResolveHook; load: NodeLoaderHooksAPI2.LoadHook; - globalPreload?: NodeLoaderHooksAPI2.GlobalPreload; + globalPreload?: NodeLoaderHooksAPI2.GlobalPreloadHook; } export namespace NodeLoaderHooksAPI2 { export type ResolveHook = ( @@ -75,7 +75,7 @@ export namespace NodeLoaderHooksAPI2 { export interface NodeImportAssertions { type?: 'json'; } - export type GlobalPreload = () => string; + export type GlobalPreloadHook = () => string; } export type NodeLoaderHooksFormat = 'builtin' | 'commonjs' | 'dynamic' | 'json' | 'module' | 'wasm'; @@ -92,10 +92,10 @@ const newHooksAPI = versionGteLt(process.versions.node, '16.12.0'); export function filterHooksByAPIVersion( hooks: NodeLoaderHooksAPI1 & NodeLoaderHooksAPI2 ): NodeLoaderHooksAPI1 | NodeLoaderHooksAPI2 { - const { getFormat, load, resolve, transformSource } = hooks; + const { getFormat, load, resolve, transformSource, globalPreload } = hooks; // Explicit return type to avoid TS's non-ideal inferred type const hooksAPI: NodeLoaderHooksAPI1 | NodeLoaderHooksAPI2 = newHooksAPI - ? { resolve, load, getFormat: undefined, transformSource: undefined } + ? { resolve, load, globalPreload, getFormat: undefined, transformSource: undefined } : { resolve, getFormat, transformSource, load: undefined }; return hooksAPI; } diff --git a/yarn.lock b/yarn.lock index 035ecbe33..a6ac91200 100644 --- a/yarn.lock +++ b/yarn.lock @@ -241,16 +241,17 @@ __metadata: languageName: node linkType: hard -"@cspotcode/ava-lib@https://github.com/cspotcode/ava-lib#bbbed83f393342b51dc6caf2ddf775a3e89371d8": +"@cspotcode/ava-lib@https://github.com/cspotcode/ava-lib#805aab17b2b89c388596b6dc2b4eece403c5fb87": version: 0.0.1 - resolution: "@cspotcode/ava-lib@https://github.com/cspotcode/ava-lib.git#commit=bbbed83f393342b51dc6caf2ddf775a3e89371d8" + resolution: "@cspotcode/ava-lib@https://github.com/cspotcode/ava-lib.git#commit=805aab17b2b89c388596b6dc2b4eece403c5fb87" dependencies: "@types/node": "*" + chalk: 4.1.2 throat: ^6.0.1 peerDependencies: ava: "*" expect: "*" - checksum: 3ca30bbfe81abb537e1e96addd272b34daf19ecba56f13a5785115bc3433dc5309e733ab8440384531b7a74b88f58eb11c9151e62b75c5c219eccc7dd8b058ec + checksum: 3b453c2f1dd64eeb531a8b6da4cd4d9b24364b31b6692aca5d646d40479e46db56acc4141d93871732f33ab86e91b4c6a08674d1bcf49fc8c8ff8dab9f005742 languageName: node linkType: hard @@ -1257,6 +1258,16 @@ __metadata: languageName: node linkType: hard +"chalk@npm:4.1.2, chalk@npm:^4.0.0": + version: 4.1.2 + resolution: "chalk@npm:4.1.2" + dependencies: + ansi-styles: ^4.1.0 + supports-color: ^7.1.0 + checksum: fe75c9d5c76a7a98d45495b91b2172fa3b7a09e0cc9370e5c8feb1c567b85c4288e2b3fded7cfdd7359ac28d6b3844feb8b82b8686842e93d23c827c417e83fc + languageName: node + linkType: hard + "chalk@npm:^2.0.0": version: 2.4.2 resolution: "chalk@npm:2.4.2" @@ -1268,16 +1279,6 @@ __metadata: languageName: node linkType: hard -"chalk@npm:^4.0.0": - version: 4.1.2 - resolution: "chalk@npm:4.1.2" - dependencies: - ansi-styles: ^4.1.0 - supports-color: ^7.1.0 - checksum: fe75c9d5c76a7a98d45495b91b2172fa3b7a09e0cc9370e5c8feb1c567b85c4288e2b3fded7cfdd7359ac28d6b3844feb8b82b8686842e93d23c827c417e83fc - languageName: node - linkType: hard - "chalk@npm:^5.2.0": version: 5.2.0 resolution: "chalk@npm:5.2.0" @@ -3810,7 +3811,7 @@ __metadata: resolution: "ts-node@workspace:." dependencies: "@TypeStrong/fs-fixture-builder": "https://github.com/Typestrong/fs-fixture-builder.git#3099e53621daf99db971af29c96145dc115693cd" - "@cspotcode/ava-lib": "https://github.com/cspotcode/ava-lib#bbbed83f393342b51dc6caf2ddf775a3e89371d8" + "@cspotcode/ava-lib": "https://github.com/cspotcode/ava-lib#805aab17b2b89c388596b6dc2b4eece403c5fb87" "@cspotcode/expect-stream": "https://github.com/cspotcode/node-expect-stream#4e425ff1eef240003af8716291e80fbaf3e3ae8f" "@cspotcode/source-map-support": ^0.8.0 "@microsoft/api-extractor": ^7.19.4 From db1fdf90dec42bff8ffd93d0de494b8088eae9c4 Mon Sep 17 00:00:00 2001 From: isaacs Date: Fri, 1 Sep 2023 10:17:29 -0700 Subject: [PATCH 07/14] test: fix node defaultLoad error message check --- src/test/esm-loader.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/esm-loader.spec.ts b/src/test/esm-loader.spec.ts index 942533e67..c90bef2bf 100644 --- a/src/test/esm-loader.spec.ts +++ b/src/test/esm-loader.spec.ts @@ -95,7 +95,7 @@ test.suite('esm', (test) => { }); expect(r.err).not.toBe(null); // expect error from node's default resolver - expect(r.stderr).toMatch(/Error \[ERR_UNSUPPORTED_ESM_URL_SCHEME\]:.*(?:\n.*){0,2}\n *at defaultResolve/); + expect(r.stderr).toMatch(/Error \[ERR_UNSUPPORTED_ESM_URL_SCHEME\]:.*(?:\n.*){0,2}\n *at default(Load|Resolve)/); }); test('should bypass import cache when changing search params', async () => { From d68a15026857d5dfa17c406c373d04c18fa3e03f Mon Sep 17 00:00:00 2001 From: isaacs Date: Fri, 1 Sep 2023 16:41:14 -0700 Subject: [PATCH 08/14] loader: TSError that survives thread comms channel When an error is thrown in the loader thread, it must be passed through the comms channel to be printed in the main thread. Node has some heuristics to try to reconstitute errors properly, but they don't function very well if the error has a custom inspect method, or properties that are not compatible with JSON.stringify, so the TSErrors raised by the source transforms don't get printed in any sort of useful way. This catches those errors, and creates a new error that can go through the comms channel intact. Another possible approach would be to update the shape of the errors raised by source transforms, but that would be a much more extensive change with further reaching consequences. --- src/esm.ts | 38 +++++++++++++++++++++--------- tests/esm-custom-loader/loader.mjs | 2 +- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/esm.ts b/src/esm.ts index c11775bfd..29a896ff4 100644 --- a/src/esm.ts +++ b/src/esm.ts @@ -1,4 +1,4 @@ -import { register, RegisterOptions, Service } from './index'; +import { register, RegisterOptions, Service, type TSError } from './index'; import { parse as parseUrl, format as formatUrl, UrlWithStringQuery, fileURLToPath, pathToFileURL } from 'url'; import { extname, resolve as pathResolve } from 'path'; import * as assert from 'assert'; @@ -223,7 +223,7 @@ export function createEsmHooks(tsNodeService: Service) { format: NodeLoaderHooksFormat; source: string | Buffer | undefined; }> { - return addShortCircuitFlag(async () => { + return await addShortCircuitFlag(async () => { // If we get a format hint from resolve() on the context then use it // otherwise call the old getFormat() hook using node's old built-in defaultGetFormat() that ships with ts-node const format = @@ -251,8 +251,23 @@ export function createEsmHooks(tsNodeService: Service) { }); // Call the old hook - const { source: transformedSource } = await transformSource(rawSource, { url, format }, defaultTransformSource); - source = transformedSource; + try { + const { source: transformedSource } = await transformSource( + rawSource, + { url, format }, + defaultTransformSource + ); + source = transformedSource; + } catch (er) { + // throw an error that can make it through the loader thread + // comms channel intact. + const tsErr = er as TSError; + const err = new Error(tsErr.message.trimEnd()); + const { diagnosticCodes } = tsErr; + Object.assign(err, { diagnosticCodes }); + Error.captureStackTrace(err, load); + throw err; + } } return { format, source }; @@ -360,11 +375,12 @@ export function createEsmHooks(tsNodeService: Service) { } async function addShortCircuitFlag(fn: () => Promise) { - const ret = await fn(); - // Not sure if this is necessary; being lazy. Can revisit in the future. - if (ret == null) return ret; - return { - ...ret, - shortCircuit: true, - }; + return fn().then((ret) => { + // Not sure if this is necessary; being lazy. Can revisit in the future. + if (ret == null) return ret; + return { + ...ret, + shortCircuit: true, + }; + }); } diff --git a/tests/esm-custom-loader/loader.mjs b/tests/esm-custom-loader/loader.mjs index 74b82b02c..4da5b970c 100755 --- a/tests/esm-custom-loader/loader.mjs +++ b/tests/esm-custom-loader/loader.mjs @@ -11,4 +11,4 @@ const tsNodeInstance = register({ }, }); -export const { resolve, getFormat, transformSource, load } = createEsmHooks(tsNodeInstance); +export const { globalPreload, resolve, getFormat, transformSource, load } = createEsmHooks(tsNodeInstance); From 04d9419fedacd02f499f901ef3b1b32575776f33 Mon Sep 17 00:00:00 2001 From: isaacs Date: Fri, 1 Sep 2023 17:03:01 -0700 Subject: [PATCH 09/14] loader: Set default 'pretty' option properly Set the default `options.pretty` value based on stderr rather than stdout, as this is where errors are printed. The loader thread does not get a process.stderr.isTTY set, because its "stderr" is actually a pipe. If `options.pretty` is not set explicitly, the GlobalPreload's `context.port` is used to send a message from the main thread indicating the state of stderr.isTTY. Adds `Service.setPrettyErrors` method to enable setting this value when needed. --- src/esm.ts | 24 ++++++++++++++++++++++-- src/index.ts | 15 +++++++++++---- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/esm.ts b/src/esm.ts index 29a896ff4..c4566e0df 100644 --- a/src/esm.ts +++ b/src/esm.ts @@ -4,6 +4,7 @@ import { extname, resolve as pathResolve } from 'path'; import * as assert from 'assert'; import { normalizeSlashes, versionGteLt } from './util'; import { createRequire } from 'module'; +import type { MessagePort } from 'worker_threads'; // Note: On Windows, URLs look like this: file:///D:/dev/@TypeStrong/ts-node-examples/foo.ts @@ -75,7 +76,7 @@ export namespace NodeLoaderHooksAPI2 { export interface NodeImportAssertions { type?: 'json'; } - export type GlobalPreloadHook = () => string; + export type GlobalPreloadHook = (context?: { port: MessagePort }) => string; } export type NodeLoaderHooksFormat = 'builtin' | 'commonjs' | 'dynamic' | 'json' | 'module' | 'wasm'; @@ -123,10 +124,29 @@ export function createEsmHooks(tsNodeService: Service) { globalPreload: useLoaderThread ? globalPreload : undefined, }); - function globalPreload() { + function globalPreload({ port }: { port?: MessagePort } = {}) { + // The loader thread doesn't get process.stderr.isTTY properly, + // so this signal lets us infer it based on the state of the main + // thread, but only relevant if options.pretty is unset. + let stderrTTYSignal: string; + if (tsNodeService.options.pretty === undefined) { + port?.on('message', (data) => { + if (data?.stderrIsTTY) { + tsNodeService.setPrettyErrors(true); + } + }); + stderrTTYSignal = ` + port.postMessage({ + stderrIsTTY: !!process.stderr.isTTY + }); + `; + } else { + stderrTTYSignal = ''; + } return ` const { createRequire } = getBuiltin('module'); const require = createRequire(${JSON.stringify(__filename)}); + ${stderrTTYSignal} require('./index').register(); `; } diff --git a/src/index.ts b/src/index.ts index c7f320530..e294b00f8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -480,6 +480,7 @@ export interface Service { ignored(fileName: string): boolean; compile(code: string, fileName: string, lineOffset?: number): string; getTypeInfo(code: string, fileName: string, position: number): TypeInfo; + setPrettyErrors(pretty: boolean): void; /** @internal */ configFilePath: string | undefined; /** @internal */ @@ -719,11 +720,16 @@ export function createFromPreloadedConfig(foundConfigResult: ReturnType string; - const formatDiagnostics = shouldHavePrettyErrors - ? ts.formatDiagnosticsWithColorAndContext || ts.formatDiagnostics - : ts.formatDiagnostics; + function setPrettyErrors(pretty: boolean) { + shouldHavePrettyErrors = pretty; + formatDiagnostics = shouldHavePrettyErrors + ? ts.formatDiagnosticsWithColorAndContext || ts.formatDiagnostics + : ts.formatDiagnostics; + } + setPrettyErrors(options.pretty !== undefined ? options.pretty : !!process.stderr.isTTY); function createTSError(diagnostics: ReadonlyArray<_ts.Diagnostic>) { const diagnosticText = formatDiagnostics(diagnostics, diagnosticHost); @@ -1282,6 +1288,7 @@ export function createFromPreloadedConfig(foundConfigResult: ReturnType Date: Fri, 1 Sep 2023 17:29:15 -0700 Subject: [PATCH 10/14] use built-in source map support in node v20 The @cspotcode/source-map-support module does not function properly on Node 20, resulting in incorrect stack traces. Fortunately, the built-in source map support in Node is now quite reliable. This does have the following (somewhat subtle) changes to error output: - When a call site is in a method defined within a constructor function, it picks up the function name *as well as* the type name and method name. So, in tests where a method is called and throws within the a function constructor, we see `Foo.Foo.bar` instead of `Foo.bar`. - Call sites displays show filenames instead of file URLs. - The call site display puts the `^` character under the `throw` rather than the construction point of the error object. This is closer to how normal un-transpiled JavaScript behaves, and thus somewhat preferrable, but isn't possible when all we have to go on is the Error stack property, so it is a change. I haven't been able to figure out why exactly, but the call sites appear to be somewhat different in the repl/eval contexts as a result of this change. It almost seems like the @cspotcode/source-map-support was applying source maps to the vm-evaluated scripts, but I don't see how that could be, and in fact, there's a comment in the code stating that that *isn't* the case. But the line number showing up in an Error.stack property is `1` prior to this change (matching the location in the TS source) and is `2` afterwards (matching the location in the compiled JS). An argument could be made that specific line numbers are a bit meaningless in a REPL anyway, and the best approach is to just make those tests accept either result. One possible approach to provide built-in source map support for the repl would be to refactor the `appendCompileAndEvalInput` to diff and append the *input* TS, and compile within the `runInContext` method. If the transpiled code was prepended with `process.setSourceMapsEnabled(true);`, then Error stacks and call sites would be properly source mapped by Node internally. --- src/index.ts | 6 ++++++ src/test/esm-loader.spec.ts | 21 +++++++++++--------- src/test/index.spec.ts | 38 +++++++++---------------------------- src/test/sourcemaps.spec.ts | 33 +++++++++++++++++++------------- 4 files changed, 47 insertions(+), 51 deletions(-) diff --git a/src/index.ts b/src/index.ts index e294b00f8..14c1490a5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -691,7 +691,13 @@ export function createFromPreloadedConfig(foundConfigResult: ReturnType { cwd: join(TEST_DIR, './esm'), }); expect(r.err).not.toBe(null); - const expectedModuleUrl = pathToFileURL(join(TEST_DIR, './esm/throw error.ts')).toString(); + // on node 20, this will be a path. prior versions, a file: url + const expectedModulePath = join(TEST_DIR, './esm/throw error.ts'); + const expectedModuleUrl = pathToFileURL(expectedModulePath).toString(); + const expectedModulePrint = versionGteLt(process.versions.node, '20.0.0') ? expectedModulePath : expectedModuleUrl; expect(r.err!.message).toMatch( - [ - `${expectedModuleUrl}:100`, - " bar() { throw new Error('this is a demo'); }", - ' ^', - 'Error: this is a demo', - ` at Foo.bar (${expectedModuleUrl}:100:17)`, - ].join('\n') + [`${expectedModulePrint}:100`, " bar() { throw new Error('this is a demo'); }"].join('\n') ); + // the ^ and number of line-breaks is platform-specific + // also, node 20 puts the type in here when source mapping it, so it + // shows as Foo.Foo.bar + expect(r.err!.message).toMatch(/^ at (Foo\.){1,2}bar \(/m); + expect(r.err!.message).toMatch(`Foo.bar (${expectedModulePrint}:100:17)`); }); test.suite('supports experimental-specifier-resolution=node', (test) => { @@ -95,7 +98,7 @@ test.suite('esm', (test) => { }); expect(r.err).not.toBe(null); // expect error from node's default resolver - expect(r.stderr).toMatch(/Error \[ERR_UNSUPPORTED_ESM_URL_SCHEME\]:.*(?:\n.*){0,2}\n *at default(Load|Resolve)/); + expect(r.stderr).toMatch(/Error \[ERR_UNSUPPORTED_ESM_URL_SCHEME\]:.*(?:\n.*){0,10}\n *at default(Load|Resolve)/); }); test('should bypass import cache when changing search params', async () => { diff --git a/src/test/index.spec.ts b/src/test/index.spec.ts index 884808a77..417871239 100644 --- a/src/test/index.spec.ts +++ b/src/test/index.spec.ts @@ -182,12 +182,7 @@ test.suite('ts-node', (test) => { } expect(r.err.message).toMatch( - [ - `${join(TEST_DIR, 'throw error.ts')}:100`, - " bar() { throw new Error('this is a demo'); }", - ' ^', - 'Error: this is a demo', - ].join('\n') + [`${join(TEST_DIR, 'throw error.ts')}:100`, " bar() { throw new Error('this is a demo'); }"].join('\n') ); }); @@ -198,12 +193,7 @@ test.suite('ts-node', (test) => { } expect(r.err.message).toMatch( - [ - `${join(TEST_DIR, 'throw error.ts')}:100`, - " bar() { throw new Error('this is a demo'); }", - ' ^', - 'Error: this is a demo', - ].join('\n') + [`${join(TEST_DIR, 'throw error.ts')}:100`, " bar() { throw new Error('this is a demo'); }"].join('\n') ); }); @@ -214,11 +204,7 @@ test.suite('ts-node', (test) => { } expect(r.err.message).toMatch( - [ - `${join(TEST_DIR, 'throw error.ts')}:100`, - " bar() { throw new Error('this is a demo'); }", - ' ^', - ].join('\n') + [`${join(TEST_DIR, 'throw error.ts')}:100`, " bar() { throw new Error('this is a demo'); }"].join('\n') ); }); @@ -309,12 +295,9 @@ test.suite('ts-node', (test) => { const r = await exec(`${CMD_TS_NODE_WITH_PROJECT_FLAG} "throw error react tsx.tsx"`); expect(r.err).not.toBe(null); expect(r.err!.message).toMatch( - [ - `${join(TEST_DIR, './throw error react tsx.tsx')}:100`, - " bar() { throw new Error('this is a demo'); }", - ' ^', - 'Error: this is a demo', - ].join('\n') + [`${join(TEST_DIR, './throw error react tsx.tsx')}:100`, " bar() { throw new Error('this is a demo'); }"].join( + '\n' + ) ); }); @@ -322,12 +305,9 @@ test.suite('ts-node', (test) => { const r = await exec(`${CMD_TS_NODE_WITH_PROJECT_FLAG} --transpile-only "throw error react tsx.tsx"`); expect(r.err).not.toBe(null); expect(r.err!.message).toMatch( - [ - `${join(TEST_DIR, './throw error react tsx.tsx')}:100`, - " bar() { throw new Error('this is a demo'); }", - ' ^', - 'Error: this is a demo', - ].join('\n') + [`${join(TEST_DIR, './throw error react tsx.tsx')}:100`, " bar() { throw new Error('this is a demo'); }"].join( + '\n' + ) ); }); diff --git a/src/test/sourcemaps.spec.ts b/src/test/sourcemaps.spec.ts index 4aae34757..e51ebed04 100644 --- a/src/test/sourcemaps.spec.ts +++ b/src/test/sourcemaps.spec.ts @@ -1,4 +1,5 @@ import * as expect from 'expect'; +import { versionGteLt } from '../util'; import { createExec, createExecTester, CMD_TS_NODE_WITH_PROJECT_FLAG, ctxTsNode, TEST_DIR } from './helpers'; import { context } from './testlib'; const test = context(ctxTsNode); @@ -10,17 +11,23 @@ const exec = createExecTester({ }), }); -test('Redirects source-map-support to @cspotcode/source-map-support so that third-party libraries get correct source-mapped locations', async () => { - const r = await exec({ - flags: `./legacy-source-map-support-interop/index.ts`, +const useBuiltInSourceMaps = versionGteLt(process.versions.node, '20.0.0'); + +if (useBuiltInSourceMaps) { + test.skip('Skip source-map-support redirection on node 20', () => {}); +} else { + test('Redirects source-map-support to @cspotcode/source-map-support so that third-party libraries get correct source-mapped locations', async () => { + const r = await exec({ + flags: `./legacy-source-map-support-interop/index.ts`, + }); + expect(r.err).toBeNull(); + expect(r.stdout.split('\n')).toMatchObject([ + expect.stringContaining('.ts:2 '), + 'true', + 'true', + expect.stringContaining('.ts:100:'), + expect.stringContaining('.ts:101 '), + '', + ]); }); - expect(r.err).toBeNull(); - expect(r.stdout.split('\n')).toMatchObject([ - expect.stringContaining('.ts:2 '), - 'true', - 'true', - expect.stringContaining('.ts:100:'), - expect.stringContaining('.ts:101 '), - '', - ]); -}); +} From 5c83838bfd6371073add24c369584eb3387d481d Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 5 Sep 2023 16:21:16 -0700 Subject: [PATCH 11/14] test: line numbers in eval are not reliably source mapped Re: https://github.com/TypeStrong/ts-node/pull/2009#issuecomment-1707197817 --- src/test/repl/repl-environment.spec.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/test/repl/repl-environment.spec.ts b/src/test/repl/repl-environment.spec.ts index 0d8f2470b..f74117377 100644 --- a/src/test/repl/repl-environment.spec.ts +++ b/src/test/repl/repl-environment.spec.ts @@ -160,7 +160,7 @@ test.suite('[eval], , and [stdin] execute with correct globals', (test) => modulePaths, exportsTest: true, // Note: vanilla node uses different name. See #1360 - stackTest: expect.stringContaining(` at ${join(TEST_DIR, `[stdin].ts`)}:1:`), + stackTest: expect.stringContaining(` at ${join(TEST_DIR, `[stdin].ts`)}:`), moduleAccessorsTest: null, argv: [tsNodeExe], }, @@ -189,7 +189,7 @@ test.suite('[eval], , and [stdin] execute with correct globals', (test) => // Note: vanilla node REPL does not set exports exportsTest: true, // Note: vanilla node uses different name. See #1360 - stackTest: expect.stringContaining(` at ${join(TEST_DIR, replFile)}:4:`), + stackTest: expect.stringContaining(` at ${join(TEST_DIR, replFile)}:`), moduleAccessorsTest: true, argv: [tsNodeExe], }, @@ -237,7 +237,7 @@ test.suite('[eval], , and [stdin] execute with correct globals', (test) => modulePaths: [...modulePaths], exportsTest: true, // Note: vanilla node uses different name. See #1360 - stackTest: expect.stringContaining(` at ${join(TEST_DIR, `[eval].ts`)}:1:`), + stackTest: expect.stringContaining(` at ${join(TEST_DIR, `[eval].ts`)}:`), moduleAccessorsTest: true, argv: [tsNodeExe], }, @@ -262,7 +262,7 @@ test.suite('[eval], , and [stdin] execute with correct globals', (test) => modulePaths, exportsTest: true, // Note: vanilla node uses different name. See #1360 - stackTest: expect.stringContaining(` at ${join(TEST_DIR, `[eval].ts`)}:1:`), + stackTest: expect.stringContaining(` at ${join(TEST_DIR, `[eval].ts`)}:`), moduleAccessorsTest: true, argv: [tsNodeExe, './repl/script.js'], }, @@ -287,7 +287,7 @@ test.suite('[eval], , and [stdin] execute with correct globals', (test) => modulePaths, exportsTest: true, // Note: vanilla node uses different name. See #1360 - stackTest: expect.stringContaining(` at ${join(TEST_DIR, `[eval].ts`)}:1:`), + stackTest: expect.stringContaining(` at ${join(TEST_DIR, `[eval].ts`)}:`), moduleAccessorsTest: true, argv: [tsNodeExe, './does-not-exist.js'], }, @@ -312,7 +312,7 @@ test.suite('[eval], , and [stdin] execute with correct globals', (test) => modulePaths, exportsTest: true, // Note: vanilla node uses different name. See #1360 - stackTest: expect.stringContaining(` at ${join(TEST_DIR, `[eval].ts`)}:1:`), + stackTest: expect.stringContaining(` at ${join(TEST_DIR, `[eval].ts`)}:`), moduleAccessorsTest: true, argv: [tsNodeExe], }, @@ -404,7 +404,7 @@ test.suite('[eval], , and [stdin] execute with correct globals', (test) => // moduleAccessorsTest: true, // Note: vanilla node uses different name. See #1360 - stackTest: expect.stringContaining(` at ${join(TEST_DIR, replFile)}:1:`), + stackTest: expect.stringContaining(` at ${join(TEST_DIR, replFile)}:`), }, }); } @@ -433,7 +433,7 @@ test.suite('[eval], , and [stdin] execute with correct globals', (test) => // Note: vanilla node REPL does not set exports exportsTest: true, // Note: vanilla node uses different name. See #1360 - stackTest: expect.stringContaining(` at ${join(TEST_DIR, replFile)}:1:`), + stackTest: expect.stringContaining(` at ${join(TEST_DIR, replFile)}:`), moduleAccessorsTest: true, }, }); From 49c74a0eea5bcb9cb51190057782525d3a5b3dab Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 5 Sep 2023 17:32:52 -0700 Subject: [PATCH 12/14] add support for '--import ts-node/import' This also adds a type for the loader hooks API v3, as globalPreload is scheduled for removal in node v21, at which point '--loader ts-node/esm' will no longer work, and '--import ts-node/import' will be the way forward. --- README.md | 2 +- esm.mjs | 2 +- src/esm.ts | 39 +++++++++++++++++++++++++++++---------- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 40a40947f..cb2fb9981 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,7 @@ To test your version of `env` for compatibility with `-S`: ## node flags and other tools -You can register ts-node without using our CLI: `node -r ts-node/register` and `node --loader ts-node/esm` +You can register ts-node without using our CLI: `node -r ts-node/register`, `node --loader ts-node/esm`, or `node --import ts-node/import` in node 20.6 and above. In many cases, setting [`NODE_OPTIONS`](https://nodejs.org/api/cli.html#cli_node_options_options) will enable `ts-node` within other node tools, child processes, and worker threads. This can be combined with other node flags. diff --git a/esm.mjs b/esm.mjs index 4a4b2e323..07ed7008a 100644 --- a/esm.mjs +++ b/esm.mjs @@ -4,4 +4,4 @@ const require = createRequire(fileURLToPath(import.meta.url)); /** @type {import('./dist/esm')} */ const esm = require('./dist/esm'); -export const { resolve, load, getFormat, transformSource, globalPreload } = esm.registerAndCreateEsmHooks(); +export const { initialize, resolve, load, getFormat, transformSource, globalPreload } = esm.registerAndCreateEsmHooks(); diff --git a/src/esm.ts b/src/esm.ts index c4566e0df..8de6fb075 100644 --- a/src/esm.ts +++ b/src/esm.ts @@ -79,6 +79,17 @@ export namespace NodeLoaderHooksAPI2 { export type GlobalPreloadHook = (context?: { port: MessagePort }) => string; } +export interface NodeLoaderHooksAPI3 { + resolve: NodeLoaderHooksAPI2.ResolveHook; + load: NodeLoaderHooksAPI2.LoadHook; + initialize?: NodeLoaderHooksAPI3.InitializeHook; +} +export namespace NodeLoaderHooksAPI3 { + // technically this can be anything that can be passed through a postMessage channel, + // but defined here based on how ts-node uses it. + export type InitializeHook = (data: any) => void | Promise; +} + export type NodeLoaderHooksFormat = 'builtin' | 'commonjs' | 'dynamic' | 'json' | 'module' | 'wasm'; export type NodeImportConditions = unknown; @@ -87,17 +98,24 @@ export interface NodeImportAssertions { } // The hooks API changed in node version X so we need to check for backwards compatibility. -const newHooksAPI = versionGteLt(process.versions.node, '16.12.0'); +const hooksAPIVersion = versionGteLt(process.versions.node, '21.0.0') + ? 3 + : versionGteLt(process.versions.node, '16.12.0') + ? 2 + : 1; /** @internal */ export function filterHooksByAPIVersion( - hooks: NodeLoaderHooksAPI1 & NodeLoaderHooksAPI2 -): NodeLoaderHooksAPI1 | NodeLoaderHooksAPI2 { - const { getFormat, load, resolve, transformSource, globalPreload } = hooks; + hooks: NodeLoaderHooksAPI1 & NodeLoaderHooksAPI2 & NodeLoaderHooksAPI3 +): NodeLoaderHooksAPI1 | NodeLoaderHooksAPI2 | NodeLoaderHooksAPI3 { + const { getFormat, load, resolve, transformSource, globalPreload, initialize } = hooks; // Explicit return type to avoid TS's non-ideal inferred type - const hooksAPI: NodeLoaderHooksAPI1 | NodeLoaderHooksAPI2 = newHooksAPI - ? { resolve, load, globalPreload, getFormat: undefined, transformSource: undefined } - : { resolve, getFormat, transformSource, load: undefined }; + const hooksAPI: NodeLoaderHooksAPI1 | NodeLoaderHooksAPI2 | NodeLoaderHooksAPI3 = + hooksAPIVersion === 3 + ? { resolve, load, initialize, globalPreload: undefined, transformSource: undefined, getFormat: undefined } + : hooksAPIVersion === 2 + ? { resolve, load, globalPreload, initialize: undefined, getFormat: undefined, transformSource: undefined } + : { resolve, getFormat, transformSource, initialize: undefined, globalPreload: undefined, load: undefined }; return hooksAPI; } @@ -122,6 +140,7 @@ export function createEsmHooks(tsNodeService: Service) { getFormat, transformSource, globalPreload: useLoaderThread ? globalPreload : undefined, + initialize: undefined, }); function globalPreload({ port }: { port?: MessagePort } = {}) { @@ -129,9 +148,9 @@ export function createEsmHooks(tsNodeService: Service) { // so this signal lets us infer it based on the state of the main // thread, but only relevant if options.pretty is unset. let stderrTTYSignal: string; - if (tsNodeService.options.pretty === undefined) { - port?.on('message', (data) => { - if (data?.stderrIsTTY) { + if (port && tsNodeService.options.pretty === undefined) { + port.on('message', (data: { stderrIsTTY?: boolean }) => { + if (data.stderrIsTTY) { tsNodeService.setPrettyErrors(true); } }); From ed07dd96ea1afaeec5425059c85812e0a7493e19 Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 7 Sep 2023 12:54:32 -0700 Subject: [PATCH 13/14] test: allow windows line endings, node-version-specific path displays --- src/test/esm-loader.spec.ts | 14 +++++++------- src/test/index.spec.ts | 30 +++++++++--------------------- src/test/register.spec.ts | 6 +++--- 3 files changed, 19 insertions(+), 31 deletions(-) diff --git a/src/test/esm-loader.spec.ts b/src/test/esm-loader.spec.ts index 39d5c14c9..44626a38d 100644 --- a/src/test/esm-loader.spec.ts +++ b/src/test/esm-loader.spec.ts @@ -50,17 +50,15 @@ test.suite('esm', (test) => { }); expect(r.err).not.toBe(null); // on node 20, this will be a path. prior versions, a file: url - const expectedModulePath = join(TEST_DIR, './esm/throw error.ts'); - const expectedModuleUrl = pathToFileURL(expectedModulePath).toString(); - const expectedModulePrint = versionGteLt(process.versions.node, '20.0.0') ? expectedModulePath : expectedModuleUrl; + // on windows in node 20, it's a quasi-url like d:/path/to/throw%20error.ts expect(r.err!.message).toMatch( - [`${expectedModulePrint}:100`, " bar() { throw new Error('this is a demo'); }"].join('\n') + /[\\\/]throw( |%20)error\.ts:100\n bar\(\) \{ throw new Error\('this is a demo'\); \}/ ); // the ^ and number of line-breaks is platform-specific // also, node 20 puts the type in here when source mapping it, so it // shows as Foo.Foo.bar expect(r.err!.message).toMatch(/^ at (Foo\.){1,2}bar \(/m); - expect(r.err!.message).toMatch(`Foo.bar (${expectedModulePrint}:100:17)`); + expect(r.err!.message).toMatch(/^ at (Foo\.){1,2}bar ([^\n]+[\\\/]throw( |%20)error\.ts:100:17)`/m); }); test.suite('supports experimental-specifier-resolution=node', (test) => { @@ -97,8 +95,10 @@ test.suite('esm', (test) => { cwd: join(TEST_DIR, './esm-import-http-url'), }); expect(r.err).not.toBe(null); - // expect error from node's default resolver - expect(r.stderr).toMatch(/Error \[ERR_UNSUPPORTED_ESM_URL_SCHEME\]:.*(?:\n.*){0,10}\n *at default(Load|Resolve)/); + // expect error from node's default resolver, has a few different names in different node versions + expect(r.stderr).toMatch( + /Error \[ERR_UNSUPPORTED_ESM_URL_SCHEME\]:.*(?:\n.*){0,10}\n *at (default|next)(Load|Resolve)/ + ); }); test('should bypass import cache when changing search params', async () => { diff --git a/src/test/index.spec.ts b/src/test/index.spec.ts index 417871239..ff22c837b 100644 --- a/src/test/index.spec.ts +++ b/src/test/index.spec.ts @@ -181,9 +181,7 @@ test.suite('ts-node', (test) => { throw new Error('Command was expected to fail, but it succeeded.'); } - expect(r.err.message).toMatch( - [`${join(TEST_DIR, 'throw error.ts')}:100`, " bar() { throw new Error('this is a demo'); }"].join('\n') - ); + expect(r.err.message.replace(/\r\n/g, '\n')).toMatch(/throw( |%20)error\.ts:100\n bar\(\) \{ throw new Error\('this is a demo'\); \}/); }); test('should work with source maps in --transpile-only mode', async () => { @@ -192,9 +190,7 @@ test.suite('ts-node', (test) => { throw new Error('Command was expected to fail, but it succeeded.'); } - expect(r.err.message).toMatch( - [`${join(TEST_DIR, 'throw error.ts')}:100`, " bar() { throw new Error('this is a demo'); }"].join('\n') - ); + expect(r.err.message.replace(/\r\n/g, '\n')).toMatch(/throw( |%20)error\.ts:100\n bar\(\) \{ throw new Error\('this is a demo'\); \}/); }); test('eval should work with source maps', async () => { @@ -203,9 +199,7 @@ test.suite('ts-node', (test) => { throw new Error('Command was expected to fail, but it succeeded.'); } - expect(r.err.message).toMatch( - [`${join(TEST_DIR, 'throw error.ts')}:100`, " bar() { throw new Error('this is a demo'); }"].join('\n') - ); + expect(r.err.message.replace(/\r\n/g, '\n')).toMatch(/throw( |%20)error\.ts:100\n bar\(\) \{ throw new Error\('this is a demo'\); \}/); }); for (const flavor of [ @@ -294,20 +288,16 @@ test.suite('ts-node', (test) => { test('should use source maps with react tsx', async () => { const r = await exec(`${CMD_TS_NODE_WITH_PROJECT_FLAG} "throw error react tsx.tsx"`); expect(r.err).not.toBe(null); - expect(r.err!.message).toMatch( - [`${join(TEST_DIR, './throw error react tsx.tsx')}:100`, " bar() { throw new Error('this is a demo'); }"].join( - '\n' - ) + expect(r.err!.message.replace(/\r\n/g, '\n')).toMatch( + /throw( |%20)error( |%20)react( |%20)tsx\.tsx:100\n bar\(\) \{ throw new Error\('this is a demo'\); \}/ ); }); test('should use source maps with react tsx in --transpile-only mode', async () => { const r = await exec(`${CMD_TS_NODE_WITH_PROJECT_FLAG} --transpile-only "throw error react tsx.tsx"`); expect(r.err).not.toBe(null); - expect(r.err!.message).toMatch( - [`${join(TEST_DIR, './throw error react tsx.tsx')}:100`, " bar() { throw new Error('this is a demo'); }"].join( - '\n' - ) + expect(r.err!.message.replace(/\r\n/g, '\n')).toMatch( + /throw( |%20)error( |%20)react( |%20)tsx\.tsx:100\n bar\(\) \{ throw new Error\('this is a demo'\); \}/ ); }); @@ -420,8 +410,7 @@ test.suite('ts-node', (test) => { expect(r.err).not.toBe(null); expect(r.stderr.replace(/\r\n/g, '\n')).toMatch( 'TSError: ⨯ Unable to compile TypeScript:\n' + - "maxnodemodulesjsdepth/other.ts(4,7): error TS2322: Type 'string' is not assignable to type 'boolean'.\n" + - '\n' + "maxnodemodulesjsdepth/other.ts(4,7): error TS2322: Type 'string' is not assignable to type 'boolean'." ); }); @@ -430,8 +419,7 @@ test.suite('ts-node', (test) => { expect(r.err).not.toBe(null); expect(r.stderr.replace(/\r\n/g, '\n')).toMatch( 'TSError: ⨯ Unable to compile TypeScript:\n' + - "maxnodemodulesjsdepth-scoped/other.ts(7,7): error TS2322: Type 'string' is not assignable to type 'boolean'.\n" + - '\n' + "maxnodemodulesjsdepth-scoped/other.ts(7,7): error TS2322: Type 'string' is not assignable to type 'boolean'." ); }); }); diff --git a/src/test/register.spec.ts b/src/test/register.spec.ts index 9c9f512c2..b6e4cfa08 100644 --- a/src/test/register.spec.ts +++ b/src/test/register.spec.ts @@ -107,9 +107,9 @@ test.suite('register(create(options))', (test) => { try { require('../../tests/throw error'); } catch (error: any) { - exp(error.stack).toMatch( - ['Error: this is a demo', ` at Foo.bar (${join(TEST_DIR, './throw error.ts')}:100:17)`].join('\n') - ); + // on windows in node 20, this is printed as a quasi-url, like + // d:/path/to/throw%20error.ts + exp(error.stack).toMatch(/Error: this is a demo\n at Foo\.bar \([^)]+[\\\/]throw( %20)error\.ts:100:17\)\n/); } }); From b614b1be4d3dacbaf26b31a9320e31287eaf4774 Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 7 Sep 2023 12:54:32 -0700 Subject: [PATCH 14/14] test: allow win32 line endings, path displays --- src/test/esm-loader.spec.ts | 10 +++++----- src/test/index.spec.ts | 12 +++++++++--- src/test/register.spec.ts | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/test/esm-loader.spec.ts b/src/test/esm-loader.spec.ts index 44626a38d..022f9c9f8 100644 --- a/src/test/esm-loader.spec.ts +++ b/src/test/esm-loader.spec.ts @@ -51,14 +51,14 @@ test.suite('esm', (test) => { expect(r.err).not.toBe(null); // on node 20, this will be a path. prior versions, a file: url // on windows in node 20, it's a quasi-url like d:/path/to/throw%20error.ts - expect(r.err!.message).toMatch( + expect(r.err!.message.replace(/\r\n/g, '\n')).toMatch( /[\\\/]throw( |%20)error\.ts:100\n bar\(\) \{ throw new Error\('this is a demo'\); \}/ ); // the ^ and number of line-breaks is platform-specific // also, node 20 puts the type in here when source mapping it, so it // shows as Foo.Foo.bar - expect(r.err!.message).toMatch(/^ at (Foo\.){1,2}bar \(/m); - expect(r.err!.message).toMatch(/^ at (Foo\.){1,2}bar ([^\n]+[\\\/]throw( |%20)error\.ts:100:17)`/m); + expect(r.err!.message.replace(/\r\n/g, '\n')).toMatch(/^ at (Foo\.){1,2}bar \(/m); + expect(r.err!.message.replace(/\r\n/g, '\n')).toMatch(/^ at (Foo\.){1,2}bar ([^\n\)]+[\\\/]throw( |%20)error\.ts:100:17)/m); }); test.suite('supports experimental-specifier-resolution=node', (test) => { @@ -96,7 +96,7 @@ test.suite('esm', (test) => { }); expect(r.err).not.toBe(null); // expect error from node's default resolver, has a few different names in different node versions - expect(r.stderr).toMatch( + expect(r.stderr.replace(/\r\n/g, '\n')).toMatch( /Error \[ERR_UNSUPPORTED_ESM_URL_SCHEME\]:.*(?:\n.*){0,10}\n *at (default|next)(Load|Resolve)/ ); }); @@ -106,7 +106,7 @@ test.suite('esm', (test) => { cwd: join(TEST_DIR, './esm-import-cache'), }); expect(r.err).toBe(null); - expect(r.stdout).toBe('log1\nlog2\nlog2\n'); + expect(r.stdout.replace(/\r\n/g, '\n')).toBe('log1\nlog2\nlog2\n'); }); test('should support transpile only mode via dedicated loader entrypoint', async () => { diff --git a/src/test/index.spec.ts b/src/test/index.spec.ts index ff22c837b..f23c662ac 100644 --- a/src/test/index.spec.ts +++ b/src/test/index.spec.ts @@ -181,7 +181,9 @@ test.suite('ts-node', (test) => { throw new Error('Command was expected to fail, but it succeeded.'); } - expect(r.err.message.replace(/\r\n/g, '\n')).toMatch(/throw( |%20)error\.ts:100\n bar\(\) \{ throw new Error\('this is a demo'\); \}/); + expect(r.err.message.replace(/\r\n/g, '\n')).toMatch( + /throw( |%20)error\.ts:100\n bar\(\) \{ throw new Error\('this is a demo'\); \}/ + ); }); test('should work with source maps in --transpile-only mode', async () => { @@ -190,7 +192,9 @@ test.suite('ts-node', (test) => { throw new Error('Command was expected to fail, but it succeeded.'); } - expect(r.err.message.replace(/\r\n/g, '\n')).toMatch(/throw( |%20)error\.ts:100\n bar\(\) \{ throw new Error\('this is a demo'\); \}/); + expect(r.err.message.replace(/\r\n/g, '\n')).toMatch( + /throw( |%20)error\.ts:100\n bar\(\) \{ throw new Error\('this is a demo'\); \}/ + ); }); test('eval should work with source maps', async () => { @@ -199,7 +203,9 @@ test.suite('ts-node', (test) => { throw new Error('Command was expected to fail, but it succeeded.'); } - expect(r.err.message.replace(/\r\n/g, '\n')).toMatch(/throw( |%20)error\.ts:100\n bar\(\) \{ throw new Error\('this is a demo'\); \}/); + expect(r.err.message.replace(/\r\n/g, '\n')).toMatch( + /throw( |%20)error\.ts:100\n bar\(\) \{ throw new Error\('this is a demo'\); \}/ + ); }); for (const flavor of [ diff --git a/src/test/register.spec.ts b/src/test/register.spec.ts index b6e4cfa08..2cae6de32 100644 --- a/src/test/register.spec.ts +++ b/src/test/register.spec.ts @@ -109,7 +109,7 @@ test.suite('register(create(options))', (test) => { } catch (error: any) { // on windows in node 20, this is printed as a quasi-url, like // d:/path/to/throw%20error.ts - exp(error.stack).toMatch(/Error: this is a demo\n at Foo\.bar \([^)]+[\\\/]throw( %20)error\.ts:100:17\)\n/); + exp(error.stack).toMatch(/Error: this is a demo\n at Foo\.bar \([^)]+[\\\/]throw( |%20)error\.ts:100:17\)\n/); } });