Transition to ECMAScript Modules (compiler-explorer#4780)

Makes the Compiler Explorer app, and all the tooling ESM compatible.
Things that have been done:

1. The package.json has `type: module` now
2. All relative imports have a .js ending
3. All directory imports are now directory/index.js to comply with ESM
standards
4. Dependency node-graceful is now imported into tree, because the
package is broken under esm
5. Dependency p-queue has been bumped to 7.x with ESM support
6. Dependency profanities has been bumped to 3.x with ESM support
7. Webpack config is now both ESM and CommonJS compatible
8. Non-ESM compatible imports have been rewritten
9. ESLint configuration has been tweaked to not fail on .js imports
10. Mocha is now hacked together and ran with ts-node-esm
11. Webpack is now hacked together and ran with ts-node-esm
12. Webpack config is now ESM compatible, so that it can be used in the
dev server
13. Cypress code still runs commonjs, and has been excluded from the
tsconfig
14. All sinon mock tests have been commented out, because sinon module
mocks do not work with ESModules (because ESModules are immutable)

A lot of tests are now giving warnings/errors to stdout, yet still pass.
Docenizer codegenerator scripts have been updated, but I did not re-run
them, and instead just changed their code.

---------

Co-authored-by: Matt Godbolt <[email protected]>

Author: junlarsen

.eslintrc.yml

@@ -52,6 +52,7 @@ rules:
   import/no-self-import: error
   import/no-useless-path-segments: error
   import/no-webpack-loader-syntax: error
+  import/no-unresolved: off # Doesn't like .js files
   import/order:
     - error
     - alphabetize:
@@ -121,6 +122,7 @@ rules:
   no-process-exit: off
   node/no-unsupported-features/es-syntax: off
   node/shebang: off
+  node/no-missing-import: off # Doesn't like .js imports of .interfaces.ts files
   promise/catch-or-return: error
   promise/no-new-statics: error
   promise/no-return-wrap: error

.gitignore

@@ -2,9 +2,9 @@
 # build generated files
 /.node-bin
 /.npm-updated
-/.nyc_output
 /node_modules
 /out
+/coverage
 
 # generated from opcode docenzier.py scripts
 /asm-docs*

.idea/codeStyles/Project.xml

@@ -1,6 +1,2 @@
 recursive: true
 ignore: test/filter-tests.js
-require:
-  - esm
-  - ts-node/register
-  - source-map-support/register

.idea/jsLibraryMappings.xml

@@ -1,5 +1 @@
 recursive: true
-require:
-  - esm
-  - ts-node/register
-  - source-map-support/register

.idea/runConfigurations/app_js.xml

@@ -4,9 +4,9 @@
 # build generated files
 /.node-bin
 /.npm-updated
-/.nyc_output
 /node_modules
 /out
+/coverage
 
 # generated from opcode docenzier.py scripts
 /asm-docs*
@@ -36,6 +36,7 @@ newrelic_agent.log
 etc
 examples
 out
+cypress
 
 # Autogenerated files
 lib/asm-docs/generated/asm-docs-*

.mocharc-min.yml

@@ -77,7 +77,7 @@ prebuild: prereqs
 
 .PHONY: run-only
 run-only: node-installed  ## Runs the site like it runs in production without building it
-	env NODE_ENV=production $(NODE) $(NODE_ARGS) -r esm ./out/dist/app.js --webpackContent ./out/webpack/static $(EXTRA_ARGS)
+	env NODE_ENV=production $(NODE) $(NODE_ARGS) ./out/dist/app.js --webpackContent ./out/webpack/static $(EXTRA_ARGS)
 
 .PHONY: run
 run:  ## Runs the site like it runs in production
@@ -86,15 +86,15 @@ run:  ## Runs the site like it runs in production
 
 .PHONY: dev
 dev: prereqs ## Runs the site as a developer; including live reload support and installation of git hooks
-	./node_modules/.bin/supervisor -w app.ts,lib,etc/config,static/tsconfig.json -e 'js|ts|node|properties|yaml' -n exit --exec $(NODE) $(NODE_ARGS) -- -r esm -r ts-node/register ./app.ts $(EXTRA_ARGS)
+	NODE_OPTIONS=$(NODE_ARGS) ./node_modules/.bin/supervisor -w app.ts,lib,etc/config,static/tsconfig.json -e 'js|ts|node|properties|yaml' -n exit --exec $(shell pwd)/node_modules/.bin/ts-node-esm -- ./app.ts $(EXTRA_ARGS)
 
 .PHONY: gpu-dev
 gpu-dev: prereqs ## Runs the site as a developer; including live reload support and installation of git hooks
-	./node_modules/.bin/supervisor -w app.ts,lib,etc/config,static/tsconfig.json -e 'js|ts|node|properties|yaml' -n exit --exec $(NODE) $(NODE_ARGS) -- -r esm -r ts-node/register ./app.ts --env gpu $(EXTRA_ARGS)
+	NODE_OPTIONS=$(NODE_ARGS) ./node_modules/.bin/supervisor -w app.ts,lib,etc/config,static/tsconfig.json -e 'js|ts|node|properties|yaml' -n exit --exec $(shell pwd)/node_modules/.bin/ts-node-esm -- ./app.ts --env gpu $(EXTRA_ARGS)
 
 .PHONY: debug
 debug: prereqs ## Runs the site as a developer with full debugging; including live reload support and installation of git hooks
-	./node_modules/.bin/supervisor -w app.ts,lib,etc/config,static/tsconfig.json -e 'js|ts|node|properties|yaml' -n exit --inspect 9229 --exec $(NODE) $(NODE_ARGS) -- -r esm -r ts-node/register ./app.ts --debug $(EXTRA_ARGS)
+	NODE_OPTIONS="$(NODE_ARGS) --inspect 9229" ./node_modules/.bin/supervisor -w app.ts,lib,etc/config,static/tsconfig.json -e 'js|ts|node|properties|yaml' -n exit --exec $(shell pwd)/node_modules/.bin/ts-node-esm -- ./app.ts --debug $(EXTRA_ARGS)
 
 .PHONY:
 asm-docs:

.mocharc.yml

@@ -43,34 +43,34 @@ import systemdSocket from 'systemd-socket';
 import _ from 'underscore';
 import urljoin from 'url-join';
 
-import * as aws from './lib/aws';
-import * as normalizer from './lib/clientstate-normalizer';
-import {ElementType} from './lib/common-utils';
-import {CompilationEnvironment} from './lib/compilation-env';
-import {CompilationQueue} from './lib/compilation-queue';
-import {CompilerFinder} from './lib/compiler-finder';
-// import { policy as csp } from './lib/csp';
-import {startWineInit} from './lib/exec';
-import {CompileHandler} from './lib/handlers/compile';
-import * as healthCheck from './lib/handlers/health-check';
-import {NoScriptHandler} from './lib/handlers/noscript';
-import {RouteAPI} from './lib/handlers/route-api';
-import {loadSiteTemplates} from './lib/handlers/site-templates';
-import {SourceHandler} from './lib/handlers/source';
-import {languages as allLanguages} from './lib/languages';
-import {logger, logToLoki, logToPapertrail, makeLogStream, suppressConsoleLog} from './lib/logger';
-import {setupMetricsServer} from './lib/metrics-server';
-import {ClientOptionsHandler} from './lib/options-handler';
-import * as props from './lib/properties';
-import {ShortLinkResolver} from './lib/shortener/google';
-import {sources} from './lib/sources';
-import {loadSponsorsFromString} from './lib/sponsors';
-import {getStorageTypeByKey} from './lib/storage';
-import * as utils from './lib/utils';
-import {Language, LanguageKey} from './types/languages.interfaces';
+import * as aws from './lib/aws.js';
+import * as normalizer from './lib/clientstate-normalizer.js';
+import {ElementType} from './lib/common-utils.js';
+import {CompilationEnvironment} from './lib/compilation-env.js';
+import {CompilationQueue} from './lib/compilation-queue.js';
+import {CompilerFinder} from './lib/compiler-finder.js';
+// import { policy as csp } from './lib/csp.js';
+import {startWineInit} from './lib/exec.js';
+import {CompileHandler} from './lib/handlers/compile.js';
+import * as healthCheck from './lib/handlers/health-check.js';
+import {NoScriptHandler} from './lib/handlers/noscript.js';
+import {RouteAPI} from './lib/handlers/route-api.js';
+import {loadSiteTemplates} from './lib/handlers/site-templates.js';
+import {SourceHandler} from './lib/handlers/source.js';
+import {languages as allLanguages} from './lib/languages.js';
+import {logger, logToLoki, logToPapertrail, makeLogStream, suppressConsoleLog} from './lib/logger.js';
+import {setupMetricsServer} from './lib/metrics-server.js';
+import {ClientOptionsHandler} from './lib/options-handler.js';
+import * as props from './lib/properties.js';
+import {ShortLinkResolver} from './lib/shortener/google.js';
+import {sources} from './lib/sources/index.js';
+import {loadSponsorsFromString} from './lib/sponsors.js';
+import {getStorageTypeByKey} from './lib/storage/index.js';
+import * as utils from './lib/utils.js';
+import type {Language, LanguageKey} from './types/languages.interfaces.js';
 
 // Used by assert.ts
-global.ce_base_directory = __dirname; // eslint-disable-line unicorn/prefer-module
+global.ce_base_directory = new URL('.', import.meta.url);
 
 // Parse arguments from command line 'node ./app.js args...'
 const opts = nopt({

.nycrc.yml

@@ -28,11 +28,11 @@
 import nopt from 'nopt';
 import _ from 'underscore';
 
-import {CompilerArguments} from './lib/compiler-arguments';
-import * as Parsers from './lib/compilers/argument-parsers';
-import {executeDirect} from './lib/exec';
-import {logger} from './lib/logger';
-import {padRight} from './lib/utils';
+import {CompilerArguments} from './lib/compiler-arguments.js';
+import * as Parsers from './lib/compilers/argument-parsers.js';
+import {executeDirect} from './lib/exec.js';
+import {logger} from './lib/logger.js';
+import {padRight} from './lib/utils.js';
 
 const opts = nopt({
     parser: [String],

.prettierignore

@@ -0,0 +1,12 @@
+{
+  "extends": "../tsconfig.base.json",
+  "compilerOptions": {
+    /* Module resolution */
+    "target": "esnext",
+    "module": "commonjs",
+    "moduleResolution": "classic",
+    /* Code generation */
+    "typeRoots": ["../node_modules/@types"],
+    "types": ["mocha", "chai", "chai-http"],
+  }
+}

.prettierrc.js renamed to .prettierrc.cjs

@@ -19,7 +19,7 @@ If you want to add a new language to the site, you should follow this steps:
 
   - If the language is supported by Monaco Editor (You can find the list
     [here](https://github.com/microsoft/monaco-editor/tree/main/src/basic-languages)), you should add it to the list of
-    languages inside the `MonacoEditorWebpackPlugin` config in `webpack.config.esm.js`
+    languages inside the `MonacoEditorWebpackPlugin` config in `webpack.config.esm.ts`
   - If not, you should implement your own language mode; see `static/modes/asm-mode.js` as an example. Don't forget to
     _require_ your mode file in `static/modes/_all.ts`, in alphabetical order
   - `language-key` is how your language will be referred internally by the code. In the rest of this document, replace

Makefile

@@ -54,7 +54,7 @@ Add-Content -Path $env:GITHUB_OUTPUT -Value "branch=$BRANCH"
 Add-Content -Path $env:GITHUB_OUTPUT -Value "release_name=$RELEASE_NAME"
 
 # Run to make sure we haven't just made something that won't work
-node -r esm ./app.js --version --dist
+../../node_modules/.bin/ts-node-esm ./app.js --version --dist
 
 Remove-Item -Path "$ROOT/out/dist-bin" -Force -Recurse  -ErrorAction Ignore
 New-Item -Path $ROOT -Name "out/dist-bin" -Force -ItemType "directory"

app.ts

@@ -45,7 +45,7 @@ echo "branch=${GITHUB_REF#refs/heads/}" >> "${GITHUB_OUTPUT}"
 echo "release_name=${RELEASE_NAME}" >> "${GITHUB_OUTPUT}"
 
 # Run to make sure we haven't just made something that won't work
-node -r esm ./app.js --version --dist
+../../node_modules/.bin/ts-node-esm ./app.js --version --dist
 
 rm -rf "${ROOT}/out/dist-bin"
 mkdir -p "${ROOT}/out/dist-bin"

compiler-args-app.ts

@@ -135,7 +135,7 @@ def parse_descriptions(line, line_num, cpu_type, instructions):
 
 
 def write_script(filename, instructions):
-    script = ["import {AssemblyInstructionInfo} from '../base';",
+    script = ["import {AssemblyInstructionInfo} from '../base.js';",
               "",
               "export function getAsmOpcode(opcode: string | undefined): AssemblyInstructionInfo | undefined {",
               "    if (!opcode) return;",

cypress/tsconfig.json

@@ -354,7 +354,7 @@ def main():
     print(f"Writing {len(instructions)} instructions")
     with open(args.outputpath, 'w') as f:
         f.write("""
-import {AssemblyInstructionInfo} from '../base';
+import {AssemblyInstructionInfo} from '../base.js';
 
 export function getAsmOpcode(opcode: string | undefined): AssemblyInstructionInfo | undefined {
     if (!opcode) return;

docs/AddingALanguage.md

@@ -179,7 +179,7 @@ def docenizer():
     print("Writing {} instructions".format(len(instructions)))
     with open(args.outputpath, 'w') as f:
         f.write("""
-import {AssemblyInstructionInfo} from '../base';
+import {AssemblyInstructionInfo} from '../base.js';
 
 export function getAsmOpcode(opcode: string | undefined): AssemblyInstructionInfo | undefined {
     if (!opcode) return;

etc/scripts/build-dist-win.ps1

@@ -89,7 +89,7 @@ def process_description(desc):
 def write_script(filename, instructions):
     log_message(f"writing to {filename}...")
     with open(filename, "w") as script:
-        script.write("import {AssemblyInstructionInfo} from '../base';\n")
+        script.write("import {AssemblyInstructionInfo} from '../base.js';\n")
         script.write("\n")
         script.write("export function getAsmOpcode(opcode: string | undefined): AssemblyInstructionInfo | undefined {\n")
         script.write("    if (!opcode) return;\n")

etc/scripts/build-dist.sh

@@ -126,7 +126,7 @@ def main():
     print(f"Writing {len(instructions)} instructions")
     with open(args.outputpath, 'w') as f:
         f.write("""
-import {AssemblyInstructionInfo} from '../base';
+import {AssemblyInstructionInfo} from '../base.js';
 
 export function getAsmOpcode(opcode: string | undefined): AssemblyInstructionInfo | undefined {
     if (!opcode) return;

etc/scripts/docenizers/docenizer-6502.py

@@ -87,7 +87,7 @@ const main = async () => {
         .slice(1) // Drop 1 because the first is the "mne monic"
         .map(it => extract($(it), $))
         .flat();
-    console.log('import {AssemblyInstructionInfo} from \'../base\';');
+    console.log('import {AssemblyInstructionInfo} from \'../base.js\';');
     console.log('');
     console.log('export function getAsmOpcode(opcode: string | undefined): AssemblyInstructionInfo | undefined {');
     console.log('    if (!opcode) return;');

etc/scripts/docenizers/docenizer-amd64.py

@@ -49,7 +49,7 @@ const $ = cheerio.load(contents);
 const names = getInstructionList($.root(), $);
 const info = names.map((x) => getInstructionInfo(x, $.root(), $));
 
-console.log('import {AssemblyInstructionInfo} from \'../base\';');
+console.log('import {AssemblyInstructionInfo} from \'../base.js\';');
 console.log('');
 console.log('export function getAsmOpcode(opcode: string | undefined): AssemblyInstructionInfo | undefined {');
 console.log('    if (!opcode) return;');

etc/scripts/docenizers/docenizer-arm32.py

@@ -114,7 +114,7 @@ def main():
     print(f"Writing {len(instructions)} instructions")
     with open(args.outputpath, 'w') as f:
         f.write("""
-import {AssemblyInstructionInfo} from '../base';
+import {AssemblyInstructionInfo} from '../base.js';
 
 export function getAsmOpcode(opcode: string | undefined): AssemblyInstructionInfo | undefined {
     if (!opcode) return;

etc/scripts/docenizers/docenizer-avr.py

@@ -0,0 +1,3 @@
+{
+  "type": "commonjs"
+}

etc/scripts/docenizers/docenizer-evm.py

@@ -1,7 +1,7 @@
-import {execSync} from 'child_process';
-import {getHashDigest} from 'loader-utils';
-import * as pug from 'pug';
-import path from 'path';
+const { execSync } = require('child_process');
+const { getHashDigest } = require('loader-utils');
+const pug = require('pug');
+const path = require('path');
 
 // If you edit either cookies.pug or privacy.pug be aware this will trigger a popup on the users' next visit.
 // Knowing the last versions here helps us be aware when this happens. If you get an error here and you _haven't_
@@ -20,7 +20,7 @@ function _execGit(command) {
     return gitResult.toString();
 }
 
-export default function(content) {
+module.exports = function(content) {
     const filePath = this.resourcePath;
     const filename = path.basename(filePath, '.pug');
     const options = this.getOptions();

etc/scripts/docenizers/docenizer-java.ts

@@ -54,14 +54,16 @@ module.exports = {
                 sourceType: 'module',
                 ecmaVersion: 'latest',
                 tsconfigRootDir: __dirname,
-                project: './tsconfig.json',
+                project: '../tsconfig.json',
             },
             rules: {
                 '@typescript-eslint/no-empty-function': 'off',
                 '@typescript-eslint/no-unused-vars': 'off',
                 '@typescript-eslint/no-var-requires': 'error',
                 '@typescript-eslint/no-explicit-any': 'off', // Too much js code still exists
                 '@typescript-eslint/ban-ts-comment': 'error',
+                'import/no-unresolved': 'off',
+                'node/no-missing-imports': 'off',
                 // TODO: Disabled for now
                 //'@typescript-eslint/no-unnecessary-condition': 'error',
                 //'@typescript-eslint/no-unnecessary-type-assertion': 'error',

etc/scripts/docenizers/docenizer-llvm.ts

@@ -22,11 +22,11 @@
 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 // POSSIBILITY OF SUCH DAMAGE.
 
-export {Amd64DocumentationProvider} from './amd64';
-export {Arm32DocumentationProvider} from './arm32';
-export {AvrDocumentationProvider} from './avr';
-export {EvmDocumentationProvider} from './evm';
-export {JavaDocumentationProvider} from './java';
-export {LLVMDocumentationProvider} from './llvm';
-export {Mos6502DocumentationProvider} from './mos6502';
-export {PythonDocumentationProvider} from './python';
+export {Amd64DocumentationProvider} from './amd64.js';
+export {Arm32DocumentationProvider} from './arm32.js';
+export {AvrDocumentationProvider} from './avr.js';
+export {EvmDocumentationProvider} from './evm.js';
+export {JavaDocumentationProvider} from './java.js';
+export {LLVMDocumentationProvider} from './llvm.js';
+export {Mos6502DocumentationProvider} from './mos6502.js';
+export {PythonDocumentationProvider} from './python.js';

etc/scripts/docenizers/docenizer-python.py

@@ -22,8 +22,8 @@
 // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 // POSSIBILITY OF SUCH DAMAGE.
 
-import {AssemblyInstructionInfo, BaseAssemblyDocumentationProvider} from './base';
-import {getAsmOpcode} from './generated/asm-docs-amd64';
+import {AssemblyInstructionInfo, BaseAssemblyDocumentationProvider} from './base.js';
+import {getAsmOpcode} from './generated/asm-docs-amd64.js';
 
 export class Amd64DocumentationProvider extends BaseAssemblyDocumentationProvider {
     private static readonly ATT_SUFFIX_REMOVER = /^([a-z]+)[blqw]$/i;