nim doc --backend:js, nim doc --doccmd:-d:foo, nim r --backend:js, --doccmd:skip + other improvements (#14278)

* `nim doc --backend:js|cpp...`
`nim doc --doccmd:'-d:foo --threads:on'`
`nim r --backend:cpp...` (implies --run --usenimcache)
* --usenimcache works with all targets
* --docCmd:skip now skips compiling snippets; 50X speedup for doc/manual.rst

Author: timotheecour

compiler/ccgthreadvars.nim

@@ -46,7 +46,7 @@ proc generateThreadLocalStorage(m: BModule) =
 
 proc generateThreadVarsSize(m: BModule) =
   if m.g.nimtv != nil:
-    let externc = if m.config.cmd == cmdCompileToCpp or
+    let externc = if m.config.backend == backendCpp or
                        sfCompileToCpp in m.module.flags: "extern \"C\" "
                   else: ""
     m.s[cfsProcs].addf(

compiler/cgen.nim

@@ -279,7 +279,7 @@ proc genProc(m: BModule, prc: PSym)
 proc raiseInstr(p: BProc): Rope
 
 template compileToCpp(m: BModule): untyped =
-  m.config.cmd == cmdCompileToCpp or sfCompileToCpp in m.module.flags
+  m.config.backend == backendCpp or sfCompileToCpp in m.module.flags
 
 proc getTempName(m: BModule): Rope =
   result = m.tmpBase & rope(m.labels)
@@ -1066,11 +1066,11 @@ proc genProcAux(m: BModule, prc: PSym) =
 proc requiresExternC(m: BModule; sym: PSym): bool {.inline.} =
   result = (sfCompileToCpp in m.module.flags and
            sfCompileToCpp notin sym.getModule().flags and
-           m.config.cmd != cmdCompileToCpp) or (
+           m.config.backend != backendCpp) or (
            sym.flags * {sfInfixCall, sfCompilerProc, sfMangleCpp} == {} and
            sym.flags * {sfImportc, sfExportc} != {} and
            sym.magic == mNone and
-           m.config.cmd == cmdCompileToCpp)
+           m.config.backend == backendCpp)
 
 proc genProcPrototype(m: BModule, sym: PSym) =
   useHeader(m, sym)
@@ -1867,7 +1867,7 @@ proc writeHeader(m: BModule) =
 proc getCFile(m: BModule): AbsoluteFile =
   let ext =
       if m.compileToCpp: ".nim.cpp"
-      elif m.config.cmd == cmdCompileToOC or sfCompileToObjc in m.module.flags: ".nim.m"
+      elif m.config.backend == backendObjc or sfCompileToObjc in m.module.flags: ".nim.m"
       else: ".nim.c"
   result = changeFileExt(completeCfilePath(m.config, withPackageName(m.config, m.cfilename)), ext)
 

compiler/cmdlinehelper.nim

@@ -78,7 +78,8 @@ proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: Confi
   # XXX This is hacky. We need to find a better way.
   case conf.command
   of "cpp", "compiletocpp":
-    conf.cmd = cmdCompileToCpp
+    conf.backend = backendCpp
+    conf.cmd = cmdCompileToBackend
   else:
     discard
 

compiler/commands.nim

@@ -434,11 +434,18 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "outdir":
     expectArg(conf, switch, arg, pass, info)
     conf.outDir = processPath(conf, arg, info, notRelativeToProj=true)
+  of "usenimcache":
+    processOnOffSwitchG(conf, {optUseNimcache}, arg, pass, info)
   of "docseesrcurl":
     expectArg(conf, switch, arg, pass, info)
     conf.docSeeSrcUrl = arg
   of "docroot":
     conf.docRoot = if arg.len == 0: "@default" else: arg
+  of "backend", "b":
+    let backend = parseEnum(arg.normalize, TBackend.default)
+    if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg)
+    conf.backend = backend
+  of "doccmd": conf.docCmd = arg
   of "mainmodule", "m":
     discard "allow for backwards compatibility, but don't do anything"
   of "define", "d":
@@ -495,7 +502,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
         defineSymbol(conf.symbols, "gcmarkandsweep")
       of "destructors", "arc":
         conf.selectedGC = gcArc
-        if conf.cmd != cmdCompileToCpp:
+        if conf.backend != backendCpp:
           conf.exc = excGoto
         defineSymbol(conf.symbols, "gcdestructors")
         defineSymbol(conf.symbols, "gcarc")
@@ -506,7 +513,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
           defineSymbol(conf.symbols, "nimV2")
       of "orc":
         conf.selectedGC = gcOrc
-        if conf.cmd != cmdCompileToCpp:
+        if conf.backend != backendCpp:
           conf.exc = excGoto
         defineSymbol(conf.symbols, "gcdestructors")
         defineSymbol(conf.symbols, "gcorc")

compiler/docgen.nim

@@ -22,6 +22,7 @@ import
 const
   exportSection = skField
   htmldocsDir = RelativeDir"htmldocs"
+  docCmdSkip = "skip"
 
 type
   TSections = array[TSymKind, Rope]
@@ -196,6 +197,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
   initStrTable result.types
   result.onTestSnippet =
     proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) =
+      if conf.docCmd == docCmdSkip: return
       inc(gen.id)
       var d = TDocumentor(gen)
       var outp: AbsoluteFile
@@ -444,21 +446,26 @@ proc testExample(d: PDoc; ex: PNode) =
   d.examples.add "import r\"" & outp.string & "\"\n"
 
 proc runAllExamples(d: PDoc) =
-  if d.examples.len == 0: return
+  let docCmd = d.conf.docCmd
+  let backend = d.conf.backend
+  # This used to be: `let backend = if isDefined(d.conf, "js"): "js"` (etc), however
+  # using `-d:js` (etc) cannot work properly, eg would fail with `importjs`
+  # since semantics are affected by `config.backend`, not by isDefined(d.conf, "js")
+  if d.examples.len == 0 or docCmd == docCmdSkip: return
   let outputDir = d.conf.getNimcacheDir / RelativeDir"runnableExamples"
   let outp = outputDir / RelativeFile(extractFilename(d.filename.changeFileExt"" &
       "_examples.nim"))
   writeFile(outp, d.examples)
-  let backend = if isDefined(d.conf, "js"): "js"
-                elif isDefined(d.conf, "cpp"): "cpp"
-                elif isDefined(d.conf, "objc"): "objc"
-                else: "c"
-  if os.execShellCmd(os.getAppFilename() & " " & backend &
-                    " --warning[UnusedImport]:off" &
-                    " --path:" & quoteShell(d.conf.projectPath) &
-                    " --nimcache:" & quoteShell(outputDir) &
-                    " -r " & quoteShell(outp)) != 0:
-    quit "[Examples] failed: see " & outp.string
+  let cmd = "$nim $backend -r --warning:UnusedImport:off --path:$path --nimcache:$nimcache $docCmd $file" % [
+    "nim", os.getAppFilename(),
+    "backend", $d.conf.backend,
+    "path", quoteShell(d.conf.projectPath),
+    "nimcache", quoteShell(outputDir),
+    "file", quoteShell(outp),
+    "docCmd", docCmd,
+  ]
+  if os.execShellCmd(cmd) != 0:
+    quit "[runnableExamples] failed: generated file: '$1' cmd: $2" % [outp.string, cmd]
   else:
     # keep generated source file `outp` to allow inspection.
     rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string])

compiler/extccomp.nim

@@ -307,14 +307,12 @@ proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string =
   # use ``cpu.os.cc`` for cross compilation, unless ``--compileOnly`` is given
   # for niminst support
   let fullSuffix =
-    if conf.cmd == cmdCompileToCpp:
-      ".cpp" & suffix
-    elif conf.cmd == cmdCompileToOC:
-      ".objc" & suffix
-    elif conf.cmd == cmdCompileToJS:
-      ".js" & suffix
+    case conf.backend
+    of backendCpp, backendJs, backendObjc: "." & $conf.backend & suffix
+    of backendC: suffix
     else:
-      suffix
+      doAssert false
+      ""
 
   if (conf.target.hostOS != conf.target.targetOS or conf.target.hostCPU != conf.target.targetCPU) and
       optCompileOnly notin conf.globalOptions:
@@ -481,10 +479,10 @@ proc needsExeExt(conf: ConfigRef): bool {.inline.} =
            (conf.target.hostOS == osWindows)
 
 proc useCpp(conf: ConfigRef; cfile: AbsoluteFile): bool =
-  conf.cmd == cmdCompileToCpp and not cfile.string.endsWith(".c")
+  conf.backend == backendCpp and not cfile.string.endsWith(".c")
 
 proc envFlags(conf: ConfigRef): string =
-  result = if conf.cmd == cmdCompileToCpp:
+  result = if conf.backend == backendCpp:
             getEnv("CXXFLAGS")
           else:
             getEnv("CFLAGS")
@@ -535,7 +533,7 @@ proc ccHasSaneOverflow*(conf: ConfigRef): bool =
 
 proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string =
   result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe
-           elif optMixedMode in conf.globalOptions and conf.cmd != cmdCompileToCpp: CC[compiler].cppCompiler
+           elif optMixedMode in conf.globalOptions and conf.backend != backendCpp: CC[compiler].cppCompiler
            else: getCompilerExe(conf, compiler, AbsoluteFile"")
 
 proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
@@ -626,8 +624,10 @@ proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash =
     getCompileCFileCmd(conf, cfile))
 
 proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
-  if conf.cmd notin {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToLLVM, cmdNone}:
-    return false
+  case conf.backend
+  of backendInvalid: doAssert false
+  of backendJs: return false # pre-existing behavior, but not sure it's good
+  else: discard
 
   var hashFile = toGeneratedFile(conf, conf.withPackageName(cfile.cname), "sha1")
   var currentHash = footprint(conf, cfile)

compiler/lambdalifting.nim

@@ -235,7 +235,7 @@ template isIterator*(owner: PSym): bool =
 proc liftingHarmful(conf: ConfigRef; owner: PSym): bool {.inline.} =
   ## lambda lifting can be harmful for JS-like code generators.
   let isCompileTime = sfCompileTime in owner.flags or owner.kind == skMacro
-  result = conf.cmd == cmdCompileToJS and not isCompileTime
+  result = conf.backend == backendJs and not isCompileTime
 
 proc createTypeBoundOpsLL(g: ModuleGraph; refType: PType; info: TLineInfo; owner: PSym) =
   createTypeBoundOps(g, nil, refType.lastSon, info)
@@ -846,14 +846,14 @@ proc liftIterToProc*(g: ModuleGraph; fn: PSym; body: PNode; ptrType: PType): PNo
   fn.typ.callConv = oldCC
 
 proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool): PNode =
-  # XXX gCmd == cmdCompileToJS does not suffice! The compiletime stuff needs
+  # XXX backend == backendJs does not suffice! The compiletime stuff needs
   # the transformation even when compiling to JS ...
 
   # However we can do lifting for the stuff which is *only* compiletime.
   let isCompileTime = sfCompileTime in fn.flags or fn.kind == skMacro
 
   if body.kind == nkEmpty or (
-      g.config.cmd == cmdCompileToJS and not isCompileTime) or
+      g.config.backend == backendJs and not isCompileTime) or
       fn.skipGenericOwner.kind != skModule:
 
     # ignore forward declaration:

compiler/main.nim

@@ -119,16 +119,14 @@ when not defined(leanCompiler):
     let conf = graph.config
     conf.exc = excCpp
 
-    if conf.outDir.isEmpty:
-      conf.outDir = conf.projectPath
+    setOutDir(conf)
     if conf.outFile.isEmpty:
       conf.outFile = RelativeFile(conf.projectName & ".js")
 
     #incl(gGlobalOptions, optSafeCode)
     setTarget(graph.config.target, osJS, cpuJS)
     #initDefines()
     defineSymbol(graph.config.symbols, "ecmascript") # For backward compatibility
-    defineSymbol(graph.config.symbols, "js")
     semanticPasses(graph)
     registerPass(graph, JSgenPass)
     compileProject(graph)
@@ -190,44 +188,53 @@ proc mainCommand*(graph: ModuleGraph) =
   conf.lastCmdTime = epochTime()
   conf.searchPaths.add(conf.libpath)
   setId(100)
-  template handleC() =
-    conf.cmd = cmdCompileToC
-    if conf.exc == excNone: conf.exc = excSetjmp
-    defineSymbol(graph.config.symbols, "c")
-    commandCompileToC(graph)
+
+  ## Calling `setOutDir(conf)` unconditionally would fix regression
+  ## https://github.com/nim-lang/Nim/issues/6583#issuecomment-625711125
+  when false: setOutDir(conf)
+  if optUseNimcache in conf.globalOptions: setOutDir(conf)
+
+  template handleBackend(backend2: TBackend) =
+    conf.backend = backend2
+    conf.cmd = cmdCompileToBackend
+    defineSymbol(graph.config.symbols, $backend2)
+    case backend2
+    of backendC:
+      if conf.exc == excNone: conf.exc = excSetjmp
+      commandCompileToC(graph)
+    of backendCpp:
+      if conf.exc == excNone: conf.exc = excCpp
+      commandCompileToC(graph)
+    of backendObjc:
+      commandCompileToC(graph)
+    of backendJs:
+      when defined(leanCompiler):
+        globalError(conf, unknownLineInfo, "compiler wasn't built with JS code generator")
+      else:
+        if conf.hcrOn:
+          # XXX: At the moment, system.nim cannot be compiled in JS mode
+          # with "-d:useNimRtl". The HCR option has been processed earlier
+          # and it has added this define implictly, so we must undo that here.
+          # A better solution might be to fix system.nim
+          undefSymbol(conf.symbols, "useNimRtl")
+        commandCompileToJS(graph)
+    of backendInvalid: doAssert false
+
   case conf.command.normalize
-  of "c", "cc", "compile", "compiletoc": handleC() # compile means compileToC currently
-  of "cpp", "compiletocpp":
-    conf.cmd = cmdCompileToCpp
-    if conf.exc == excNone: conf.exc = excCpp
-    defineSymbol(graph.config.symbols, "cpp")
-    commandCompileToC(graph)
-  of "objc", "compiletooc":
-    conf.cmd = cmdCompileToOC
-    defineSymbol(graph.config.symbols, "objc")
-    commandCompileToC(graph)
+  of "c", "cc", "compile", "compiletoc": handleBackend(backendC) # compile means compileToC currently
+  of "cpp", "compiletocpp": handleBackend(backendCpp)
+  of "objc", "compiletooc": handleBackend(backendObjc)
+  of "js", "compiletojs": handleBackend(backendJs)
   of "r": # different from `"run"`!
     conf.globalOptions.incl {optRun, optUseNimcache}
-    handleC()
+    handleBackend(conf.backend)
   of "run":
     conf.cmd = cmdRun
     when hasTinyCBackend:
       extccomp.setCC(conf, "tcc", unknownLineInfo)
       commandCompileToC(graph)
     else:
       rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
-  of "js", "compiletojs":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with JS code generator"
-    else:
-      conf.cmd = cmdCompileToJS
-      if conf.hcrOn:
-        # XXX: At the moment, system.nim cannot be compiled in JS mode
-        # with "-d:useNimRtl". The HCR option has been processed earlier
-        # and it has added this define implictly, so we must undo that here.
-        # A better solution might be to fix system.nim
-        undefSymbol(conf.symbols, "useNimRtl")
-      commandCompileToJS(graph)
   of "doc0":
     when defined(leanCompiler):
       quit "compiler wasn't built with documentation generator"

compiler/nim.nim

@@ -88,17 +88,19 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
       tccgen.run(conf, conf.arguments)
   if optRun in conf.globalOptions:
     let output = conf.absOutFile
-    let ex = quoteShell output
     case conf.cmd
-    of cmdCompileToJS:
-      execExternalProgram(conf, findNodeJs() & " " & ex & ' ' & conf.arguments)
+    of cmdCompileToBackend:
+      var cmdPrefix = ""
+      case conf.backend
+      of backendC, backendCpp, backendObjc: discard
+      of backendJs: cmdPrefix = findNodeJs() & " "
+      else: doAssert false, $conf.backend
+      execExternalProgram(conf, cmdPrefix & output.quoteShell & ' ' & conf.arguments)
     of cmdDoc, cmdRst2html:
       if conf.arguments.len > 0:
         # reserved for future use
         rawMessage(conf, errGenerated, "'$1 cannot handle arguments" % [$conf.cmd])
       openDefaultBrowser($output)
-    of cmdCompileToC, cmdCompileToCpp, cmdCompileToOC:
-      execExternalProgram(conf, ex & ' ' & conf.arguments)
     else:
       # support as needed
       rawMessage(conf, errGenerated, "'$1 cannot handle --run" % [$conf.cmd])