move tinyc to a separate repo and allow installing external dependencency (eg tinyc) from koch / library code (#13850)

* remove tinyc
* installDeps
* update tinyc paths

Author: timotheecour

.gitignore

@@ -65,7 +65,7 @@ testament.db
 /tests/**/*.json
 /tests/**/*.js
 /csources
-dist/
+/dist/
 
 # Private directories and files (IDEs)
 .*/

compiler/cgen.nim

@@ -1941,8 +1941,8 @@ proc writeModule(m: BModule, pending: bool) =
     var code = genModule(m, cf)
     if code != nil or m.config.symbolFiles != disabledSf:
       when hasTinyCBackend:
-        if conf.cmd == cmdRun:
-          tccgen.compileCCode($code)
+        if m.config.cmd == cmdRun:
+          tccgen.compileCCode($code, m.config)
           onExit()
           return
 

compiler/main.nim

@@ -200,7 +200,7 @@ proc mainCommand*(graph: ModuleGraph) =
   of "run":
     conf.cmd = cmdRun
     when hasTinyCBackend:
-      extccomp.setCC("tcc")
+      extccomp.setCC(conf, "tcc", unknownLineInfo)
       commandCompileToC(graph)
     else:
       rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")

compiler/nim.nim

@@ -98,7 +98,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
   if conf.errorCounter != 0: return
   when hasTinyCBackend:
     if conf.cmd == cmdRun:
-      tccgen.run(conf.arguments)
+      tccgen.run(conf, conf.arguments)
   if optRun in conf.globalOptions:
     var ex = quoteShell conf.absOutFile
     if conf.cmd == cmdCompileToJS:

compiler/tccgen.nim

@@ -8,12 +8,20 @@
 #
 
 import
-  os, strutils, options, msgs, tinyc
+  os, strutils, options, msgs, tinyc, lineinfos, sequtils
 
-{.compile: "../tinyc/libtcc.c".}
+const tinyPrefix = "dist/nim-tinyc-archive".unixToNativePath
+const nimRoot = currentSourcePath.parentDir.parentDir
+const tinycRoot = nimRoot / tinyPrefix
+when not dirExists(tinycRoot):
+  static: doAssert false, $(tinycRoot, "requires: ./koch installdeps tinyc")
+{.compile: tinycRoot / "tinyc/libtcc.c".}
+
+var
+  gConf: ConfigRef # ugly but can be cleaned up if this is revived
 
 proc tinyCErrorHandler(closure: pointer, msg: cstring) {.cdecl.} =
-  rawMessage(errGenerated, $msg)
+  rawMessage(gConf, errGenerated, $msg)
 
 proc initTinyCState: PccState =
   result = openCCState()
@@ -25,7 +33,7 @@ var
 
 proc addFile(filename: string) =
   if addFile(gTinyC, filename) != 0'i32:
-    rawMessage(errCannotOpenFile, filename)
+    rawMessage(gConf, errCannotOpenFile, filename)
 
 proc setupEnvironment =
   when defined(amd64):
@@ -35,42 +43,47 @@ proc setupEnvironment =
   when defined(linux):
     defineSymbol(gTinyC, "__linux__", nil)
     defineSymbol(gTinyC, "__linux", nil)
-  var nimDir = getPrefixDir()
+
+  var nimDir = getPrefixDir(gConf).string
+  var tinycRoot = nimDir / tinyPrefix
+  let libpath = nimDir / "lib"
 
   addIncludePath(gTinyC, libpath)
   when defined(windows):
-    addSysincludePath(gTinyC, nimDir / "tinyc/win32/include")
-  addSysincludePath(gTinyC, nimDir / "tinyc/include")
+    addSysincludePath(gTinyC, tinycRoot / "tinyc/win32/include")
+  addSysincludePath(gTinyC, tinycRoot / "tinyc/include")
   when defined(windows):
     defineSymbol(gTinyC, "_WIN32", nil)
     # we need Mingw's headers too:
-    var gccbin = getConfigVar("gcc.path") % ["nim", nimDir]
+    var gccbin = getConfigVar("gcc.path") % ["nim", tinycRoot]
     addSysincludePath(gTinyC, gccbin /../ "include")
-    #addFile(nimDir / r"tinyc\win32\wincrt1.o")
-    addFile(nimDir / r"tinyc\win32\alloca86.o")
-    addFile(nimDir / r"tinyc\win32\chkstk.o")
-    #addFile(nimDir / r"tinyc\win32\crt1.o")
+    #addFile(tinycRoot / r"tinyc\win32\wincrt1.o")
+    addFile(tinycRoot / r"tinyc\win32\alloca86.o")
+    addFile(tinycRoot / r"tinyc\win32\chkstk.o")
+    #addFile(tinycRoot / r"tinyc\win32\crt1.o")
 
-    #addFile(nimDir / r"tinyc\win32\dllcrt1.o")
-    #addFile(nimDir / r"tinyc\win32\dllmain.o")
-    addFile(nimDir / r"tinyc\win32\libtcc1.o")
+    #addFile(tinycRoot / r"tinyc\win32\dllcrt1.o")
+    #addFile(tinycRoot / r"tinyc\win32\dllmain.o")
+    addFile(tinycRoot / r"tinyc\win32\libtcc1.o")
 
-    #addFile(nimDir / r"tinyc\win32\lib\crt1.c")
-    #addFile(nimDir / r"tinyc\lib\libtcc1.c")
+    #addFile(tinycRoot / r"tinyc\win32\lib\crt1.c")
+    #addFile(tinycRoot / r"tinyc\lib\libtcc1.c")
   else:
     addSysincludePath(gTinyC, "/usr/include")
     when defined(amd64):
       addSysincludePath(gTinyC, "/usr/include/x86_64-linux-gnu")
 
-proc compileCCode*(ccode: string) =
+proc compileCCode*(ccode: string, conf: ConfigRef) =
+  gConf = conf
   if not libIncluded:
     libIncluded = true
     setupEnvironment()
   discard compileString(gTinyC, ccode)
 
-proc run*(args: string) =
-  var s = @[cstring(gProjectName)] & map(split(args), proc(x: string): cstring = cstring(x))
+proc run*(conf: ConfigRef, args: string) =
+  doAssert gConf == conf
+  var s = @[cstring(conf.projectName)] & map(split(args), proc(x: string): cstring = cstring(x))
   var err = tinyc.run(gTinyC, cint(s.len), cast[cstringArray](addr(s[0]))) != 0'i32
   closeCCState(gTinyC)
-  if err: rawMessage(errExecutionOfProgramFailed, "")
+  if err: rawMessage(conf, errUnknown, "")
 

koch.nim

@@ -31,6 +31,7 @@ import
   os, strutils, parseopt, osproc
 
 import tools / kochdocs
+import tools / deps
 
 const VersionAsString = system.NimVersion
 
@@ -73,6 +74,7 @@ Commands for core developers:
   zip                      builds the installation zip package
   xz                       builds the installation tar.xz package
   testinstall              test tar.xz package; Unix only!
+  installdeps [options]    installs external dependency (eg tinyc) to dist/
   tests [options]          run the testsuite (run a subset of tests by
                            specifying a category, e.g. `tests cat async`)
   temp options             creates a temporary compiler for testing
@@ -120,25 +122,21 @@ proc copyExe(source, dest: string) =
 
 const
   compileNimInst = "tools/niminst/niminst"
+  distDir = "dist"
 
 proc csource(args: string) =
   nimexec(("cc $1 -r $3 --var:version=$2 --var:mingw=none csource " &
            "--main:compiler/nim.nim compiler/installer.ini $1") %
        [args, VersionAsString, compileNimInst])
 
 proc bundleC2nim(args: string) =
-  if not dirExists("dist/c2nim/.git"):
-    exec("git clone -q https://github.com/nim-lang/c2nim.git dist/c2nim")
+  cloneDependency(distDir, "https://github.com/nim-lang/c2nim.git")
   nimCompile("dist/c2nim/c2nim",
              options = "--noNimblePath --path:. " & args)
 
 proc bundleNimbleExe(latest: bool, args: string) =
-  if not dirExists("dist/nimble/.git"):
-    exec("git clone -q https://github.com/nim-lang/nimble.git dist/nimble")
-  if not latest:
-    withDir("dist/nimble"):
-      exec("git fetch")
-      exec("git checkout " & NimbleStableCommit)
+  let commit = if latest: "HEAD" else: NimbleStableCommit
+  cloneDependency(distDir, "https://github.com/nim-lang/nimble.git", commit = commit)
   # installer.ini expects it under $nim/bin
   nimCompile("dist/nimble/src/nimble.nim",
              options = "-d:release --nilseqs:on " & args)
@@ -156,6 +154,7 @@ proc buildNimble(latest: bool, args: string) =
         while dirExists("dist/nimble" & $id):
           inc id
         installDir = "dist/nimble" & $id
+      # consider using/adapting cloneDependency
       exec("git clone -q https://github.com/nim-lang/nimble.git " & installDir)
     withDir(installDir):
       if latest:
@@ -504,6 +503,17 @@ proc hostInfo(): string =
   "hostOS: $1, hostCPU: $2, int: $3, float: $4, cpuEndian: $5, cwd: $6" %
     [hostOS, hostCPU, $int.sizeof, $float.sizeof, $cpuEndian, getCurrentDir()]
 
+proc installDeps(dep: string, commit = "") =
+  # the hashes/urls are version controlled here, so can be changed seamlessly
+  # and tied to a nim release (mimicking git submodules)
+  var commit = commit
+  case dep
+  of "tinyc":
+    if commit.len == 0: commit = "916cc2f94818a8a382dd8d4b8420978816c1dfb3"
+    cloneDependency(distDir, "https://github.com/timotheecour/nim-tinyc-archive", commit)
+  else: doAssert false, "unsupported: " & dep
+  # xxx: also add linenoise, niminst etc, refs https://github.com/nim-lang/RFCs/issues/206
+
 proc runCI(cmd: string) =
   doAssert cmd.len == 0, cmd # avoid silently ignoring
   echo "runCI: ", cmd
@@ -648,6 +658,7 @@ when isMainModule:
       of "distrohelper": geninstall()
       of "install": install(op.cmdLineRest)
       of "testinstall": testUnixInstall(op.cmdLineRest)
+      of "installdeps": installDeps(op.cmdLineRest)
       of "runci": runCI(op.cmdLineRest)
       of "test", "tests": tests(op.cmdLineRest)
       of "temp": temp(op.cmdLineRest)