IC: backend: remember produced type information (#17440)

Author: Araq

compiler/ast.nim

@@ -1229,17 +1229,10 @@ proc newSym*(symKind: TSymKind, name: PIdent, id: ItemId, owner: PSym,
   result = PSym(name: name, kind: symKind, flags: {}, info: info, itemId: id,
                 options: options, owner: owner, offset: defaultOffset)
   when false:
-    if id.item > 2141:
-      let s = getStackTrace()
-      const words = ["createTypeBoundOps",
-        "initOperators",
-        "generateInstance",
-        "semIdentDef", "addLocalDecl"]
-      for w in words:
-        if w in s:
-          x.inc w
-          return
-      x.inc "<no category>"
+    if id.module == 48 and id.item == 39:
+      writeStackTrace()
+      echo "kind ", symKind, " ", name.s
+      if owner != nil: echo owner.name.s
 
 proc astdef*(s: PSym): PNode =
   # get only the definition (initializer) portion of the ast

compiler/ccgtypes.nim

@@ -1386,7 +1386,6 @@ proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope =
   let owner = t.skipTypes(typedescPtrs).itemId.module
   if owner != m.module.position and moduleOpenForCodegen(m, owner):
     # make sure the type info is created in the owner module
-    assert m.g.modules[owner] != nil
     discard genTypeInfoV2(m.g.modules[owner], origType, info)
     # reference the type info as extern here
     discard cgsym(m, "TNimTypeV2")
@@ -1456,18 +1455,27 @@ proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope =
   result = "NTI$1$2_" % [rope(typeToC(t)), rope($sig)]
   m.typeInfoMarker[sig] = result
 
-  let owner = t.skipTypes(typedescPtrs).itemId.module
+  let old = m.g.graph.emittedTypeInfo.getOrDefault($result)
+  if old != FileIndex(0):
+    discard cgsym(m, "TNimType")
+    discard cgsym(m, "TNimNode")
+    declareNimType(m, "TNimType", result, old.int)
+    return prefixTI.rope & result & ")".rope
+
+  var owner = t.skipTypes(typedescPtrs).itemId.module
   if owner != m.module.position and moduleOpenForCodegen(m, owner):
     # make sure the type info is created in the owner module
-    assert m.g.modules[owner] != nil
     discard genTypeInfoV1(m.g.modules[owner], origType, info)
     # reference the type info as extern here
     discard cgsym(m, "TNimType")
     discard cgsym(m, "TNimNode")
     declareNimType(m, "TNimType", result, owner)
     return prefixTI.rope & result & ")".rope
+  else:
+    owner = m.module.position.int32
 
   m.g.typeInfoMarker[sig] = (str: result, owner: owner)
+  rememberEmittedTypeInfo(m.g.graph, FileIndex(owner), $result)
 
   case t.kind
   of tyEmpty, tyVoid: result = rope"0"

compiler/ic/cbackend.nim

@@ -18,7 +18,7 @@
 ## also doing cross-module dependency tracking and DCE that we don't need
 ## anymore. DCE is now done as prepass over the entire packed module graph.
 
-import std/[packedsets, algorithm]
+import std/[packedsets, algorithm, tables]
   # std/intsets would give `UnusedImport`, pending https://github.com/nim-lang/Nim/issues/14246
 import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen,
   pathutils, extccomp, msgs]
@@ -45,6 +45,10 @@ proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var Alive
 
   finalCodegenActions(g, bmod, newNodeI(nkStmtList, m.module.info))
 
+proc replayTypeInfo(g: ModuleGraph; m: var LoadedModule; origin: FileIndex) =
+  for x in mitems(m.fromDisk.emittedTypeInfo):
+    g.emittedTypeInfo[x] = origin
+
 proc addFileToLink(config: ConfigRef; m: PSym) =
   let filename = AbsoluteFile toFullPath(config, m.position.FileIndex)
   let ext =
@@ -59,11 +63,28 @@ proc addFileToLink(config: ConfigRef; m: PSym) =
                    flags: {CfileFlag.Cached})
     addFileToCompile(config, cf)
 
-proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool =
+when defined(debugDce):
+  import std / [os, packedsets]
+
+proc storeAliveSymsImpl(asymFile: AbsoluteFile; s: seq[int32]) =
+  var f = rodfiles.create(asymFile.string)
+  f.storeHeader()
+  f.storeSection aliveSymsSection
+  f.storeSeq(s)
+  close f
+
+template prepare {.dirty.} =
   let asymFile = toRodFile(config, AbsoluteFile toFullPath(config, position.FileIndex), ".alivesyms")
   var s = newSeqOfCap[int32](alive[position].len)
   for a in items(alive[position]): s.add int32(a)
   sort(s)
+
+proc storeAliveSyms(config: ConfigRef; position: int; alive: AliveSyms) =
+  prepare()
+  storeAliveSymsImpl(asymFile, s)
+
+proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool =
+  prepare()
   var f2 = rodfiles.open(asymFile.string)
   f2.loadHeader()
   f2.loadSection aliveSymsSection
@@ -73,12 +94,17 @@ proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool
   if f2.err == ok and oldData == s:
     result = false
   else:
+    when defined(debugDce):
+      let oldAsSet = toPackedSet[int32](oldData)
+      let newAsSet = toPackedSet[int32](s)
+      echo "set of live symbols changed ", asymFile.changeFileExt("rod"), " ", position, " ", f2.err
+      echo "in old but not in new ", oldAsSet.difference(newAsSet)
+      echo "in new but not in old ", newAsSet.difference(oldAsSet)
+
+      if execShellCmd(getAppFilename() & " rod " & quoteShell(asymFile.changeFileExt("rod"))) != 0:
+        echo "command failed"
     result = true
-    var f = rodfiles.create(asymFile.string)
-    f.storeHeader()
-    f.storeSection aliveSymsSection
-    f.storeSeq(s)
-    close f
+    storeAliveSymsImpl(asymFile, s)
 
 proc generateCode*(g: ModuleGraph) =
   ## The single entry point, generate C(++) code for the entire
@@ -95,6 +121,8 @@ proc generateCode*(g: ModuleGraph) =
       assert false
     of storing, outdated:
       generateCodeForModule(g, g.packed[i], alive)
+      closeRodFile(g, g.packed[i].module)
+      storeAliveSyms(g.config, g.packed[i].module.position, alive)
     of loaded:
       # Even though this module didn't change, DCE might trigger a change.
       # Consider this case: Module A uses symbol S from B and B does not use
@@ -104,3 +132,4 @@ proc generateCode*(g: ModuleGraph) =
         generateCodeForModule(g, g.packed[i], alive)
       else:
         addFileToLink(g.config, g.packed[i].module)
+        replayTypeInfo(g, g.packed[i], FileIndex(i))

compiler/ic/ic.nim

@@ -42,6 +42,8 @@ type
     methodsPerType*: seq[(PackedItemId, int, PackedItemId)]
     enumToStringProcs*: seq[(PackedItemId, PackedItemId)]
 
+    emittedTypeInfo*: seq[string]
+
     sh*: Shared
     cfg: PackedConfig
 
@@ -58,7 +60,7 @@ type
     config*: ConfigRef
 
 proc isActive*(e: PackedEncoder): bool = e.config != nil
-proc disable*(e: var PackedEncoder) = e.config = nil
+proc disable(e: var PackedEncoder) = e.config = nil
 
 template primConfigFields(fn: untyped) {.dirty.} =
   fn backend
@@ -552,6 +554,7 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
   loadSeqSection attachedOpsSection, m.attachedOps
   loadSeqSection methodsPerTypeSection, m.methodsPerType
   loadSeqSection enumToStringProcsSection, m.enumToStringProcs
+  loadSeqSection typeInfoSection, m.emittedTypeInfo
 
   close(f)
   result = f.err
@@ -614,6 +617,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
   storeSeqSection attachedOpsSection, m.attachedOps
   storeSeqSection methodsPerTypeSection, m.methodsPerType
   storeSeqSection enumToStringProcsSection, m.enumToStringProcs
+  storeSeqSection typeInfoSection, m.emittedTypeInfo
 
   close(f)
   encoder.disable()
@@ -1139,7 +1143,10 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
 
   echo "all symbols"
   for i in 0..high(m.sh.syms):
-    echo "  ", m.sh.strings[m.sh.syms[i].name], " local ID: ", i
+    if m.sh.syms[i].name != LitId(0):
+      echo "  ", m.sh.strings[m.sh.syms[i].name], " local ID: ", i, " kind ", m.sh.syms[i].kind
+    else:
+      echo "  <anon symbol?> local ID: ", i, " kind ", m.sh.syms[i].kind
 
   echo "symbols: ", m.sh.syms.len, " types: ", m.sh.types.len,
     " top level nodes: ", m.topLevel.nodes.len, " other nodes: ", m.bodies.nodes.len,

compiler/ic/rodfiles.nim

@@ -36,6 +36,7 @@ type
     attachedOpsSection
     methodsPerTypeSection
     enumToStringProcsSection
+    typeInfoSection  # required by the backend
     aliveSymsSection # beware, this is stored in a `.alivesyms` file.
 
   RodFileError* = enum

compiler/modulegraphs.nim

@@ -11,9 +11,8 @@
 ## represents a complete Nim project. Single modules can either be kept in RAM
 ## or stored in a rod-file.
 
-import ast, astalgo, intsets, tables, options, lineinfos, hashes, idents,
-  btrees, md5, ropes, msgs
-
+import std / [intsets, tables, hashes, md5]
+import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils
 import ic / [packed_ast, ic]
 
 type
@@ -60,6 +59,7 @@ type
     attachedOps*: array[TTypeAttachedOp, Table[ItemId, PSym]] # Type ID, destructors, etc.
     methodsPerType*: Table[ItemId, seq[(int, LazySym)]] # Type ID, attached methods
     enumToStringProcs*: Table[ItemId, LazySym]
+    emittedTypeInfo*: Table[string, FileIndex]
 
     startupPackedConfig*: PackedConfig
     packageSyms*: TStrTable
@@ -68,7 +68,6 @@ type
     importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies
     suggestMode*: bool # whether we are in nimsuggest mode or not.
     invalidTransitiveClosure: bool
-    systemModuleComplete*: bool
     inclToMod*: Table[FileIndex, FileIndex] # mapping of include file to the
                                             # first module that included it
     importStack*: seq[FileIndex]  # The current import stack. Used for detecting recursive
@@ -435,6 +434,7 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result.canonTypes = initTable[SigHash, PType]()
   result.symBodyHashes = initTable[int, SigHash]()
   result.operators = initOperators(result)
+  result.emittedTypeInfo = initTable[string, FileIndex]()
 
 proc resetAllModules*(g: ModuleGraph) =
   initStrTable(g.packageSyms)
@@ -455,6 +455,27 @@ proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym =
     elif fileIdx.int32 < g.ifaces.len:
       result = g.ifaces[fileIdx.int32].module
 
+proc rememberEmittedTypeInfo*(g: ModuleGraph; m: FileIndex; ti: string) =
+  #assert(not isCachedModule(g, m.int32))
+  if g.config.symbolFiles != disabledSf:
+    #assert g.encoders[m.int32].isActive
+    g.packed[m.int32].fromDisk.emittedTypeInfo.add ti
+
+proc closeRodFile*(g: ModuleGraph; m: PSym) =
+  if g.config.symbolFiles in {readOnlySf, v2Sf}:
+    # For stress testing we seek to reload the symbols from memory. This
+    # way much of the logic is tested but the test is reproducible as it does
+    # not depend on the hard disk contents!
+    let mint = m.position
+    saveRodFile(toRodFile(g.config, AbsoluteFile toFullPath(g.config, FileIndex(mint))),
+                g.encoders[mint], g.packed[mint].fromDisk)
+  elif g.config.symbolFiles == stressTest:
+    # debug code, but maybe a good idea for production? Could reduce the compiler's
+    # memory consumption considerably at the cost of more loads from disk.
+    let mint = m.position
+    simulateCachedModule(g, m, g.packed[mint].fromDisk)
+    g.packed[mint].status = loaded
+
 proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b
 
 proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) =

compiler/passes.nim

@@ -187,4 +187,9 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
     closeParser(p)
     if s.kind != llsStdIn: break
   closePasses(graph, a)
+  if graph.config.backend notin {backendC, backendCpp, backendObjc}:
+    # We only write rod files here if no C-like backend is active.
+    # The C-like backends have been patched to support the IC mechanism.
+    # They are responsible for closing the rod files. See `cbackend.nim`.
+    closeRodFile(graph, module)
   result = true

compiler/sem.nim

@@ -661,7 +661,7 @@ proc myClose(graph: ModuleGraph; context: PPassContext, n: PNode): PNode =
     result.add(c.module.ast)
   popOwner(c)
   popProcCon(c)
-  saveRodFile(c)
+  sealRodFile(c)
 
 const semPass* = makePass(myOpen, myProcess, myClose,
                           isFrontend = true)

compiler/semdata.nim

@@ -560,23 +560,10 @@ proc addToGenericCache*(c: PContext; s: PSym; inst: PType) =
   if c.config.symbolFiles != disabledSf:
     storeTypeInst(c.encoder, c.packedRepr, s, inst)
 
-proc saveRodFile*(c: PContext) =
+proc sealRodFile*(c: PContext) =
   if c.config.symbolFiles != disabledSf:
     if c.graph.vm != nil:
       for (m, n) in PCtx(c.graph.vm).vmstateDiff:
         if m == c.module:
           addPragmaComputation(c, n)
-    if sfSystemModule in c.module.flags:
-      c.graph.systemModuleComplete = true
     c.idgen.sealed = true # no further additions are allowed
-    if c.config.symbolFiles != stressTest:
-      # For stress testing we seek to reload the symbols from memory. This
-      # way much of the logic is tested but the test is reproducible as it does
-      # not depend on the hard disk contents!
-      saveRodFile(toRodFile(c.config, AbsoluteFile toFullPath(c.config, FileIndex c.module.position)),
-                  c.encoder, c.packedRepr)
-    else:
-      # debug code, but maybe a good idea for production? Could reduce the compiler's
-      # memory consumption considerably at the cost of more loads from disk.
-      simulateCachedModule(c.graph, c.module, c.packedRepr)
-      c.graph.packed[c.module.position].status = loaded