Reduce the memory footprint of dsymutil. (NFC)

This (partially) fixes a regression introduced by
https://reviews.llvm.org/D43945 / r327399, which parallelized
DwarfLinker. This patch avoids parsing and allocating the memory for
all input DIEs up front and instead only allocates them in the
concurrent loop in the AnalyzeLambda. At the end of the loop the
memory from the LinkContext is cleared again.

This reduces the peak memory needed to link the debug info of a
non-modular build of the Swift compiler by >3GB.

rdar://problem/43444464

Differential Revision: https://reviews.llvm.org/D51078

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@340650 91177308-0d34-0410-b5e6-96231b3b80d8

Author: adrian-prantl

tools/dsymutil/DwarfLinker.cpp

@@ -1570,6 +1570,7 @@ class DwarfLinker {
 
     /// Clear compile units and ranges.
     void Clear() {
+      DwarfContext.reset(nullptr);
       CompileUnits.clear();
       Ranges.clear();
     }
@@ -1605,7 +1606,7 @@ class DwarfLinker {
                                OffsetsStringPool &OffsetsStringPool,
                                UniquingStringPool &UniquingStringPoolStringPool,
                                DeclContextTree &ODRContexts, unsigned &UnitID,
-                               unsigned Indent = 0);
+                               unsigned Indent = 0, bool Quiet = false);
 
   /// Recursively add the debug info in this clang module .pcm
   /// file (and all the modules imported by it in a bottom-up fashion)
@@ -1616,7 +1617,7 @@ class DwarfLinker {
                         RangesTy &Ranges, OffsetsStringPool &OffsetsStringPool,
                         UniquingStringPool &UniquingStringPool,
                         DeclContextTree &ODRContexts, unsigned &UnitID,
-                        unsigned Indent = 0);
+                        unsigned Indent = 0, bool Quiet = false);
 
   /// Flags passed to DwarfLinker::lookForDIEsToKeep
   enum TravesalFlags {
@@ -3900,7 +3901,7 @@ bool DwarfLinker::registerModuleReference(
     const DWARFDie &CUDie, const DWARFUnit &Unit, DebugMap &ModuleMap,
     const DebugMapObject &DMO, RangesTy &Ranges, OffsetsStringPool &StringPool,
     UniquingStringPool &UniquingStringPool, DeclContextTree &ODRContexts,
-    unsigned &UnitID, unsigned Indent) {
+    unsigned &UnitID, unsigned Indent, bool Quiet) {
   std::string PCMfile = dwarf::toString(
       CUDie.find({dwarf::DW_AT_dwo_name, dwarf::DW_AT_GNU_dwo_name}), "");
   if (PCMfile.empty())
@@ -3912,11 +3913,12 @@ bool DwarfLinker::registerModuleReference(
 
   std::string Name = dwarf::toString(CUDie.find(dwarf::DW_AT_name), "");
   if (Name.empty()) {
-    reportWarning("Anonymous module skeleton CU for " + PCMfile, DMO);
+    if (!Quiet)
+      reportWarning("Anonymous module skeleton CU for " + PCMfile, DMO);
     return true;
   }
 
-  if (Options.Verbose) {
+  if (!Quiet && Options.Verbose) {
     outs().indent(Indent);
     outs() << "Found clang module reference " << PCMfile;
   }
@@ -3926,24 +3928,24 @@ bool DwarfLinker::registerModuleReference(
     // FIXME: Until PR27449 (https://llvm.org/bugs/show_bug.cgi?id=27449) is
     // fixed in clang, only warn about DWO_id mismatches in verbose mode.
     // ASTFileSignatures will change randomly when a module is rebuilt.
-    if (Options.Verbose && (Cached->second != DwoId))
+    if (!Quiet && Options.Verbose && (Cached->second != DwoId))
       reportWarning(Twine("hash mismatch: this object file was built against a "
                           "different version of the module ") +
                         PCMfile,
                     DMO);
-    if (Options.Verbose)
+    if (!Quiet && Options.Verbose)
       outs() << " [cached].\n";
     return true;
   }
-  if (Options.Verbose)
+  if (!Quiet && Options.Verbose)
     outs() << " ...\n";
 
   // Cyclic dependencies are disallowed by Clang, but we still
   // shouldn't run into an infinite loop, so mark it as processed now.
   ClangModules.insert({PCMfile, DwoId});
   if (Error E = loadClangModule(PCMfile, PCMpath, Name, DwoId, ModuleMap, DMO,
                                 Ranges, StringPool, UniquingStringPool,
-                                ODRContexts, UnitID, Indent + 2)) {
+                                ODRContexts, UnitID, Indent + 2, Quiet)) {
     consumeError(std::move(E));
     return false;
   }
@@ -3965,14 +3967,12 @@ DwarfLinker::loadObject(BinaryHolder &BinaryHolder, const DebugMapObject &Obj,
   return ErrOrObj;
 }
 
-Error DwarfLinker::loadClangModule(StringRef Filename, StringRef ModulePath,
-                                   StringRef ModuleName, uint64_t DwoId,
-                                   DebugMap &ModuleMap,
-                                   const DebugMapObject &DMO, RangesTy &Ranges,
-                                   OffsetsStringPool &StringPool,
-                                   UniquingStringPool &UniquingStringPool,
-                                   DeclContextTree &ODRContexts,
-                                   unsigned &UnitID, unsigned Indent) {
+Error DwarfLinker::loadClangModule(
+    StringRef Filename, StringRef ModulePath, StringRef ModuleName,
+    uint64_t DwoId, DebugMap &ModuleMap, const DebugMapObject &DMO,
+    RangesTy &Ranges, OffsetsStringPool &StringPool,
+    UniquingStringPool &UniquingStringPool, DeclContextTree &ODRContexts,
+    unsigned &UnitID, unsigned Indent, bool Quiet) {
   SmallString<80> Path(Options.PrependPath);
   if (sys::path::is_relative(Filename))
     sys::path::append(Path, ModulePath, Filename);
@@ -4030,7 +4030,7 @@ Error DwarfLinker::loadClangModule(StringRef Filename, StringRef ModulePath,
     auto CUDie = CU->getUnitDIE(false);
     if (!registerModuleReference(CUDie, *CU, ModuleMap, DMO, Ranges, StringPool,
                                  UniquingStringPool, ODRContexts, UnitID,
-                                 Indent)) {
+                                 Indent, Quiet)) {
       if (Unit) {
         std::string Err =
             (Filename +
@@ -4044,7 +4044,7 @@ Error DwarfLinker::loadClangModule(StringRef Filename, StringRef ModulePath,
       // ASTFileSignatures will change randomly when a module is rebuilt.
       uint64_t PCMDwoId = getDwoId(CUDie, *CU);
       if (PCMDwoId != DwoId) {
-        if (Options.Verbose)
+        if (!Quiet && Options.Verbose)
           reportWarning(
               Twine("hash mismatch: this object file was built against a "
                     "different version of the module ") +
@@ -4066,7 +4066,7 @@ Error DwarfLinker::loadClangModule(StringRef Filename, StringRef ModulePath,
   }
   if (!Unit->getOrigUnit().getUnitDIE().hasChildren())
     return Error::success();
-  if (Options.Verbose) {
+  if (!Quiet && Options.Verbose) {
     outs().indent(Indent);
     outs() << "cloning .debug_info from " << Filename << "\n";
   }
@@ -4289,12 +4289,10 @@ bool DwarfLinker::link(const DebugMap &Map) {
         CUDie.dump(outs(), 0, DumpOpts);
       }
 
-      if (!CUDie || LLVM_UNLIKELY(Options.Update) ||
-          !registerModuleReference(CUDie, *CU, ModuleMap, LinkContext.DMO,
-                                   LinkContext.Ranges, OffsetsStringPool,
-                                   UniquingStringPool, ODRContexts, UnitID)) {
-        LinkContext.CompileUnits.push_back(llvm::make_unique<CompileUnit>(
-            *CU, UnitID++, !Options.NoODR && !Options.Update, ""));
+      if (CUDie && !LLVM_UNLIKELY(Options.Update)) {
+	registerModuleReference(CUDie, *CU, ModuleMap, LinkContext.DMO,
+				LinkContext.Ranges, OffsetsStringPool,
+				UniquingStringPool, ODRContexts, UnitID);
         maybeUpdateMaxDwarfVersion(CU->getVersion());
       }
     }
@@ -4316,13 +4314,31 @@ bool DwarfLinker::link(const DebugMap &Map) {
     for (unsigned i = 0, e = NumObjects; i != e; ++i) {
       auto &LinkContext = ObjectContexts[i];
 
-      if (!LinkContext.ObjectFile) {
+      if (!LinkContext.ObjectFile || !LinkContext.DwarfContext) {
         std::unique_lock<std::mutex> LockGuard(ProcessedFilesMutex);
         ProcessedFiles.set(i);
         ProcessedFilesConditionVariable.notify_one();
         continue;
       }
 
+      for (const auto &CU : LinkContext.DwarfContext->compile_units()) {
+        // The !registerModuleReference() condition effectively skips
+        // over fully resolved skeleton units. This second pass of
+        // registerModuleReferences doesn't do any new work, but it
+        // will collect top-level errors, which are suppressed. Module
+        // warnings were already displayed in the first iteration.
+        bool Quiet = true;
+        auto CUDie = CU->getUnitDIE(false);
+        if (!CUDie || LLVM_UNLIKELY(Options.Update) ||
+            !registerModuleReference(CUDie, *CU, ModuleMap, LinkContext.DMO,
+                                     LinkContext.Ranges, OffsetsStringPool,
+                                     UniquingStringPool, ODRContexts, UnitID,
+                                     Quiet)) {
+          LinkContext.CompileUnits.push_back(llvm::make_unique<CompileUnit>(
+              *CU, UnitID++, !Options.NoODR && !Options.Update, ""));
+        }
+      }
+      
       // Now build the DIE parent links that we will use during the next phase.
       for (auto &CurrentUnit : LinkContext.CompileUnits)
         analyzeContextInfo(CurrentUnit->getOrigUnit().getUnitDIE(), 0,