Auto merge of #42332 - michaelwoerister:no-more-retracing, r=nikomatsakis

incr.comp.: Use DefPathHash-based DepNodes in the serialized DepGraph and remove obsolete DefIdDirectory

With this PR we don't store the dep-graph as a set of `DepNode<IndexIntoDefIdDirectory>` anymore but instead as a set of `DepNode<DefPathHash>`. Since a `DefPathHash` is a global identifier that is valid across compilation sessions, we don't need the `DefIdDirectory` anymore.

Since a `DepNode<DefPathHash>` is bigger than a `DepNode<IndexIntoDefIdDirectory>` and our on-disk encoding of the dep-graph is inefficient, this PR will probably increase the amount of space the dep-graph takes up on disk. I'm in the process of gathering some performance data.

The changes in here are a step towards implementing ICH-based `DepNodes` (#42294).

r? @nikomatsakis

Author: bors

src/librustc/hir/map/definitions.rs

@@ -36,7 +36,7 @@ use util::nodemap::NodeMap;
 pub struct DefPathTable {
     index_to_key: [Vec<DefKey>; 2],
     key_to_index: FxHashMap<DefKey, DefIndex>,
-    def_path_hashes: [Vec<Fingerprint>; 2],
+    def_path_hashes: [Vec<DefPathHash>; 2],
 }
 
 // Unfortunately we have to provide a manual impl of Clone because of the
@@ -57,7 +57,7 @@ impl DefPathTable {
 
     fn allocate(&mut self,
                 key: DefKey,
-                def_path_hash: Fingerprint,
+                def_path_hash: DefPathHash,
                 address_space: DefIndexAddressSpace)
                 -> DefIndex {
         let index = {
@@ -81,7 +81,7 @@ impl DefPathTable {
     }
 
     #[inline(always)]
-    pub fn def_path_hash(&self, index: DefIndex) -> Fingerprint {
+    pub fn def_path_hash(&self, index: DefIndex) -> DefPathHash {
         self.def_path_hashes[index.address_space().index()]
                             [index.as_array_index()]
     }
@@ -126,6 +126,30 @@ impl DefPathTable {
 
         Some(index)
     }
+
+    pub fn add_def_path_hashes_to(&self,
+                                  cnum: CrateNum,
+                                  out: &mut FxHashMap<DefPathHash, DefId>) {
+        for address_space in &[DefIndexAddressSpace::Low, DefIndexAddressSpace::High] {
+            let start_index = address_space.start();
+            out.extend(
+                (&self.def_path_hashes[address_space.index()])
+                    .iter()
+                    .enumerate()
+                    .map(|(index, &hash)| {
+                        let def_id = DefId {
+                            krate: cnum,
+                            index: DefIndex::new(index + start_index),
+                        };
+                        (hash, def_id)
+                    })
+            );
+        }
+    }
+
+    pub fn size(&self) -> usize {
+        self.key_to_index.len()
+    }
 }
 
 
@@ -148,8 +172,8 @@ impl Decodable for DefPathTable {
         let index_to_key_lo: Vec<DefKey> = Decodable::decode(d)?;
         let index_to_key_hi: Vec<DefKey> = Decodable::decode(d)?;
 
-        let def_path_hashes_lo: Vec<Fingerprint> = Decodable::decode(d)?;
-        let def_path_hashes_hi: Vec<Fingerprint> = Decodable::decode(d)?;
+        let def_path_hashes_lo: Vec<DefPathHash> = Decodable::decode(d)?;
+        let def_path_hashes_hi: Vec<DefPathHash> = Decodable::decode(d)?;
 
         let index_to_key = [index_to_key_lo, index_to_key_hi];
         let def_path_hashes = [def_path_hashes_lo, def_path_hashes_hi];
@@ -216,25 +240,25 @@ pub struct DefKey {
 }
 
 impl DefKey {
-    fn compute_stable_hash(&self, parent_hash: Fingerprint) -> Fingerprint {
+    fn compute_stable_hash(&self, parent_hash: DefPathHash) -> DefPathHash {
         let mut hasher = StableHasher::new();
 
         // We hash a 0u8 here to disambiguate between regular DefPath hashes,
         // and the special "root_parent" below.
         0u8.hash(&mut hasher);
         parent_hash.hash(&mut hasher);
         self.disambiguated_data.hash(&mut hasher);
-        hasher.finish()
+        DefPathHash(hasher.finish())
     }
 
-    fn root_parent_stable_hash(crate_name: &str, crate_disambiguator: &str) -> Fingerprint {
+    fn root_parent_stable_hash(crate_name: &str, crate_disambiguator: &str) -> DefPathHash {
         let mut hasher = StableHasher::new();
         // Disambiguate this from a regular DefPath hash,
         // see compute_stable_hash() above.
         1u8.hash(&mut hasher);
         crate_name.hash(&mut hasher);
         crate_disambiguator.hash(&mut hasher);
-        hasher.finish()
+        DefPathHash(hasher.finish())
     }
 }
 
@@ -296,7 +320,9 @@ impl DefPath {
 
         s.push_str(&tcx.original_crate_name(self.krate).as_str());
         s.push_str("/");
-        s.push_str(&tcx.crate_disambiguator(self.krate).as_str());
+        // Don't print the whole crate disambiguator. That's just annoying in
+        // debug output.
+        s.push_str(&tcx.crate_disambiguator(self.krate).as_str()[..7]);
 
         for component in &self.data {
             write!(s,
@@ -372,6 +398,12 @@ pub enum DefPathData {
     Typeof,
 }
 
+#[derive(Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord, Debug,
+         RustcEncodable, RustcDecodable)]
+pub struct DefPathHash(pub Fingerprint);
+
+impl_stable_hash_for!(tuple_struct DefPathHash { fingerprint });
+
 impl Definitions {
     /// Create new empty definition map.
     pub fn new() -> Definitions {
@@ -404,7 +436,7 @@ impl Definitions {
     }
 
     #[inline(always)]
-    pub fn def_path_hash(&self, index: DefIndex) -> Fingerprint {
+    pub fn def_path_hash(&self, index: DefIndex) -> DefPathHash {
         self.table.def_path_hash(index)
     }
 

src/librustc/hir/map/mod.rs

@@ -13,7 +13,7 @@ use self::MapEntry::*;
 use self::collector::NodeCollector;
 pub use self::def_collector::{DefCollector, MacroInvocationData};
 pub use self::definitions::{Definitions, DefKey, DefPath, DefPathData,
-                            DisambiguatedDefPathData};
+                            DisambiguatedDefPathData, DefPathHash};
 
 use dep_graph::{DepGraph, DepNode};
 

src/librustc/ich/hcx.rs

@@ -10,6 +10,7 @@
 
 use hir;
 use hir::def_id::DefId;
+use hir::map::DefPathHash;
 use ich::{self, CachingCodemapView};
 use session::config::DebugInfoLevel::NoDebugInfo;
 use ty;
@@ -115,7 +116,7 @@ impl<'a, 'tcx: 'a> StableHashingContext<'a, 'tcx> {
     }
 
     #[inline]
-    pub fn def_path_hash(&mut self, def_id: DefId) -> ich::Fingerprint {
+    pub fn def_path_hash(&mut self, def_id: DefId) -> DefPathHash {
         self.tcx.def_path_hash(def_id)
     }
 

src/librustc/middle/cstore.rs

@@ -26,7 +26,8 @@ use hir::def;
 use dep_graph::DepNode;
 use hir::def_id::{CrateNum, DefId, DefIndex};
 use hir::map as hir_map;
-use hir::map::definitions::{Definitions, DefKey, DisambiguatedDefPathData};
+use hir::map::definitions::{Definitions, DefKey, DisambiguatedDefPathData,
+                            DefPathTable};
 use hir::svh::Svh;
 use ich;
 use middle::lang_items;
@@ -281,7 +282,8 @@ pub trait CrateStore {
                     -> Option<DefId>;
     fn def_key(&self, def: DefId) -> DefKey;
     fn def_path(&self, def: DefId) -> hir_map::DefPath;
-    fn def_path_hash(&self, def: DefId) -> ich::Fingerprint;
+    fn def_path_hash(&self, def: DefId) -> hir_map::DefPathHash;
+    fn def_path_table(&self, cnum: CrateNum) -> Rc<DefPathTable>;
     fn struct_field_names(&self, def: DefId) -> Vec<ast::Name>;
     fn item_children(&self, did: DefId) -> Vec<def::Export>;
     fn load_macro(&self, did: DefId, sess: &Session) -> LoadedMacro;
@@ -412,8 +414,11 @@ impl CrateStore for DummyCrateStore {
     fn def_path(&self, def: DefId) -> hir_map::DefPath {
         bug!("relative_def_path")
     }
-    fn def_path_hash(&self, def: DefId) -> ich::Fingerprint {
-        bug!("wa")
+    fn def_path_hash(&self, def: DefId) -> hir_map::DefPathHash {
+        bug!("def_path_hash")
+    }
+    fn def_path_table(&self, cnum: CrateNum) -> Rc<DefPathTable> {
+        bug!("def_path_table")
     }
     fn struct_field_names(&self, def: DefId) -> Vec<ast::Name> { bug!("struct_field_names") }
     fn item_children(&self, did: DefId) -> Vec<def::Export> { bug!("item_children") }

src/librustc/ty/context.rs

@@ -18,7 +18,7 @@ use hir::TraitMap;
 use hir::def::{Def, ExportMap};
 use hir::def_id::{CrateNum, DefId, LOCAL_CRATE};
 use hir::map as hir_map;
-use hir::map::DisambiguatedDefPathData;
+use hir::map::{DisambiguatedDefPathData, DefPathHash};
 use middle::free_region::FreeRegionMap;
 use middle::lang_items;
 use middle::resolve_lifetime;
@@ -461,6 +461,10 @@ pub struct GlobalCtxt<'tcx> {
 
     pub hir: hir_map::Map<'tcx>,
 
+    /// A map from DefPathHash -> DefId. Includes DefIds from the local crate
+    /// as well as all upstream crates. Only populated in incremental mode.
+    pub def_path_hash_to_def_id: Option<FxHashMap<DefPathHash, DefId>>,
+
     pub maps: maps::Maps<'tcx>,
 
     pub mir_passes: Rc<Passes>,
@@ -686,6 +690,40 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
         let max_cnum = s.cstore.crates().iter().map(|c| c.as_usize()).max().unwrap_or(0);
         let mut providers = IndexVec::from_elem_n(extern_providers, max_cnum + 1);
         providers[LOCAL_CRATE] = local_providers;
+
+        let def_path_hash_to_def_id = if s.opts.build_dep_graph() {
+            let upstream_def_path_tables: Vec<(CrateNum, Rc<_>)> = s
+                .cstore
+                .crates()
+                .iter()
+                .map(|&cnum| (cnum, s.cstore.def_path_table(cnum)))
+                .collect();
+
+            let def_path_tables = || {
+                upstream_def_path_tables
+                    .iter()
+                    .map(|&(cnum, ref rc)| (cnum, &**rc))
+                    .chain(iter::once((LOCAL_CRATE, hir.definitions().def_path_table())))
+            };
+
+            // Precompute the capacity of the hashmap so we don't have to
+            // re-allocate when populating it.
+            let capacity = def_path_tables().map(|(_, t)| t.size()).sum::<usize>();
+
+            let mut map: FxHashMap<_, _> = FxHashMap::with_capacity_and_hasher(
+                capacity,
+                ::std::default::Default::default()
+            );
+
+            for (cnum, def_path_table) in def_path_tables() {
+                def_path_table.add_def_path_hashes_to(cnum, &mut map);
+            }
+
+            Some(map)
+        } else {
+            None
+        };
+
         tls::enter_global(GlobalCtxt {
             sess: s,
             trans_trait_caches: traits::trans::TransTraitCaches::new(dep_graph.clone()),
@@ -699,6 +737,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
             export_map: resolutions.export_map,
             fulfilled_predicates: RefCell::new(fulfilled_predicates),
             hir: hir,
+            def_path_hash_to_def_id: def_path_hash_to_def_id,
             maps: maps::Maps::new(providers),
             mir_passes,
             freevars: RefCell::new(resolutions.freevars),

src/librustc/ty/mod.rs

@@ -19,7 +19,7 @@ use dep_graph::DepNode;
 use hir::{map as hir_map, FreevarMap, TraitMap};
 use hir::def::{Def, CtorKind, ExportMap};
 use hir::def_id::{CrateNum, DefId, DefIndex, CRATE_DEF_INDEX, LOCAL_CRATE};
-use ich::{self, StableHashingContext};
+use ich::StableHashingContext;
 use middle::const_val::ConstVal;
 use middle::lang_items::{FnTraitLangItem, FnMutTraitLangItem, FnOnceTraitLangItem};
 use middle::privacy::AccessLevels;
@@ -2167,7 +2167,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
     }
 
     #[inline]
-    pub fn def_path_hash(self, def_id: DefId) -> ich::Fingerprint {
+    pub fn def_path_hash(self, def_id: DefId) -> hir_map::DefPathHash {
         if def_id.is_local() {
             self.hir.definitions().def_path_hash(def_id.index)
         } else {

src/librustc/ty/sty.rs

@@ -11,6 +11,7 @@
 //! This module contains TypeVariants and its major components
 
 use hir::def_id::DefId;
+use hir::map::DefPathHash;
 
 use middle::region;
 use ty::subst::Substs;
@@ -29,7 +30,6 @@ use util::nodemap::FxHashMap;
 use serialize;
 
 use hir;
-use ich;
 
 use self::InferTy::*;
 use self::TypeVariants::*;
@@ -873,7 +873,7 @@ impl<'a, 'tcx, 'gcx> ExistentialProjection<'tcx> {
         self.item_name // safe to skip the binder to access a name
     }
 
-    pub fn sort_key(&self, tcx: TyCtxt<'a, 'gcx, 'tcx>) -> (ich::Fingerprint, InternedString) {
+    pub fn sort_key(&self, tcx: TyCtxt<'a, 'gcx, 'tcx>) -> (DefPathHash, InternedString) {
         // We want something here that is stable across crate boundaries.
         // The DefId isn't but the `deterministic_hash` of the corresponding
         // DefPath is.
@@ -908,7 +908,7 @@ impl<'a, 'tcx, 'gcx> PolyExistentialProjection<'tcx> {
         self.skip_binder().item_name()
     }
 
-    pub fn sort_key(&self, tcx: TyCtxt<'a, 'gcx, 'tcx>) -> (ich::Fingerprint, InternedString) {
+    pub fn sort_key(&self, tcx: TyCtxt<'a, 'gcx, 'tcx>) -> (DefPathHash, InternedString) {
         self.skip_binder().sort_key(tcx)
     }
 

src/librustc/ty/trait_def.rs

@@ -9,7 +9,7 @@
 // except according to those terms.
 
 use hir::def_id::DefId;
-use ich::Fingerprint;
+use hir::map::DefPathHash;
 use traits::specialization_graph;
 use ty::fast_reject;
 use ty::fold::TypeFoldable;
@@ -33,7 +33,7 @@ pub struct TraitDef {
 
     /// The ICH of this trait's DefPath, cached here so it doesn't have to be
     /// recomputed all the time.
-    pub def_path_hash: Fingerprint,
+    pub def_path_hash: DefPathHash,
 }
 
 // We don't store the list of impls in a flat list because each cached list of
@@ -95,7 +95,7 @@ impl<'a, 'gcx, 'tcx> TraitDef {
                unsafety: hir::Unsafety,
                paren_sugar: bool,
                has_default_impl: bool,
-               def_path_hash: Fingerprint)
+               def_path_hash: DefPathHash)
                -> TraitDef {
         TraitDef {
             def_id,

src/librustc_incremental/calculate_svh/mod.rs

@@ -32,6 +32,7 @@ use std::hash::Hash;
 use rustc::dep_graph::DepNode;
 use rustc::hir;
 use rustc::hir::def_id::{CRATE_DEF_INDEX, DefId};
+use rustc::hir::map::DefPathHash;
 use rustc::hir::itemlikevisit::ItemLikeVisitor;
 use rustc::ich::{Fingerprint, StableHashingContext};
 use rustc::ty::TyCtxt;
@@ -218,7 +219,7 @@ impl<'a, 'tcx: 'a> ComputeItemHashesVisitor<'a, 'tcx> {
     {
         let tcx = self.hcx.tcx();
 
-        let mut impls: Vec<(Fingerprint, Fingerprint)> = krate
+        let mut impls: Vec<(DefPathHash, Fingerprint)> = krate
             .trait_impls
             .iter()
             .map(|(&trait_id, impls)| {