ICH: Replace old, transitive metadata hashing with direct hashing approach.

Instead of collecting all potential inputs to some metadata entry and
hashing those, we directly hash the values we are storing in metadata.
This is more accurate and doesn't suffer from quadratic blow-up when
many entries have the same dependencies.

Author: michaelwoerister

src/librustc/ich/impls_ty.rs

@@ -16,24 +16,9 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher,
                                            StableHasherResult};
 use std::hash as std_hash;
 use std::mem;
+use syntax_pos::symbol::InternedString;
 use ty;
 
-impl<'a, 'tcx> HashStable<StableHashingContext<'a, 'tcx>> for ty::TyS<'tcx> {
-    fn hash_stable<W: StableHasherResult>(&self,
-                                          hcx: &mut StableHashingContext<'a, 'tcx>,
-                                          hasher: &mut StableHasher<W>) {
-        let ty::TyS {
-            ref sty,
-
-            // The other fields just provide fast access to information that is
-            // also contained in `sty`, so no need to hash them.
-            ..
-        } = *self;
-
-        sty.hash_stable(hcx, hasher);
-    }
-}
-
 impl_stable_hash_for!(struct ty::ItemSubsts<'tcx> { substs });
 
 impl<'a, 'tcx, T> HashStable<StableHashingContext<'a, 'tcx>> for &'tcx ty::Slice<T>
@@ -288,9 +273,14 @@ for ::middle::const_val::ConstVal<'tcx> {
                 def_id.hash_stable(hcx, hasher);
                 substs.hash_stable(hcx, hasher);
             }
-            ConstVal::Struct(ref _name_value_map) => {
-                // BTreeMap<ast::Name, ConstVal<'tcx>>),
-                panic!("Ordering still unstable")
+            ConstVal::Struct(ref name_value_map) => {
+                let mut values: Vec<(InternedString, &ConstVal)> =
+                    name_value_map.iter()
+                                  .map(|(name, val)| (name.as_str(), val))
+                                  .collect();
+
+                values.sort_unstable_by_key(|&(ref name, _)| name.clone());
+                values.hash_stable(hcx, hasher);
             }
             ConstVal::Tuple(ref value) => {
                 value.hash_stable(hcx, hasher);
@@ -632,6 +622,8 @@ impl<'a, 'tcx> HashStable<StableHashingContext<'a, 'tcx>> for ty::TypeckTables<'
             ref fru_field_types,
 
             ref cast_kinds,
+
+            // FIXME(#41184): This is still ignored at the moment.
             lints: _,
             ref used_trait_imports,
             tainted_by_errors,
@@ -672,7 +664,7 @@ impl<'a, 'tcx> HashStable<StableHashingContext<'a, 'tcx>> for ty::TypeckTables<'
             ich::hash_stable_nodemap(hcx, hasher, cast_kinds);
 
             ich::hash_stable_hashset(hcx, hasher, used_trait_imports, |hcx, def_id| {
-                hcx.tcx().def_path_hash(*def_id)
+                hcx.def_path_hash(*def_id)
             });
 
             tainted_by_errors.hash_stable(hcx, hasher);

src/librustc/middle/cstore.rs

@@ -27,6 +27,7 @@ use hir::def_id::{CrateNum, DefId, DefIndex};
 use hir::map as hir_map;
 use hir::map::definitions::{Definitions, DefKey, DisambiguatedDefPathData};
 use hir::svh::Svh;
+use ich;
 use middle::lang_items;
 use ty::{self, TyCtxt};
 use session::Session;
@@ -161,6 +162,20 @@ pub struct ExternCrate {
     pub path_len: usize,
 }
 
+pub struct EncodedMetadata {
+    pub raw_data: Vec<u8>,
+    pub hashes: Vec<EncodedMetadataHash>,
+}
+
+/// The hash for some metadata that (when saving) will be exported
+/// from this crate, or which (when importing) was exported by an
+/// upstream crate.
+#[derive(Debug, RustcEncodable, RustcDecodable, Copy, Clone)]
+pub struct EncodedMetadataHash {
+    pub def_index: DefIndex,
+    pub hash: ich::Fingerprint,
+}
+
 /// A store of Rust crates, through with their metadata
 /// can be accessed.
 pub trait CrateStore {
@@ -258,7 +273,8 @@ pub trait CrateStore {
     fn encode_metadata<'a, 'tcx>(&self,
                                  tcx: TyCtxt<'a, 'tcx, 'tcx>,
                                  link_meta: &LinkMeta,
-                                 reachable: &NodeSet) -> Vec<u8>;
+                                 reachable: &NodeSet)
+                                 -> EncodedMetadata;
     fn metadata_encoding_version(&self) -> &[u8];
 }
 
@@ -417,7 +433,10 @@ impl CrateStore for DummyCrateStore {
     fn encode_metadata<'a, 'tcx>(&self,
                                  tcx: TyCtxt<'a, 'tcx, 'tcx>,
                                  link_meta: &LinkMeta,
-                                 reachable: &NodeSet) -> Vec<u8> { vec![] }
+                                 reachable: &NodeSet)
+                                 -> EncodedMetadata {
+        bug!("encode_metadata")
+    }
     fn metadata_encoding_version(&self) -> &[u8] { bug!("metadata_encoding_version") }
 }
 

src/librustc/ty/mod.rs

@@ -452,6 +452,23 @@ impl<'tcx> Hash for TyS<'tcx> {
     }
 }
 
+impl<'a, 'tcx> HashStable<StableHashingContext<'a, 'tcx>> for ty::TyS<'tcx> {
+    fn hash_stable<W: StableHasherResult>(&self,
+                                          hcx: &mut StableHashingContext<'a, 'tcx>,
+                                          hasher: &mut StableHasher<W>) {
+        let ty::TyS {
+            ref sty,
+
+            // The other fields just provide fast access to information that is
+            // also contained in `sty`, so no need to hash them.
+            flags: _,
+            region_depth: _,
+        } = *self;
+
+        sty.hash_stable(hcx, hasher);
+    }
+}
+
 pub type Ty<'tcx> = &'tcx TyS<'tcx>;
 
 impl<'tcx> serialize::UseSpecializedEncodable for Ty<'tcx> {}

src/librustc/ty/util.rs

@@ -13,7 +13,7 @@
 use hir::def_id::{DefId, LOCAL_CRATE};
 use hir::map::DefPathData;
 use infer::InferCtxt;
-// use hir::map as hir_map;
+use ich::{StableHashingContext, NodeIdHashingMode};
 use traits::{self, Reveal};
 use ty::{self, Ty, TyCtxt, TypeAndMut, TypeFlags, TypeFoldable};
 use ty::ParameterEnvironment;
@@ -25,8 +25,8 @@ use util::nodemap::FxHashMap;
 use middle::lang_items;
 
 use rustc_const_math::{ConstInt, ConstIsize, ConstUsize};
-use rustc_data_structures::stable_hasher::{StableHasher, StableHasherResult};
-
+use rustc_data_structures::stable_hasher::{StableHasher, StableHasherResult,
+                                           HashStable};
 use std::cell::RefCell;
 use std::cmp;
 use std::hash::Hash;
@@ -187,6 +187,22 @@ impl<'tcx> ParameterEnvironment<'tcx> {
     }
 }
 
+impl<'a, 'tcx> TyCtxt<'a, 'tcx, 'tcx> {
+    /// Creates a hash of the type `Ty` which will be the same no matter what crate
+    /// context it's calculated within. This is used by the `type_id` intrinsic.
+    pub fn type_id_hash(self, ty: Ty<'tcx>) -> u64 {
+        let mut hasher = StableHasher::new();
+        let mut hcx = StableHashingContext::new(self);
+
+        hcx.while_hashing_spans(false, |hcx| {
+            hcx.with_node_id_hashing_mode(NodeIdHashingMode::HashDefPath, |hcx| {
+                ty.hash_stable(hcx, &mut hasher);
+            });
+        });
+        hasher.finish()
+    }
+}
+
 impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
     pub fn has_error_field(self, ty: Ty<'tcx>) -> bool {
         match ty.sty {
@@ -339,14 +355,6 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
             .collect()
     }
 
-    /// Creates a hash of the type `Ty` which will be the same no matter what crate
-    /// context it's calculated within. This is used by the `type_id` intrinsic.
-    pub fn type_id_hash(self, ty: Ty<'tcx>) -> u64 {
-        let mut hasher = TypeIdHasher::new(self);
-        hasher.visit_ty(ty);
-        hasher.finish()
-    }
-
     /// Calculate the destructor of a given type.
     pub fn calculate_dtor(
         self,

src/librustc_data_structures/blake2b.rs

@@ -29,16 +29,23 @@ pub struct Blake2bCtx {
     t: [u64; 2],
     c: usize,
     outlen: u16,
-    finalized: bool
+    finalized: bool,
+
+    #[cfg(debug_assertions)]
+    fnv_hash: u64,
 }
 
+#[cfg(debug_assertions)]
 impl ::std::fmt::Debug for Blake2bCtx {
-    fn fmt(&self, fmt: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> {
-        try!(write!(fmt, "hash: "));
-        for v in &self.h {
-            try!(write!(fmt, "{:x}", v));
-        }
-        Ok(())
+    fn fmt(&self, fmt: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
+        write!(fmt, "{:x}", self.fnv_hash)
+    }
+}
+
+#[cfg(not(debug_assertions))]
+impl ::std::fmt::Debug for Blake2bCtx {
+    fn fmt(&self, fmt: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
+        write!(fmt, "Enable debug_assertions() for more info.")
     }
 }
 
@@ -157,6 +164,9 @@ fn blake2b_new(outlen: usize, key: &[u8]) -> Blake2bCtx {
         c: 0,
         outlen: outlen as u16,
         finalized: false,
+
+        #[cfg(debug_assertions)]
+        fnv_hash: 0xcbf29ce484222325,
     };
 
     ctx.h[0] ^= 0x01010000 ^ ((key.len() << 8) as u64) ^ (outlen as u64);
@@ -194,6 +204,16 @@ fn blake2b_update(ctx: &mut Blake2bCtx, mut data: &[u8]) {
         checked_mem_copy(data, &mut ctx.b[ctx.c .. ], bytes_to_copy);
         ctx.c += bytes_to_copy;
     }
+
+    #[cfg(debug_assertions)]
+    {
+        // compute additional FNV hash for simpler to read debug output
+        const MAGIC_PRIME: u64 = 0x00000100000001b3;
+
+        for &byte in data {
+            ctx.fnv_hash = (ctx.fnv_hash ^ byte as u64).wrapping_mul(MAGIC_PRIME);
+        }
+    }
 }
 
 fn blake2b_final(ctx: &mut Blake2bCtx)

src/librustc_data_structures/lib.rs

@@ -40,6 +40,7 @@
 #![feature(discriminant_value)]
 #![feature(specialization)]
 #![feature(manually_drop)]
+#![feature(struct_field_attributes)]
 
 #![cfg_attr(unix, feature(libc))]
 #![cfg_attr(test, feature(test))]

src/librustc_data_structures/stable_hasher.rs

@@ -40,13 +40,18 @@ fn write_signed_leb128_to_buf(buf: &mut [u8; 16], value: i64) -> usize {
 /// This hasher currently always uses the stable Blake2b algorithm
 /// and allows for variable output lengths through its type
 /// parameter.
-#[derive(Debug)]
 pub struct StableHasher<W> {
     state: Blake2bHasher,
     bytes_hashed: u64,
     width: PhantomData<W>,
 }
 
+impl<W: StableHasherResult> ::std::fmt::Debug for StableHasher<W> {
+    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
+        write!(f, "{:?}", self.state)
+    }
+}
+
 pub trait StableHasherResult: Sized {
     fn finish(hasher: StableHasher<Self>) -> Self;
 }

src/librustc_driver/driver.rs

@@ -1083,6 +1083,7 @@ pub fn phase_4_translate_to_llvm<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
          "serialize dep graph",
          || rustc_incremental::save_dep_graph(tcx,
                                               &incremental_hashes_map,
+                                              &translation.metadata.hashes,
                                               translation.link.crate_hash));
     translation
 }

src/librustc_incremental/persist/data.rs

@@ -13,6 +13,7 @@
 use rustc::dep_graph::{DepNode, WorkProduct, WorkProductId};
 use rustc::hir::def_id::DefIndex;
 use rustc::ich::Fingerprint;
+use rustc::middle::cstore::EncodedMetadataHash;
 use std::sync::Arc;
 use rustc_data_structures::fx::FxHashMap;
 
@@ -98,7 +99,7 @@ pub struct SerializedMetadataHashes {
     /// where `X` refers to some item in this crate. That `X` will be
     /// a `DefPathIndex` that gets retracted to the current `DefId`
     /// (matching the one found in this structure).
-    pub hashes: Vec<SerializedMetadataHash>,
+    pub hashes: Vec<EncodedMetadataHash>,
 
     /// For each DefIndex (as it occurs in SerializedMetadataHash), this
     /// map stores the DefPathIndex (as it occurs in DefIdDirectory), so
@@ -112,14 +113,3 @@ pub struct SerializedMetadataHashes {
     /// the DefIndex.
     pub index_map: FxHashMap<DefIndex, DefPathIndex>
 }
-
-/// The hash for some metadata that (when saving) will be exported
-/// from this crate, or which (when importing) was exported by an
-/// upstream crate.
-#[derive(Debug, RustcEncodable, RustcDecodable)]
-pub struct SerializedMetadataHash {
-    pub def_index: DefIndex,
-
-    /// the hash itself, computed by `calculate_item_hash`
-    pub hash: Fingerprint,
-}