diff --git a/src/librustc_middle/arena.rs b/src/librustc_middle/arena.rs index f6570cc95d27d..4e6dc8e13d920 100644 --- a/src/librustc_middle/arena.rs +++ b/src/librustc_middle/arena.rs @@ -94,6 +94,7 @@ macro_rules! arena_types { [few] hir_definitions: rustc_hir::definitions::Definitions, [] hir_owner: rustc_middle::hir::Owner<$tcx>, [] hir_owner_nodes: rustc_middle::hir::OwnerNodes<$tcx>, + [] hir_owner_defs: rustc_data_structures::fx::FxHashMap, // Note that this deliberately duplicates items in the `rustc_hir::arena`, // since we need to allocate this type on both the `rustc_hir` arena diff --git a/src/librustc_middle/dep_graph/mod.rs b/src/librustc_middle/dep_graph/mod.rs index 6697524279874..bcc4db9bbf75f 100644 --- a/src/librustc_middle/dep_graph/mod.rs +++ b/src/librustc_middle/dep_graph/mod.rs @@ -122,17 +122,7 @@ impl<'tcx> DepContext for TyCtxt<'tcx> { ); } } else { - // This `DefPath` does not have a - // corresponding `DepNode` (e.g. a - // struct field), and the ` DefPath` - // collided with the `DefPath` of a - // proper item that existed in the - // previous compilation session. - // - // Since the given `DefPath` does not - // denote the item that previously - // existed, we just fail to mark green. - return false; + bug!("DepNode {:?} collided (DefId {:?})", dep_node, def_id); } } else { // If the node does not exist anymore, we diff --git a/src/librustc_middle/hir/map/collector.rs b/src/librustc_middle/hir/map/collector.rs index dce06a5f7eeec..9ad65ea950091 100644 --- a/src/librustc_middle/hir/map/collector.rs +++ b/src/librustc_middle/hir/map/collector.rs @@ -133,7 +133,7 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> { hcx, hir_body_nodes, map: (0..definitions.def_index_count()) - .map(|_| HirOwnerData { signature: None, with_bodies: None }) + .map(|_| HirOwnerData { signature: None, with_bodies: None, defs: None }) .collect(), }; collector.insert_entry( @@ -229,6 +229,14 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> { id.local_id, ParentedNode { parent: entry.parent.local_id, node: entry.node }, ); + + // Check if this HirId has a DefId, and insert it in the `defs` map if so. + if let Some(def_id) = self.definitions.opt_hir_id_to_local_def_id(id) { + if data.defs.is_none() { + data.defs = Some(arena.alloc(FxHashMap::default())); + }; + data.defs.as_mut().unwrap().insert(id.local_id, def_id); + } } } diff --git a/src/librustc_middle/hir/map/mod.rs b/src/librustc_middle/hir/map/mod.rs index a6cc7cbc9207c..1896b72959039 100644 --- a/src/librustc_middle/hir/map/mod.rs +++ b/src/librustc_middle/hir/map/mod.rs @@ -4,6 +4,7 @@ use crate::hir::{Owner, OwnerNodes}; use crate::ty::query::Providers; use crate::ty::TyCtxt; use rustc_ast as ast; +use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::svh::Svh; use rustc_hir::def::{DefKind, Res}; use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, CRATE_DEF_INDEX, LOCAL_CRATE}; @@ -89,6 +90,7 @@ fn is_body_owner<'hir>(node: Node<'hir>, hir_id: HirId) -> bool { pub(super) struct HirOwnerData<'hir> { pub(super) signature: Option<&'hir Owner<'hir>>, pub(super) with_bodies: Option<&'hir mut OwnerNodes<'hir>>, + pub(super) defs: Option<&'hir mut FxHashMap>, } pub struct IndexedHir<'hir> { @@ -170,17 +172,22 @@ impl<'hir> Map<'hir> { #[inline] pub fn opt_local_def_id(&self, hir_id: HirId) -> Option { - self.tcx.definitions.opt_hir_id_to_local_def_id(hir_id) + if hir_id.local_id == ItemLocalId::from_u32(0) { + // Every HirId owner has a DefId, so we can just return it directly here + Some(hir_id.owner) + } else { + self.tcx.hir_owner_defs(hir_id.owner).and_then(|map| map.get(&hir_id.local_id).copied()) + } } #[inline] pub fn local_def_id_to_hir_id(&self, def_id: LocalDefId) -> HirId { - self.tcx.definitions.local_def_id_to_hir_id(def_id) + self.tcx.local_def_id_to_hir_id(def_id).unwrap() } #[inline] pub fn opt_local_def_id_to_hir_id(&self, def_id: LocalDefId) -> Option { - self.tcx.definitions.opt_local_def_id_to_hir_id(def_id) + self.tcx.local_def_id_to_hir_id(def_id) } pub fn def_kind(&self, local_def_id: LocalDefId) -> DefKind { diff --git a/src/librustc_middle/hir/mod.rs b/src/librustc_middle/hir/mod.rs index ae3b30217cc4a..488e520659aa9 100644 --- a/src/librustc_middle/hir/mod.rs +++ b/src/librustc_middle/hir/mod.rs @@ -92,5 +92,8 @@ pub fn provide(providers: &mut Providers) { span_bug!(hir.span(hir_id), "fn_arg_names: unexpected item {:?}", id); } }; + providers.local_def_id_to_hir_id = |tcx, id| tcx.definitions.opt_local_def_id_to_hir_id(id); + providers.hir_owner_defs = + |tcx, index| tcx.index_hir(LOCAL_CRATE).map[index].defs.as_ref().map(|defs| &**defs); map::provide(providers); } diff --git a/src/librustc_middle/query/mod.rs b/src/librustc_middle/query/mod.rs index d874edf627472..e20bc36917618 100644 --- a/src/librustc_middle/query/mod.rs +++ b/src/librustc_middle/query/mod.rs @@ -112,6 +112,24 @@ rustc_queries! { desc { |tcx| "computing the optional const parameter of `{}`", tcx.def_path_str(key.to_def_id()) } } + // Converts a `LocalDefId` to a `HirId`. + // This can be conveniently accessed by `tcx.hir().as_local_hir_id`. + // Avoid calling this query directly. + query local_def_id_to_hir_id(key: LocalDefId) -> Option { + storage(IndexVecCacheSelector) + eval_always + desc { "converting a LocalDefId to HirId" } + } + + // A map of the HIR items which also have a `DefId` in addition to their `HirId`. + // This can be conveniently accessed by `tcx.hir().opt_local_def_id`. + // Avoid calling this query directly. + query hir_owner_defs(key: LocalDefId) -> Option<&'tcx FxHashMap> { + storage(IndexVecCacheSelector) + eval_always + desc { "getting a LocalDefId map" } + } + /// Records the type of every item. query type_of(key: DefId) -> Ty<'tcx> { desc { |tcx| "computing type of `{}`", tcx.def_path_str(key) } diff --git a/src/librustc_query_system/query/caches.rs b/src/librustc_query_system/query/caches.rs index 1839e1af45eef..a445b454a5ae8 100644 --- a/src/librustc_query_system/query/caches.rs +++ b/src/librustc_query_system/query/caches.rs @@ -1,6 +1,8 @@ use crate::dep_graph::DepNodeIndex; +use crate::query::config::QueryActiveStore; use crate::query::plumbing::{QueryLookup, QueryState}; use crate::query::QueryContext; +use rustc_index::vec::{Idx, IndexVec}; use rustc_arena::TypedArena; use rustc_data_structures::fx::FxHashMap; @@ -27,6 +29,10 @@ pub trait QueryCache: QueryStorage { type Key: Hash; type Sharded: Default; + fn make_store() -> QueryActiveStore { + QueryActiveStore::Multi(Default::default()) + } + /// Checks if the query is already computed and in the cache. /// It returns the shard index and a lock guard to the shard, /// which will be used if the query is not in the cache and we need @@ -102,10 +108,10 @@ impl QueryCache for DefaultCache { OnHit: FnOnce(&V, DepNodeIndex) -> R, OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R, { - let mut lookup = state.get_lookup(&key); + let (mut lookup, key_hash) = state.get_lookup(&key, true); let lock = &mut *lookup.lock; - let result = lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key); + let result = lock.cache.raw_entry().from_key_hashed_nocheck(key_hash.unwrap(), &key); if let Some((_, value)) = result { on_hit(&value.0, value.1) } else { on_miss(key, lookup) } } @@ -180,10 +186,10 @@ impl<'tcx, K: Eq + Hash, V: 'tcx> QueryCache for ArenaCache<'tcx, K, V> { OnHit: FnOnce(&&'tcx V, DepNodeIndex) -> R, OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R, { - let mut lookup = state.get_lookup(&key); + let (mut lookup, key_hash) = state.get_lookup(&key, true); let lock = &mut *lookup.lock; - let result = lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key); + let result = lock.cache.raw_entry().from_key_hashed_nocheck(key_hash.unwrap(), &key); if let Some((_, value)) = result { on_hit(&&value.0, value.1) @@ -218,3 +224,88 @@ impl<'tcx, K: Eq + Hash, V: 'tcx> QueryCache for ArenaCache<'tcx, K, V> { f(Box::new(results)) } } + +pub struct IndexVecCacheSelector; + +impl CacheSelector for IndexVecCacheSelector { + type Cache = IndexVecCache; +} + +pub struct IndexVecCache(PhantomData<(K, V)>); + +impl Default for IndexVecCache { + fn default() -> Self { + IndexVecCache(PhantomData) + } +} + +impl QueryStorage for IndexVecCache { + type Value = V; + type Stored = V; + + #[inline] + fn store_nocache(&self, value: Self::Value) -> Self::Stored { + // We have no dedicated storage + value + } +} + +impl QueryCache for IndexVecCache { + type Key = K; + type Sharded = IndexVec>; + + fn make_store() -> QueryActiveStore { + QueryActiveStore::Single(None) + } + + #[inline(always)] + fn lookup( + &self, + state: &QueryState, + key: K, + on_hit: OnHit, + on_miss: OnMiss, + ) -> R + where + OnHit: FnOnce(&V, DepNodeIndex) -> R, + OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R, + { + let (mut lookup, _) = state.get_lookup(&key, false); + let lock = &mut *lookup.lock; + + let result = lock.cache.get(key); + + if let Some(Some(value)) = result { + on_hit(&value.1, value.2) + } else { + on_miss(key, lookup) + } + } + + #[inline] + fn complete( + &self, + lock_sharded_storage: &mut Self::Sharded, + key: K, + value: V, + index: DepNodeIndex, + ) -> Self::Stored { + lock_sharded_storage.ensure_contains_elem(key, || None); + lock_sharded_storage[key] = Some((key, value.clone(), index)); + value + } + + fn iter( + &self, + shards: &Sharded, + get_shard: impl Fn(&mut L) -> &mut Self::Sharded, + f: impl for<'a> FnOnce(Box + 'a>) -> R, + ) -> R { + let mut shards = shards.lock_shards(); + let mut shards: Vec<_> = shards.iter_mut().map(|shard| get_shard(shard)).collect(); + let results = shards.iter_mut().flat_map(|shard| { + shard.iter().flat_map(|v| v.as_ref().map(|val| (&val.0, &val.1, val.2))) + }); + f(Box::new(results)) + } +} diff --git a/src/librustc_query_system/query/config.rs b/src/librustc_query_system/query/config.rs index 549056570f9bc..5f24b090d9c12 100644 --- a/src/librustc_query_system/query/config.rs +++ b/src/librustc_query_system/query/config.rs @@ -5,10 +5,12 @@ use crate::dep_graph::SerializedDepNodeIndex; use crate::query::caches::QueryCache; use crate::query::plumbing::CycleError; use crate::query::{QueryContext, QueryState}; +use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::profiling::ProfileCategory; use rustc_data_structures::fingerprint::Fingerprint; use std::borrow::Cow; +use std::collections::hash_map::Entry; use std::fmt::Debug; use std::hash::Hash; @@ -70,6 +72,115 @@ impl QueryVtable { } } +pub enum QueryActiveStore { + Single(Option<(K, V)>), + Multi(FxHashMap), +} + +impl QueryActiveStore { + pub fn is_empty(&self) -> bool { + match self { + QueryActiveStore::Single(val) => val.is_none(), + QueryActiveStore::Multi(val) => val.is_empty(), + } + } + pub fn iter(&self) -> Box + '_> { + match self { + QueryActiveStore::Single(val) => { + Box::new(val.as_ref().map(|entry| (&entry.0, &entry.1)).into_iter()) + } + QueryActiveStore::Multi(val) => Box::new(val.iter()), + } + } + pub fn get_or_insert( + &mut self, + key: K, + present: impl FnOnce(&mut V) -> R, + absent: impl FnOnce() -> (V, R), + ) -> R { + match self { + QueryActiveStore::Single(entry) => { + if let Some((_key, val)) = entry.as_mut() { + present(val) + } else { + let (val, res) = absent(); + *entry = Some((key, val)); + res + } + } + QueryActiveStore::Multi(map) => match map.entry(key) { + Entry::Occupied(mut entry) => present(entry.get_mut()), + Entry::Vacant(entry) => { + let (val, res) = absent(); + entry.insert(val); + res + } + }, + } + } + pub fn insert(&mut self, key: K, val: V) { + match self { + QueryActiveStore::Single(entry) => *entry = Some((key, val)), + QueryActiveStore::Multi(map) => { + map.insert(key, val); + } + } + } + pub fn remove(&mut self, key: &K) -> Option { + match self { + QueryActiveStore::Single(entry) => entry.take().map(|(_k, v)| v), + QueryActiveStore::Multi(map) => map.remove(key), + } + } +} + +/*pub trait QueryActiveStore { + fn is_empty(&self) -> bool; + //fn iter(&self) -> Box + '_>; + fn get_or_insert(&mut self, key: K, present: impl FnOnce(&mut V) -> R, absent: impl FnOnce() -> (V, R)) -> R; + fn remove(&mut self, key: &K); +} + +impl QueryActiveStore for FxHashMap { + fn is_empty(&self) -> bool { FxHashMap::is_empty(self) } + //fn iter(&self) -> Box + '_> { Box::new(FxHashMap::iter(self)) } + fn get_or_insert(&mut self, key: K, present: impl FnOnce(&mut V) -> R, absent: impl FnOnce() -> (V, R)) -> R { + match self.entry(key) { + Entry::Occupied(mut entry) => { + present(entry.get_mut()) + } + Entry::Vacant(entry) => { + let (val, res) = absent(); + entry.insert(val); + res + } + } + } + fn remove(&mut self, key: &K) { + FxHashMap::remove(self, key); + } +} + +impl QueryActiveStore for RefCell> { + fn is_empty(&self) -> bool { self.borrow().is_none() } + /*fn iter(&self) -> Box + '_> { + Box::new(self.borrow().as_ref().map(|val| (&val.0, &val.1)).into_iter()) + }*/ + fn get_or_insert(&mut self, key: K, present: impl FnOnce(&mut V) -> R, absent: impl FnOnce() -> (V, R)) -> R { + let entry = &mut *self.borrow_mut(); + if let Some((_key, val)) = entry { + present(val) + } else { + let (val, res) = absent(); + *entry = Some((key, val)); + res + } + } + fn remove(&mut self, key: &K) { + self.borrow_mut().take(); + } +}*/ + pub trait QueryAccessors: QueryConfig { const ANON: bool; const EVAL_ALWAYS: bool; diff --git a/src/librustc_query_system/query/mod.rs b/src/librustc_query_system/query/mod.rs index 49097725bc9b9..946881344cf71 100644 --- a/src/librustc_query_system/query/mod.rs +++ b/src/librustc_query_system/query/mod.rs @@ -8,15 +8,17 @@ pub use self::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo}; mod caches; pub use self::caches::{ - ArenaCacheSelector, CacheSelector, DefaultCacheSelector, QueryCache, QueryStorage, + ArenaCacheSelector, CacheSelector, DefaultCacheSelector, IndexVecCacheSelector, QueryCache, + QueryStorage, }; mod config; pub use self::config::{QueryAccessors, QueryConfig, QueryDescription}; +use rustc_data_structures::fx::FxHashMap; + use crate::dep_graph::{DepContext, DepGraph}; -use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stable_hasher::HashStable; use rustc_data_structures::sync::Lock; use rustc_data_structures::thin_vec::ThinVec; diff --git a/src/librustc_query_system/query/plumbing.rs b/src/librustc_query_system/query/plumbing.rs index ae042cc808126..b47c2a6711000 100644 --- a/src/librustc_query_system/query/plumbing.rs +++ b/src/librustc_query_system/query/plumbing.rs @@ -5,7 +5,7 @@ use crate::dep_graph::{DepKind, DepNode}; use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; use crate::query::caches::QueryCache; -use crate::query::config::{QueryDescription, QueryVtable, QueryVtableExt}; +use crate::query::config::{QueryActiveStore, QueryDescription, QueryVtable, QueryVtableExt}; use crate::query::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryShardJobId}; use crate::query::QueryContext; @@ -13,13 +13,12 @@ use crate::query::QueryContext; use rustc_data_structures::cold_path; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxHashMap, FxHasher}; -use rustc_data_structures::sharded::Sharded; +use rustc_data_structures::sharded::{Sharded, SHARDS}; use rustc_data_structures::sync::{Lock, LockGuard}; use rustc_data_structures::thin_vec::ThinVec; use rustc_errors::{Diagnostic, FatalError}; use rustc_span::source_map::DUMMY_SP; use rustc_span::Span; -use std::collections::hash_map::Entry; use std::convert::TryFrom; use std::fmt::Debug; use std::hash::{Hash, Hasher}; @@ -31,18 +30,12 @@ use std::sync::atomic::{AtomicUsize, Ordering}; pub(super) struct QueryStateShard { pub(super) cache: C, - active: FxHashMap>, + active: QueryActiveStore>, /// Used to generate unique ids for active jobs. jobs: u32, } -impl Default for QueryStateShard { - fn default() -> QueryStateShard { - QueryStateShard { cache: Default::default(), active: Default::default(), jobs: 0 } - } -} - pub struct QueryState { cache: C, shards: Sharded>, @@ -55,17 +48,28 @@ impl QueryState { pub(super) fn get_lookup<'tcx>( &'tcx self, key: &C::Key, - ) -> QueryLookup<'tcx, CTX, C::Key, C::Sharded> { - // We compute the key's hash once and then use it for both the - // shard lookup and the hashmap lookup. This relies on the fact - // that both of them use `FxHasher`. - let mut hasher = FxHasher::default(); - key.hash(&mut hasher); - let key_hash = hasher.finish(); - - let shard = self.shards.get_shard_index_by_hash(key_hash); - let lock = self.shards.get_shard_by_index(shard).lock(); - QueryLookup { key_hash, shard, lock } + needs_hash: bool, + ) -> (QueryLookup<'tcx, CTX, C::Key, C::Sharded>, Option) { + let key_hash = if needs_hash || SHARDS != 1 { + // We compute the key's hash once and then use it for both the + // shard lookup and the hashmap lookup. This relies on the fact + // that both of them use `FxHasher`. + let mut hasher = FxHasher::default(); + key.hash(&mut hasher); + Some(hasher.finish()) + } else { + None + }; + + let (shard, lock) = if SHARDS == 1 { + (0, self.shards.get_shard_by_index(0).lock()) + } else { + let shard = self.shards.get_shard_index_by_hash(key_hash.unwrap()); + let lock = self.shards.get_shard_by_index(shard).lock(); + (shard, lock) + }; + + (QueryLookup { shard, lock }, key_hash) } } @@ -89,7 +93,12 @@ impl QueryState { ) -> R { self.cache.iter(&self.shards, |shard| &mut shard.cache, f) } +} +impl QueryState +where + C::Key: Eq + Hash + Clone + Debug, +{ #[inline(always)] pub fn all_inactive(&self) -> bool { let shards = self.shards.lock_shards(); @@ -130,7 +139,11 @@ impl Default for QueryState { fn default() -> QueryState { QueryState { cache: C::default(), - shards: Default::default(), + shards: Sharded::new(|| QueryStateShard { + cache: Default::default(), + active: C::make_store(), + jobs: 0, + }), #[cfg(debug_assertions)] cache_hits: AtomicUsize::new(0), } @@ -139,7 +152,6 @@ impl Default for QueryState { /// Values used when checking a query cache which can be reused on a cache-miss to execute the query. pub struct QueryLookup<'tcx, CTX: QueryContext, K, C> { - pub(super) key_hash: u64, shard: usize, pub(super) lock: LockGuard<'tcx, QueryStateShard>, } @@ -183,9 +195,13 @@ where { let lock = &mut *lookup.lock; - let (latch, mut _query_blocked_prof_timer) = match lock.active.entry((*key).clone()) { - Entry::Occupied(mut entry) => { - match entry.get_mut() { + let shard = lookup.shard; + let jobs = &mut lock.jobs; + + let res = lock.active.get_or_insert( + (*key).clone(), + |result| { + match result { QueryResult::Started(job) => { // For parallel queries, we'll block and wait until the query running // in another thread has completed. Record how long we wait in the @@ -197,34 +213,39 @@ where }; // Create the id of the job we're waiting for - let id = QueryJobId::new(job.id, lookup.shard, query.dep_kind); + let id = QueryJobId::new(job.id, shard, query.dep_kind); - (job.latch(id), _query_blocked_prof_timer) + Err((job.latch(id), _query_blocked_prof_timer)) } QueryResult::Poisoned => FatalError.raise(), } - } - Entry::Vacant(entry) => { + }, + || { // No job entry for this query. Return a new one to be started later. // Generate an id unique within this shard. - let id = lock.jobs.checked_add(1).unwrap(); - lock.jobs = id; + let id = jobs.checked_add(1).unwrap(); + *jobs = id; let id = QueryShardJobId(NonZeroU32::new(id).unwrap()); - let global_id = QueryJobId::new(id, lookup.shard, query.dep_kind); + let global_id = QueryJobId::new(id, shard, query.dep_kind); let job = tcx.current_query_job(); let job = QueryJob::new(id, span, job); - entry.insert(QueryResult::Started(job)); + let result = QueryResult::Started(job); let owner = JobOwner { state, id: global_id, key: (*key).clone() }; - return TryGetJob::NotYetStarted(owner); - } - }; + (result, Ok(TryGetJob::NotYetStarted(owner))) + }, + ); mem::drop(lookup.lock); + let (latch, mut _query_blocked_prof_timer) = match res { + Ok(val) => return val, + Err(val) => val, + }; + // If we are single-threaded we know that we have cycle error, // so we just return the error. #[cfg(not(parallel_compiler))]