Skip to content

Commit 1ebfab5

Browse files
trans: Collect all accesses between trans-items, not just inlining edges.
1 parent 8bba5ad commit 1ebfab5

File tree

1 file changed

+105
-29
lines changed

1 file changed

+105
-29
lines changed

src/librustc_trans/collector.rs

+105-29
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,8 @@ use rustc::util::nodemap::{FxHashSet, FxHashMap, DefIdMap};
208208

209209
use trans_item::{TransItem, DefPathBasedNames, InstantiationMode};
210210

211+
use rustc_data_structures::bitvec::BitVector;
212+
211213
#[derive(PartialEq, Eq, Hash, Clone, Copy, Debug)]
212214
pub enum TransItemCollectionMode {
213215
Eager,
@@ -217,12 +219,16 @@ pub enum TransItemCollectionMode {
217219
/// Maps every translation item to all translation items it references in its
218220
/// body.
219221
pub struct InliningMap<'tcx> {
220-
// Maps a source translation item to a range of target translation items
221-
// that are potentially inlined by LLVM into the source.
222+
// Maps a source translation item to the range of translation items
223+
// accessed by it.
222224
// The two numbers in the tuple are the start (inclusive) and
223225
// end index (exclusive) within the `targets` vecs.
224226
index: FxHashMap<TransItem<'tcx>, (usize, usize)>,
225227
targets: Vec<TransItem<'tcx>>,
228+
229+
// Contains one bit per translation item in the `targets` field. That bit
230+
// is true if that translation item needs to be inlined into every CGU.
231+
inlines: BitVector,
226232
}
227233

228234
impl<'tcx> InliningMap<'tcx> {
@@ -231,33 +237,61 @@ impl<'tcx> InliningMap<'tcx> {
231237
InliningMap {
232238
index: FxHashMap(),
233239
targets: Vec::new(),
240+
inlines: BitVector::new(1024),
234241
}
235242
}
236243

237-
fn record_inlining_canditates<I>(&mut self,
238-
source: TransItem<'tcx>,
239-
targets: I)
240-
where I: Iterator<Item=TransItem<'tcx>>
244+
fn record_accesses<I>(&mut self,
245+
source: TransItem<'tcx>,
246+
targets: I)
247+
where I: Iterator<Item=(TransItem<'tcx>, bool)>
241248
{
242249
assert!(!self.index.contains_key(&source));
243250

244251
let start_index = self.targets.len();
245-
self.targets.extend(targets);
252+
let (targets_size_hint, targets_size_hint_max) = targets.size_hint();
253+
debug_assert_eq!(targets_size_hint_max, Some(targets_size_hint));
254+
let new_items_count = targets_size_hint;
255+
let new_items_count_total = new_items_count + self.targets.len();
256+
257+
self.targets.reserve(new_items_count);
258+
self.inlines.grow(new_items_count_total);
259+
260+
for (i, (target, inline)) in targets.enumerate() {
261+
self.targets.push(target);
262+
if inline {
263+
self.inlines.insert(i + start_index);
264+
}
265+
}
266+
246267
let end_index = self.targets.len();
247268
self.index.insert(source, (start_index, end_index));
248269
}
249270

250271
// Internally iterate over all items referenced by `source` which will be
251272
// made available for inlining.
252273
pub fn with_inlining_candidates<F>(&self, source: TransItem<'tcx>, mut f: F)
253-
where F: FnMut(TransItem<'tcx>) {
254-
if let Some(&(start_index, end_index)) = self.index.get(&source)
255-
{
256-
for candidate in &self.targets[start_index .. end_index] {
257-
f(*candidate)
274+
where F: FnMut(TransItem<'tcx>)
275+
{
276+
if let Some(&(start_index, end_index)) = self.index.get(&source) {
277+
for (i, candidate) in self.targets[start_index .. end_index]
278+
.iter()
279+
.enumerate() {
280+
if self.inlines.contains(start_index + i) {
281+
f(*candidate);
282+
}
258283
}
259284
}
260285
}
286+
287+
// Internally iterate over all items and the things each accesses.
288+
pub fn iter_accesses<F>(&self, mut f: F)
289+
where F: FnMut(TransItem<'tcx>, &[TransItem<'tcx>])
290+
{
291+
for (&accessor, &(start_index, end_index)) in &self.index {
292+
f(accessor, &self.targets[start_index .. end_index])
293+
}
294+
}
261295
}
262296

263297
pub fn collect_crate_translation_items<'a, 'tcx>(scx: &SharedCrateContext<'a, 'tcx>,
@@ -340,7 +374,7 @@ fn collect_items_rec<'a, 'tcx: 'a>(scx: &SharedCrateContext<'a, 'tcx>,
340374

341375
recursion_depth_reset = None;
342376

343-
collect_neighbours(scx, instance, &mut neighbors);
377+
collect_neighbours(scx, instance, true, &mut neighbors);
344378
}
345379
TransItem::Fn(instance) => {
346380
// Sanity check whether this ended up being collected accidentally
@@ -352,14 +386,14 @@ fn collect_items_rec<'a, 'tcx: 'a>(scx: &SharedCrateContext<'a, 'tcx>,
352386
recursion_depths));
353387
check_type_length_limit(scx.tcx(), instance);
354388

355-
collect_neighbours(scx, instance, &mut neighbors);
389+
collect_neighbours(scx, instance, false, &mut neighbors);
356390
}
357391
TransItem::GlobalAsm(..) => {
358392
recursion_depth_reset = None;
359393
}
360394
}
361395

362-
record_inlining_canditates(scx.tcx(), starting_point, &neighbors[..], inlining_map);
396+
record_accesses(scx.tcx(), starting_point, &neighbors[..], inlining_map);
363397

364398
for neighbour in neighbors {
365399
collect_items_rec(scx, neighbour, visited, recursion_depths, inlining_map);
@@ -372,19 +406,20 @@ fn collect_items_rec<'a, 'tcx: 'a>(scx: &SharedCrateContext<'a, 'tcx>,
372406
debug!("END collect_items_rec({})", starting_point.to_string(scx.tcx()));
373407
}
374408

375-
fn record_inlining_canditates<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
409+
fn record_accesses<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
376410
caller: TransItem<'tcx>,
377411
callees: &[TransItem<'tcx>],
378412
inlining_map: &mut InliningMap<'tcx>) {
379413
let is_inlining_candidate = |trans_item: &TransItem<'tcx>| {
380414
trans_item.instantiation_mode(tcx) == InstantiationMode::LocalCopy
381415
};
382416

383-
let inlining_candidates = callees.into_iter()
384-
.map(|x| *x)
385-
.filter(is_inlining_candidate);
417+
let accesses = callees.into_iter()
418+
.map(|trans_item| {
419+
(*trans_item, is_inlining_candidate(trans_item))
420+
});
386421

387-
inlining_map.record_inlining_canditates(caller, inlining_candidates);
422+
inlining_map.record_accesses(caller, accesses);
388423
}
389424

390425
fn check_recursion_limit<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
@@ -459,7 +494,8 @@ struct MirNeighborCollector<'a, 'tcx: 'a> {
459494
scx: &'a SharedCrateContext<'a, 'tcx>,
460495
mir: &'a mir::Mir<'tcx>,
461496
output: &'a mut Vec<TransItem<'tcx>>,
462-
param_substs: &'tcx Substs<'tcx>
497+
param_substs: &'tcx Substs<'tcx>,
498+
const_context: bool,
463499
}
464500

465501
impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> {
@@ -540,7 +576,7 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> {
540576
let substs = self.scx.tcx().trans_apply_param_substs(self.param_substs,
541577
&substs);
542578
let instance = monomorphize::resolve(self.scx, def_id, substs);
543-
collect_neighbours(self.scx, instance, self.output);
579+
collect_neighbours(self.scx, instance, true, self.output);
544580
}
545581

546582
self.super_constant(constant, location);
@@ -556,8 +592,16 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> {
556592
match *kind {
557593
mir::TerminatorKind::Call { ref func, .. } => {
558594
let callee_ty = func.ty(self.mir, tcx);
559-
let callee_ty = tcx.trans_apply_param_substs(self.param_substs, &callee_ty);
560-
visit_fn_use(self.scx, callee_ty, true, &mut self.output);
595+
596+
let skip_const = self.const_context && match callee_ty.sty {
597+
ty::TyFnDef(def_id, _) => self.scx.tcx().is_const_fn(def_id),
598+
_ => false
599+
};
600+
601+
if !skip_const {
602+
let callee_ty = tcx.trans_apply_param_substs(self.param_substs, &callee_ty);
603+
visit_fn_use(self.scx, callee_ty, true, &mut self.output);
604+
}
561605
}
562606
mir::TerminatorKind::Drop { ref location, .. } |
563607
mir::TerminatorKind::DropAndReplace { ref location, .. } => {
@@ -576,6 +620,22 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> {
576620

577621
self.super_terminator_kind(block, kind, location);
578622
}
623+
624+
fn visit_static(&mut self,
625+
static_: &mir::Static<'tcx>,
626+
context: mir::visit::LvalueContext<'tcx>,
627+
location: Location) {
628+
debug!("visiting static {:?} @ {:?}", static_.def_id, location);
629+
630+
let tcx = self.scx.tcx();
631+
let instance = Instance::mono(tcx, static_.def_id);
632+
if should_trans_locally(tcx, &instance) {
633+
let node_id = tcx.hir.as_local_node_id(static_.def_id).unwrap();
634+
self.output.push(TransItem::Static(node_id));
635+
}
636+
637+
self.super_static(static_, context, location);
638+
}
579639
}
580640

581641
fn visit_drop_use<'a, 'tcx>(scx: &SharedCrateContext<'a, 'tcx>,
@@ -850,8 +910,14 @@ impl<'b, 'a, 'v> ItemLikeVisitor<'v> for RootCollector<'b, 'a, 'v> {
850910
// const items only generate translation items if they are
851911
// actually used somewhere. Just declaring them is insufficient.
852912
}
853-
hir::ItemFn(.., ref generics, _) => {
854-
if !generics.is_type_parameterized() {
913+
hir::ItemFn(_, _, constness, _, ref generics, _) => {
914+
let is_const = match constness {
915+
hir::Constness::Const => true,
916+
hir::Constness::NotConst => false,
917+
};
918+
919+
if !generics.is_type_parameterized() &&
920+
(!is_const || self.mode == TransItemCollectionMode::Eager) {
855921
let def_id = self.scx.tcx().hir.local_def_id(item.id);
856922

857923
debug!("RootCollector: ItemFn({})",
@@ -872,12 +938,13 @@ impl<'b, 'a, 'v> ItemLikeVisitor<'v> for RootCollector<'b, 'a, 'v> {
872938
fn visit_impl_item(&mut self, ii: &'v hir::ImplItem) {
873939
match ii.node {
874940
hir::ImplItemKind::Method(hir::MethodSig {
941+
constness,
875942
ref generics,
876943
..
877944
}, _) => {
878945
let hir_map = &self.scx.tcx().hir;
879946
let parent_node_id = hir_map.get_parent_node(ii.id);
880-
let is_impl_generic = match hir_map.expect_item(parent_node_id) {
947+
let is_impl_generic = || match hir_map.expect_item(parent_node_id) {
881948
&hir::Item {
882949
node: hir::ItemImpl(_, _, _, ref generics, ..),
883950
..
@@ -889,7 +956,14 @@ impl<'b, 'a, 'v> ItemLikeVisitor<'v> for RootCollector<'b, 'a, 'v> {
889956
}
890957
};
891958

892-
if !generics.is_type_parameterized() && !is_impl_generic {
959+
let is_const = match constness {
960+
hir::Constness::Const => true,
961+
hir::Constness::NotConst => false,
962+
};
963+
964+
if (!is_const || self.mode == TransItemCollectionMode::Eager) &&
965+
!generics.is_type_parameterized() &&
966+
!is_impl_generic() {
893967
let def_id = self.scx.tcx().hir.local_def_id(ii.id);
894968

895969
debug!("RootCollector: MethodImplItem({})",
@@ -958,6 +1032,7 @@ fn create_trans_items_for_default_impls<'a, 'tcx>(scx: &SharedCrateContext<'a, '
9581032
/// Scan the MIR in order to find function calls, closures, and drop-glue
9591033
fn collect_neighbours<'a, 'tcx>(scx: &SharedCrateContext<'a, 'tcx>,
9601034
instance: Instance<'tcx>,
1035+
const_context: bool,
9611036
output: &mut Vec<TransItem<'tcx>>)
9621037
{
9631038
let mir = scx.tcx().instance_mir(instance.def);
@@ -966,7 +1041,8 @@ fn collect_neighbours<'a, 'tcx>(scx: &SharedCrateContext<'a, 'tcx>,
9661041
scx: scx,
9671042
mir: &mir,
9681043
output: output,
969-
param_substs: instance.substs
1044+
param_substs: instance.substs,
1045+
const_context,
9701046
};
9711047

9721048
visitor.visit_mir(&mir);

0 commit comments

Comments
 (0)