@@ -208,6 +208,8 @@ use rustc::util::nodemap::{FxHashSet, FxHashMap, DefIdMap};
208
208
209
209
use trans_item:: { TransItem , DefPathBasedNames , InstantiationMode } ;
210
210
211
+ use rustc_data_structures:: bitvec:: BitVector ;
212
+
211
213
#[ derive( PartialEq , Eq , Hash , Clone , Copy , Debug ) ]
212
214
pub enum TransItemCollectionMode {
213
215
Eager ,
@@ -217,12 +219,16 @@ pub enum TransItemCollectionMode {
217
219
/// Maps every translation item to all translation items it references in its
218
220
/// body.
219
221
pub struct InliningMap < ' tcx > {
220
- // Maps a source translation item to a range of target translation items
221
- // that are potentially inlined by LLVM into the source .
222
+ // Maps a source translation item to the range of translation items
223
+ // accessed by it .
222
224
// The two numbers in the tuple are the start (inclusive) and
223
225
// end index (exclusive) within the `targets` vecs.
224
226
index : FxHashMap < TransItem < ' tcx > , ( usize , usize ) > ,
225
227
targets : Vec < TransItem < ' tcx > > ,
228
+
229
+ // Contains one bit per translation item in the `targets` field. That bit
230
+ // is true if that translation item needs to be inlined into every CGU.
231
+ inlines : BitVector ,
226
232
}
227
233
228
234
impl < ' tcx > InliningMap < ' tcx > {
@@ -231,33 +237,61 @@ impl<'tcx> InliningMap<'tcx> {
231
237
InliningMap {
232
238
index : FxHashMap ( ) ,
233
239
targets : Vec :: new ( ) ,
240
+ inlines : BitVector :: new ( 1024 ) ,
234
241
}
235
242
}
236
243
237
- fn record_inlining_canditates < I > ( & mut self ,
238
- source : TransItem < ' tcx > ,
239
- targets : I )
240
- where I : Iterator < Item =TransItem < ' tcx > >
244
+ fn record_accesses < I > ( & mut self ,
245
+ source : TransItem < ' tcx > ,
246
+ targets : I )
247
+ where I : Iterator < Item =( TransItem < ' tcx > , bool ) >
241
248
{
242
249
assert ! ( !self . index. contains_key( & source) ) ;
243
250
244
251
let start_index = self . targets . len ( ) ;
245
- self . targets . extend ( targets) ;
252
+ let ( targets_size_hint, targets_size_hint_max) = targets. size_hint ( ) ;
253
+ debug_assert_eq ! ( targets_size_hint_max, Some ( targets_size_hint) ) ;
254
+ let new_items_count = targets_size_hint;
255
+ let new_items_count_total = new_items_count + self . targets . len ( ) ;
256
+
257
+ self . targets . reserve ( new_items_count) ;
258
+ self . inlines . grow ( new_items_count_total) ;
259
+
260
+ for ( i, ( target, inline) ) in targets. enumerate ( ) {
261
+ self . targets . push ( target) ;
262
+ if inline {
263
+ self . inlines . insert ( i + start_index) ;
264
+ }
265
+ }
266
+
246
267
let end_index = self . targets . len ( ) ;
247
268
self . index . insert ( source, ( start_index, end_index) ) ;
248
269
}
249
270
250
271
// Internally iterate over all items referenced by `source` which will be
251
272
// made available for inlining.
252
273
pub fn with_inlining_candidates < F > ( & self , source : TransItem < ' tcx > , mut f : F )
253
- where F : FnMut ( TransItem < ' tcx > ) {
254
- if let Some ( & ( start_index, end_index) ) = self . index . get ( & source)
255
- {
256
- for candidate in & self . targets [ start_index .. end_index] {
257
- f ( * candidate)
274
+ where F : FnMut ( TransItem < ' tcx > )
275
+ {
276
+ if let Some ( & ( start_index, end_index) ) = self . index . get ( & source) {
277
+ for ( i, candidate) in self . targets [ start_index .. end_index]
278
+ . iter ( )
279
+ . enumerate ( ) {
280
+ if self . inlines . contains ( start_index + i) {
281
+ f ( * candidate) ;
282
+ }
258
283
}
259
284
}
260
285
}
286
+
287
+ // Internally iterate over all items and the things each accesses.
288
+ pub fn iter_accesses < F > ( & self , mut f : F )
289
+ where F : FnMut ( TransItem < ' tcx > , & [ TransItem < ' tcx > ] )
290
+ {
291
+ for ( & accessor, & ( start_index, end_index) ) in & self . index {
292
+ f ( accessor, & self . targets [ start_index .. end_index] )
293
+ }
294
+ }
261
295
}
262
296
263
297
pub fn collect_crate_translation_items < ' a , ' tcx > ( scx : & SharedCrateContext < ' a , ' tcx > ,
@@ -340,7 +374,7 @@ fn collect_items_rec<'a, 'tcx: 'a>(scx: &SharedCrateContext<'a, 'tcx>,
340
374
341
375
recursion_depth_reset = None ;
342
376
343
- collect_neighbours ( scx, instance, & mut neighbors) ;
377
+ collect_neighbours ( scx, instance, true , & mut neighbors) ;
344
378
}
345
379
TransItem :: Fn ( instance) => {
346
380
// Sanity check whether this ended up being collected accidentally
@@ -352,14 +386,14 @@ fn collect_items_rec<'a, 'tcx: 'a>(scx: &SharedCrateContext<'a, 'tcx>,
352
386
recursion_depths) ) ;
353
387
check_type_length_limit ( scx. tcx ( ) , instance) ;
354
388
355
- collect_neighbours ( scx, instance, & mut neighbors) ;
389
+ collect_neighbours ( scx, instance, false , & mut neighbors) ;
356
390
}
357
391
TransItem :: GlobalAsm ( ..) => {
358
392
recursion_depth_reset = None ;
359
393
}
360
394
}
361
395
362
- record_inlining_canditates ( scx. tcx ( ) , starting_point, & neighbors[ ..] , inlining_map) ;
396
+ record_accesses ( scx. tcx ( ) , starting_point, & neighbors[ ..] , inlining_map) ;
363
397
364
398
for neighbour in neighbors {
365
399
collect_items_rec ( scx, neighbour, visited, recursion_depths, inlining_map) ;
@@ -372,19 +406,20 @@ fn collect_items_rec<'a, 'tcx: 'a>(scx: &SharedCrateContext<'a, 'tcx>,
372
406
debug ! ( "END collect_items_rec({})" , starting_point. to_string( scx. tcx( ) ) ) ;
373
407
}
374
408
375
- fn record_inlining_canditates < ' a , ' tcx > ( tcx : TyCtxt < ' a , ' tcx , ' tcx > ,
409
+ fn record_accesses < ' a , ' tcx > ( tcx : TyCtxt < ' a , ' tcx , ' tcx > ,
376
410
caller : TransItem < ' tcx > ,
377
411
callees : & [ TransItem < ' tcx > ] ,
378
412
inlining_map : & mut InliningMap < ' tcx > ) {
379
413
let is_inlining_candidate = |trans_item : & TransItem < ' tcx > | {
380
414
trans_item. instantiation_mode ( tcx) == InstantiationMode :: LocalCopy
381
415
} ;
382
416
383
- let inlining_candidates = callees. into_iter ( )
384
- . map ( |x| * x)
385
- . filter ( is_inlining_candidate) ;
417
+ let accesses = callees. into_iter ( )
418
+ . map ( |trans_item| {
419
+ ( * trans_item, is_inlining_candidate ( trans_item) )
420
+ } ) ;
386
421
387
- inlining_map. record_inlining_canditates ( caller, inlining_candidates ) ;
422
+ inlining_map. record_accesses ( caller, accesses ) ;
388
423
}
389
424
390
425
fn check_recursion_limit < ' a , ' tcx > ( tcx : TyCtxt < ' a , ' tcx , ' tcx > ,
@@ -459,7 +494,8 @@ struct MirNeighborCollector<'a, 'tcx: 'a> {
459
494
scx : & ' a SharedCrateContext < ' a , ' tcx > ,
460
495
mir : & ' a mir:: Mir < ' tcx > ,
461
496
output : & ' a mut Vec < TransItem < ' tcx > > ,
462
- param_substs : & ' tcx Substs < ' tcx >
497
+ param_substs : & ' tcx Substs < ' tcx > ,
498
+ const_context : bool ,
463
499
}
464
500
465
501
impl < ' a , ' tcx > MirVisitor < ' tcx > for MirNeighborCollector < ' a , ' tcx > {
@@ -540,7 +576,7 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> {
540
576
let substs = self . scx . tcx ( ) . trans_apply_param_substs ( self . param_substs ,
541
577
& substs) ;
542
578
let instance = monomorphize:: resolve ( self . scx , def_id, substs) ;
543
- collect_neighbours ( self . scx , instance, self . output ) ;
579
+ collect_neighbours ( self . scx , instance, true , self . output ) ;
544
580
}
545
581
546
582
self . super_constant ( constant, location) ;
@@ -556,8 +592,16 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> {
556
592
match * kind {
557
593
mir:: TerminatorKind :: Call { ref func, .. } => {
558
594
let callee_ty = func. ty ( self . mir , tcx) ;
559
- let callee_ty = tcx. trans_apply_param_substs ( self . param_substs , & callee_ty) ;
560
- visit_fn_use ( self . scx , callee_ty, true , & mut self . output ) ;
595
+
596
+ let skip_const = self . const_context && match callee_ty. sty {
597
+ ty:: TyFnDef ( def_id, _) => self . scx . tcx ( ) . is_const_fn ( def_id) ,
598
+ _ => false
599
+ } ;
600
+
601
+ if !skip_const {
602
+ let callee_ty = tcx. trans_apply_param_substs ( self . param_substs , & callee_ty) ;
603
+ visit_fn_use ( self . scx , callee_ty, true , & mut self . output ) ;
604
+ }
561
605
}
562
606
mir:: TerminatorKind :: Drop { ref location, .. } |
563
607
mir:: TerminatorKind :: DropAndReplace { ref location, .. } => {
@@ -576,6 +620,22 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> {
576
620
577
621
self . super_terminator_kind ( block, kind, location) ;
578
622
}
623
+
624
+ fn visit_static ( & mut self ,
625
+ static_ : & mir:: Static < ' tcx > ,
626
+ context : mir:: visit:: LvalueContext < ' tcx > ,
627
+ location : Location ) {
628
+ debug ! ( "visiting static {:?} @ {:?}" , static_. def_id, location) ;
629
+
630
+ let tcx = self . scx . tcx ( ) ;
631
+ let instance = Instance :: mono ( tcx, static_. def_id ) ;
632
+ if should_trans_locally ( tcx, & instance) {
633
+ let node_id = tcx. hir . as_local_node_id ( static_. def_id ) . unwrap ( ) ;
634
+ self . output . push ( TransItem :: Static ( node_id) ) ;
635
+ }
636
+
637
+ self . super_static ( static_, context, location) ;
638
+ }
579
639
}
580
640
581
641
fn visit_drop_use < ' a , ' tcx > ( scx : & SharedCrateContext < ' a , ' tcx > ,
@@ -850,8 +910,14 @@ impl<'b, 'a, 'v> ItemLikeVisitor<'v> for RootCollector<'b, 'a, 'v> {
850
910
// const items only generate translation items if they are
851
911
// actually used somewhere. Just declaring them is insufficient.
852
912
}
853
- hir:: ItemFn ( .., ref generics, _) => {
854
- if !generics. is_type_parameterized ( ) {
913
+ hir:: ItemFn ( _, _, constness, _, ref generics, _) => {
914
+ let is_const = match constness {
915
+ hir:: Constness :: Const => true ,
916
+ hir:: Constness :: NotConst => false ,
917
+ } ;
918
+
919
+ if !generics. is_type_parameterized ( ) &&
920
+ ( !is_const || self . mode == TransItemCollectionMode :: Eager ) {
855
921
let def_id = self . scx . tcx ( ) . hir . local_def_id ( item. id ) ;
856
922
857
923
debug ! ( "RootCollector: ItemFn({})" ,
@@ -872,12 +938,13 @@ impl<'b, 'a, 'v> ItemLikeVisitor<'v> for RootCollector<'b, 'a, 'v> {
872
938
fn visit_impl_item ( & mut self , ii : & ' v hir:: ImplItem ) {
873
939
match ii. node {
874
940
hir:: ImplItemKind :: Method ( hir:: MethodSig {
941
+ constness,
875
942
ref generics,
876
943
..
877
944
} , _) => {
878
945
let hir_map = & self . scx . tcx ( ) . hir ;
879
946
let parent_node_id = hir_map. get_parent_node ( ii. id ) ;
880
- let is_impl_generic = match hir_map. expect_item ( parent_node_id) {
947
+ let is_impl_generic = || match hir_map. expect_item ( parent_node_id) {
881
948
& hir:: Item {
882
949
node : hir:: ItemImpl ( _, _, _, ref generics, ..) ,
883
950
..
@@ -889,7 +956,14 @@ impl<'b, 'a, 'v> ItemLikeVisitor<'v> for RootCollector<'b, 'a, 'v> {
889
956
}
890
957
} ;
891
958
892
- if !generics. is_type_parameterized ( ) && !is_impl_generic {
959
+ let is_const = match constness {
960
+ hir:: Constness :: Const => true ,
961
+ hir:: Constness :: NotConst => false ,
962
+ } ;
963
+
964
+ if ( !is_const || self . mode == TransItemCollectionMode :: Eager ) &&
965
+ !generics. is_type_parameterized ( ) &&
966
+ !is_impl_generic ( ) {
893
967
let def_id = self . scx . tcx ( ) . hir . local_def_id ( ii. id ) ;
894
968
895
969
debug ! ( "RootCollector: MethodImplItem({})" ,
@@ -958,6 +1032,7 @@ fn create_trans_items_for_default_impls<'a, 'tcx>(scx: &SharedCrateContext<'a, '
958
1032
/// Scan the MIR in order to find function calls, closures, and drop-glue
959
1033
fn collect_neighbours < ' a , ' tcx > ( scx : & SharedCrateContext < ' a , ' tcx > ,
960
1034
instance : Instance < ' tcx > ,
1035
+ const_context : bool ,
961
1036
output : & mut Vec < TransItem < ' tcx > > )
962
1037
{
963
1038
let mir = scx. tcx ( ) . instance_mir ( instance. def ) ;
@@ -966,7 +1041,8 @@ fn collect_neighbours<'a, 'tcx>(scx: &SharedCrateContext<'a, 'tcx>,
966
1041
scx : scx,
967
1042
mir : & mir,
968
1043
output : output,
969
- param_substs : instance. substs
1044
+ param_substs : instance. substs ,
1045
+ const_context,
970
1046
} ;
971
1047
972
1048
visitor. visit_mir ( & mir) ;
0 commit comments