Implemented first steps of a new, leaner hygiene data encoding scheme.

Author: ibabushkin

src/librustc_metadata/decoder.rs

@@ -244,7 +244,7 @@ impl<'a, 'tcx> SpecializedDecoder<hygiene::Mark> for DecodeContext<'a, 'tcx> {
         if !self.cdata().hygiene_data_being_decoded.get() && mark != 0  {
             let imported_hygiene = self.cdata().imported_hygiene_data();
 
-            Ok(hygiene::Mark::from_u32(mark + imported_hygiene.mark_translation_offset))
+            Ok(imported_hygiene.translate_mark(hygiene::Mark::from_u32(mark)))
         } else {
             Ok(hygiene::Mark::from_u32(mark))
         }
@@ -258,7 +258,7 @@ impl<'a, 'tcx> SpecializedDecoder<SyntaxContext> for DecodeContext<'a, 'tcx> {
         if !self.cdata().hygiene_data_being_decoded.get() && ctxt != 0 {
             let imported_hygiene = self.cdata().imported_hygiene_data();
 
-            Ok(SyntaxContext::from_u32(ctxt + imported_hygiene.ctxt_translation_offset))
+            Ok(imported_hygiene.translate_ctxt(SyntaxContext::from_u32(ctxt)))
         } else {
             Ok(SyntaxContext::from_u32(ctxt))
         }

src/librustc_metadata/encoder.rs

@@ -29,7 +29,7 @@ use rustc::ty::{self, Ty, TyCtxt, ReprOptions};
 use rustc::session::config::{self, CrateTypeProcMacro};
 use rustc::util::nodemap::{FxHashMap, NodeSet};
 
-use rustc_serialize::{Encodable, Encoder, SpecializedEncoder, opaque};
+use rustc_serialize::{Encodable, Encoder, SpecializedEncoder, UseSpecializedEncodable, opaque};
 
 use std::hash::Hash;
 use std::intrinsics;
@@ -61,6 +61,13 @@ pub struct EncodeContext<'a, 'tcx: 'a> {
 
     pub metadata_hashes: EncodedMetadataHashes,
     pub compute_ich: bool,
+    // We need to encode hygiene info after all spans and possibly other data structures
+    // that reference it have been encoded already, since we only encode those elements of it
+    // that are actually used to avoid excessive memory usage. Thus, we need to keep track of
+    // whether we already encoded the hygiene info (and thus committed to a specific set of
+    // information to encode) to make sure we can catch bugs introduced by further changes
+    // quickly.
+    already_encoded_hygiene_data: bool,
 }
 
 macro_rules! encoder_methods {
@@ -136,6 +143,26 @@ impl<'a, 'tcx> SpecializedEncoder<ty::GenericPredicates<'tcx>> for EncodeContext
     }
 }
 
+impl<'a, 'tcx> SpecializedEncoder<hygiene::SyntaxContext> for EncodeContext<'a, 'tcx> {
+    fn specialized_encode(&mut self, ctxt: &hygiene::SyntaxContext) -> Result<(), Self::Error> {
+        if self.already_encoded_hygiene_data {
+            bug!("trying to encode syntax context `{:?}` after encoding hygiene data!", ctxt);
+        } else {
+            ctxt.default_encode(self)
+        }
+    }
+}
+
+impl<'a, 'tcx> SpecializedEncoder<hygiene::Mark> for EncodeContext<'a, 'tcx> {
+    fn specialized_encode(&mut self, mark: &hygiene::Mark) -> Result<(), Self::Error> {
+        if self.already_encoded_hygiene_data {
+            bug!("trying to encode mark `{:?}` after encoding hygiene data!", mark);
+        } else {
+            mark.default_encode(self)
+        }
+    }
+}
+
 impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
 
     pub fn position(&self) -> usize {
@@ -323,9 +350,9 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         self.lazy_seq_ref(adapted.iter().map(|rc| &**rc))
     }
 
-    fn encode_hygiene_data(&mut self) -> Lazy<hygiene::HygieneData> {
-        self.tcx.sess.cstore.ensure_hygiene_data_loaded();
-        hygiene::HygieneData::safe_with(|data| self.lazy(data))
+    fn encode_hygiene_data(&mut self) -> Lazy<hygiene::HygieneDataMap> {
+        // TODO(twk): remove the `ensure_hygiene_data_loaded` method!
+        hygiene::HygieneData::safe_with(|data| self.lazy(&data.to_map()))
     }
 
     fn encode_crate_root(&mut self) -> Lazy<CrateRoot> {
@@ -372,11 +399,6 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         let codemap = self.encode_codemap();
         let codemap_bytes = self.position() - i;
 
-        // Encode hygiene data
-        i = self.position();
-        let hygiene_data = self.encode_hygiene_data();
-        let hygiene_data_bytes = self.position() - i;
-
         // Encode DefPathTable
         i = self.position();
         let def_path_table = self.encode_def_path_table();
@@ -407,6 +429,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         let index = items.write_index(&mut self.opaque.cursor);
         let index_bytes = self.position() - i;
 
+        // Encode hygiene data
+        i = self.position();
+        let hygiene_data = self.encode_hygiene_data();
+        let hygiene_data_bytes = self.position() - i;
+        self.already_encoded_hygiene_data = true;
+
         let tcx = self.tcx;
         let link_meta = self.link_meta;
         let is_proc_macro = tcx.sess.crate_types.borrow().contains(&CrateTypeProcMacro);
@@ -437,11 +465,11 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             lang_items_missing,
             native_libraries,
             codemap,
-            hygiene_data,
             def_path_table,
             impls,
             exported_symbols,
             index,
+            hygiene_data,
         });
 
         let total_bytes = self.position();
@@ -466,10 +494,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             println!("         codemap bytes: {}", codemap_bytes);
             println!("            impl bytes: {}", impl_bytes);
             println!("    exp. symbols bytes: {}", exported_symbols_bytes);
-            println!("    hygiene data bytes: {}", hygiene_data_bytes);
             println!("  def-path table bytes: {}", def_path_table_bytes);
             println!("            item bytes: {}", item_bytes);
             println!("           index bytes: {}", index_bytes);
+            println!("    hygiene data bytes: {}", hygiene_data_bytes);
             println!("            zero bytes: {}", zero_bytes);
             println!("           total bytes: {}", total_bytes);
         }
@@ -1689,6 +1717,7 @@ pub fn encode_metadata<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
             predicate_shorthands: Default::default(),
             metadata_hashes: EncodedMetadataHashes::new(),
             compute_ich,
+            already_encoded_hygiene_data: false,
         };
 
         // Encode the rustc version string in a predictable location.

src/librustc_metadata/schema.rs

@@ -204,11 +204,11 @@ pub struct CrateRoot {
     pub lang_items_missing: LazySeq<lang_items::LangItem>,
     pub native_libraries: LazySeq<NativeLibrary>,
     pub codemap: LazySeq<syntax_pos::FileMap>,
-    pub hygiene_data: Lazy<hygiene::HygieneData>,
     pub def_path_table: Lazy<hir::map::definitions::DefPathTable>,
     pub impls: LazySeq<TraitImpls>,
     pub exported_symbols: LazySeq<DefIndex>,
     pub index: LazySeq<index::Index>,
+    pub hygiene_data: Lazy<hygiene::HygieneDataMap>,
 }
 
 #[derive(RustcEncodable, RustcDecodable)]