auto merge of #8420 : blake2-ppc/rust/shrink-token, r=cmr

`enum Token` was 192 bytes (64-bit), as pointed out by pnkfelix; the only
bloating variant being `INTERPOLATED(nonterminal)`.

Updating `enum nonterminal` to use ~ where variants included big types,
shrunk size_of(Token) to 32 bytes (64-bit).

I am unsure if the `nt_ident` variant should have an indirection, with
ast::ident being only 16 bytes (64-bit), but without this, enum Token
would be 40 bytes.

A dumb benchmark says that compilation time is unchanged, while peak
memory usage for compiling std.rs is down 3%

Before::

    $ time ./x86_64-unknown-linux-gnu/stage1/bin/rustc --cfg stage1 src/libstd/std.rs
    19.00user 0.39system 0:19.41elapsed 99%CPU (0avgtext+0avgdata 627820maxresident)k
    0inputs+28896outputs (0major+228665minor)pagefaults 0swaps
    $ time ./x86_64-unknown-linux-gnu/stage1/bin/rustc -O --cfg stage1 src/libstd/std.rs
    31.64user 0.34system 0:32.02elapsed 99%CPU (0avgtext+0avgdata 629876maxresident)k
    0inputs+22432outputs (0major+229411minor)pagefaults 0swaps

After::

    $ time ./x86_64-unknown-linux-gnu/stage1/bin/rustc --cfg stage1 src/libstd/std.rs
    19.07user 0.45system 0:19.55elapsed 99%CPU (0avgtext+0avgdata 609384maxresident)k
    0inputs+28896outputs (0major+221997minor)pagefaults 0swaps

    $ time ./x86_64-unknown-linux-gnu/stage1/bin/rustc -O --cfg stage1 src/libstd/std.rs
    31.90user 0.34system 0:32.28elapsed 99%CPU (0avgtext+0avgdata 612080maxresident)k
    0inputs+22432outputs (0major+223726minor)pagefaults 0swaps

Author: bors

src/libsyntax/ext/tt/macro_parser.rs

@@ -419,18 +419,18 @@ pub fn parse_nt(p: &Parser, name: &str) -> nonterminal {
         Some(i) => token::nt_item(i),
         None => p.fatal("expected an item keyword")
       },
-      "block" => token::nt_block(p.parse_block()),
+      "block" => token::nt_block(~p.parse_block()),
       "stmt" => token::nt_stmt(p.parse_stmt(~[])),
       "pat" => token::nt_pat(p.parse_pat()),
       "expr" => token::nt_expr(p.parse_expr()),
-      "ty" => token::nt_ty(p.parse_ty(false /* no need to disambiguate*/)),
+      "ty" => token::nt_ty(~p.parse_ty(false /* no need to disambiguate*/)),
       // this could be handled like a token, since it is one
       "ident" => match *p.token {
-        token::IDENT(sn,b) => { p.bump(); token::nt_ident(sn,b) }
+        token::IDENT(sn,b) => { p.bump(); token::nt_ident(~sn,b) }
         _ => p.fatal(~"expected ident, found "
                      + token::to_str(get_ident_interner(), p.token))
       },
-      "path" => token::nt_path(p.parse_path_with_tps(false)),
+      "path" => token::nt_path(~p.parse_path_with_tps(false)),
       "attr" => token::nt_attr(@p.parse_attribute(false)),
       "tt" => {
         *p.quote_depth += 1u; //but in theory, non-quoted tts might be useful

src/libsyntax/ext/tt/transcribe.rs

@@ -279,7 +279,7 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
               /* sidestep the interpolation tricks for ident because
               (a) idents can be in lots of places, so it'd be a pain
               (b) we actually can, since it's a token. */
-              matched_nonterminal(nt_ident(sn,b)) => {
+              matched_nonterminal(nt_ident(~sn,b)) => {
                 r.cur_span = sp; r.cur_tok = IDENT(sn,b);
                 r.stack.idx += 1u;
                 return ret_val;

src/libsyntax/parse/parser.rs

@@ -146,7 +146,7 @@ macro_rules! maybe_whole_expr (
                     Some($p.mk_expr(
                         ($p).span.lo,
                         ($p).span.hi,
-                        expr_path(/* bad */ (*pt).clone())))
+                        expr_path(/* bad */ (**pt).clone())))
                 }
                 _ => None
             };
@@ -235,8 +235,8 @@ macro_rules! maybe_whole (
                 _ => None
             };
             match __found__ {
-                Some(INTERPOLATED(token::$constructor(x))) => {
-                    return (~[], x.clone())
+                Some(INTERPOLATED(token::$constructor(ref x))) => {
+                    return (~[], (**x).clone())
                 }
                 _ => {}
             }
@@ -939,7 +939,7 @@ impl Parser {
     // Useless second parameter for compatibility with quasiquote macros.
     // Bleh!
     pub fn parse_ty(&self, _: bool) -> Ty {
-        maybe_whole!(self, nt_ty);
+        maybe_whole!(deref self, nt_ty);
 
         let lo = self.span.lo;
 
@@ -1293,7 +1293,7 @@ impl Parser {
 
     // parse a path that doesn't have type parameters attached
     pub fn parse_path_without_tps(&self) -> ast::Path {
-        maybe_whole!(self, nt_path);
+        maybe_whole!(deref self, nt_path);
         let (ids,is_global,sp) = self.parse_path();
         ast::Path { span: sp,
                      global: is_global,
@@ -1306,7 +1306,7 @@ impl Parser {
                                         before_tps: Option<&fn()>) -> ast::Path {
         debug!("parse_path_with_tps(colons=%b)", colons);
 
-        maybe_whole!(self, nt_path);
+        maybe_whole!(deref self, nt_path);
         let lo = self.span.lo;
         let path = self.parse_path_without_tps();
         if colons && !self.eat(&token::MOD_SEP) {
@@ -3100,7 +3100,7 @@ impl Parser {
 
     // parse a block. No inner attrs are allowed.
     pub fn parse_block(&self) -> Block {
-        maybe_whole!(self, nt_block);
+        maybe_whole!(deref self, nt_block);
 
         let lo = self.span.lo;
         if self.eat_keyword(keywords::Unsafe) {

src/libsyntax/parse/token.rs

@@ -99,14 +99,14 @@ pub enum Token {
 /// For interpolation during macro expansion.
 pub enum nonterminal {
     nt_item(@ast::item),
-    nt_block(ast::Block),
+    nt_block(~ast::Block),
     nt_stmt(@ast::stmt),
     nt_pat( @ast::pat),
     nt_expr(@ast::expr),
-    nt_ty(   ast::Ty),
-    nt_ident(ast::ident, bool),
+    nt_ty(  ~ast::Ty),
+    nt_ident(~ast::ident, bool),
     nt_attr(@ast::Attribute),   // #[foo]
-    nt_path( ast::Path),
+    nt_path(~ast::Path),
     nt_tt(  @ast::token_tree), //needs @ed to break a circularity
     nt_matchers(~[ast::matcher])
 }