rustc: Don't use DelimToken::None if possible

This commit fixes a regression from #44601 where lowering attribute to HIR now
involves expanding interpolated tokens to their actual tokens. In that commit
all interpolated tokens were surrounded with a `DelimToken::None` group of
tokens, but this ended up causing regressions like #44730 where the various
attribute parsers in `syntax/attr.rs` weren't ready to cope with
`DelimToken::None`. Instead of fixing the parser in `attr.rs` this commit
instead opts to just avoid the `DelimToken::None` in the first place, ensuring
that the token stream should look the same as it did before where possible.

Closes #44730

Author: alexcrichton

src/librustc/hir/lowering.rs

@@ -65,7 +65,7 @@ use syntax::codemap::{self, respan, Spanned, CompilerDesugaringKind};
 use syntax::std_inject;
 use syntax::symbol::{Symbol, keywords};
 use syntax::tokenstream::{TokenStream, TokenTree, Delimited};
-use syntax::parse::token::{Token, DelimToken};
+use syntax::parse::token::Token;
 use syntax::util::small_vector::SmallVector;
 use syntax::visit::{self, Visitor};
 use syntax_pos::Span;
@@ -606,10 +606,12 @@ impl<'a> LoweringContext<'a> {
     }
 
     fn lower_token_stream(&mut self, tokens: TokenStream) -> TokenStream {
-        tokens.into_trees().map(|tree| self.lower_token_tree(tree)).collect()
+        tokens.into_trees()
+            .flat_map(|tree| self.lower_token_tree(tree).into_trees())
+            .collect()
     }
 
-    fn lower_token_tree(&mut self, tree: TokenTree) -> TokenTree {
+    fn lower_token_tree(&mut self, tree: TokenTree) -> TokenStream {
         match tree {
             TokenTree::Token(span, token) => {
                 self.lower_token(token, span)
@@ -618,23 +620,19 @@ impl<'a> LoweringContext<'a> {
                 TokenTree::Delimited(span, Delimited {
                     delim: delimited.delim,
                     tts: self.lower_token_stream(delimited.tts.into()).into(),
-                })
+                }).into()
             }
         }
     }
 
-    fn lower_token(&mut self, token: Token, span: Span) -> TokenTree {
+    fn lower_token(&mut self, token: Token, span: Span) -> TokenStream {
         match token {
             Token::Interpolated(_) => {}
-            other => return TokenTree::Token(span, other),
+            other => return TokenTree::Token(span, other).into(),
         }
 
         let tts = token.interpolated_to_tokenstream(&self.sess.parse_sess, span);
-        let tts = self.lower_token_stream(tts);
-        TokenTree::Delimited(span, Delimited {
-            delim: DelimToken::NoDelim,
-            tts: tts.into(),
-        })
+        self.lower_token_stream(tts)
     }
 
     fn lower_arm(&mut self, arm: &Arm) -> hir::Arm {

src/test/run-pass/issue-44730.rs

@@ -0,0 +1,24 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! dox
+
+#![deny(missing_docs)]
+
+macro_rules! doc {
+    ($e:expr) => (
+        #[doc = $e]
+        pub struct Foo;
+    )
+}
+
+doc!("a");
+
+fn main() {}