Skip to content

Commit df29f9b

Browse files
committed
Improve fake_ident_or_unknown_prefix.
- Rename it as `invalid_ident_or_prefix`, which matches the possible outputs (`InvalidIdent` or `InvalidPrefix`). - Use the local wrapper for `is_xid_continue`, for consistency. - Make it clear what `\u{200d}` means.
1 parent 03ee484 commit df29f9b

File tree

1 file changed

+5
-6
lines changed
  • compiler/rustc_lexer/src

1 file changed

+5
-6
lines changed

compiler/rustc_lexer/src/lib.rs

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -468,7 +468,7 @@ impl Cursor<'_> {
468468
Literal { kind, suffix_start }
469469
}
470470
// Identifier starting with an emoji. Only lexed for graceful error recovery.
471-
c if !c.is_ascii() && c.is_emoji_char() => self.fake_ident_or_unknown_prefix(),
471+
c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident_or_prefix(),
472472
_ => Unknown,
473473
};
474474
let res = Token::new(token_kind, self.pos_within_token());
@@ -552,17 +552,16 @@ impl Cursor<'_> {
552552
// we see a prefix here, it is definitely an unknown prefix.
553553
match self.first() {
554554
'#' | '"' | '\'' => UnknownPrefix,
555-
c if !c.is_ascii() && c.is_emoji_char() => self.fake_ident_or_unknown_prefix(),
555+
c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident_or_prefix(),
556556
_ => Ident,
557557
}
558558
}
559559

560-
fn fake_ident_or_unknown_prefix(&mut self) -> TokenKind {
560+
fn invalid_ident_or_prefix(&mut self) -> TokenKind {
561561
// Start is already eaten, eat the rest of identifier.
562562
self.eat_while(|c| {
563-
unicode_xid::UnicodeXID::is_xid_continue(c)
564-
|| (!c.is_ascii() && c.is_emoji_char())
565-
|| c == '\u{200d}'
563+
const ZERO_WIDTH_JOINER: char = '\u{200d}';
564+
is_id_continue(c) || (!c.is_ascii() && c.is_emoji_char()) || c == ZERO_WIDTH_JOINER
566565
});
567566
// Known prefixes must have been handled earlier. So if
568567
// we see a prefix here, it is definitely an unknown prefix.

0 commit comments

Comments
 (0)