Skip to content

Commit 665ef7c

Browse files
committed
Auto merge of rust-lang#137275 - hkBst:single_pass_skip_ws_bytes, r=<try>
Make skip_whitespace do a single pass (with bytes) Probably better alternative to rust-lang#137027. r? `@nnethercote`
2 parents eeb9035 + 258115e commit 665ef7c

File tree

1 file changed

+29
-16
lines changed

1 file changed

+29
-16
lines changed

compiler/rustc_lexer/src/unescape.rs

Lines changed: 29 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -356,8 +356,10 @@ where
356356
let start = src.len() - chars.as_str().len() - c.len_utf8();
357357
let res = match c {
358358
'\\' => {
359-
match chars.clone().next() {
359+
let mut chars_clone = chars.clone();
360+
match chars_clone.next() {
360361
Some('\n') => {
362+
chars = chars_clone;
361363
// Rust language specification requires us to skip whitespaces
362364
// if unescaped '\' character is followed by '\n'.
363365
// For details see [Rust language reference]
@@ -379,30 +381,41 @@ where
379381
}
380382
}
381383

384+
/// Skip ASCII whitespace, except for the formfeed character
385+
/// (see [this issue](https://github.com/rust-lang/rust/issues/136600)).
386+
/// Warns on unescaped newline and following non-ASCII whitespace.
382387
fn skip_ascii_whitespace<F>(chars: &mut Chars<'_>, start: usize, callback: &mut F)
383388
where
384389
F: FnMut(Range<usize>, EscapeError),
385390
{
386-
let tail = chars.as_str();
387-
let first_non_space = tail
388-
.bytes()
389-
.position(|b| b != b' ' && b != b'\t' && b != b'\n' && b != b'\r')
390-
.unwrap_or(tail.len());
391-
if tail[1..first_non_space].contains('\n') {
392-
// The +1 accounts for the escaping slash.
393-
let end = start + first_non_space + 1;
391+
let mut spaces = 0;
392+
let mut contains_nl = false;
393+
394+
for byte in chars.as_str().bytes() {
395+
let is_space = matches!(byte, b' ' | b'\t' | b'\r' | b'\n');
396+
let is_newline = byte == b'\n';
397+
398+
spaces += is_space as usize;
399+
contains_nl |= is_newline;
400+
401+
if !is_space {
402+
break;
403+
}
404+
}
405+
*chars = chars.as_str()[spaces..].chars();
406+
407+
// the escaping slash and newline characters add 2 bytes
408+
let end = start + 2 + spaces;
409+
410+
if contains_nl {
394411
callback(start..end, EscapeError::MultipleSkippedLinesWarning);
395412
}
396-
let tail = &tail[first_non_space..];
397-
if let Some(c) = tail.chars().next() {
413+
if let Some(c) = chars.clone().next() {
398414
if c.is_whitespace() {
399-
// For error reporting, we would like the span to contain the character that was not
400-
// skipped. The +1 is necessary to account for the leading \ that started the escape.
401-
let end = start + first_non_space + c.len_utf8() + 1;
402-
callback(start..end, EscapeError::UnskippedWhitespaceWarning);
415+
// for error reporting, include the character that was not skipped in the span
416+
callback(start..end + c.len_utf8(), EscapeError::UnskippedWhitespaceWarning);
403417
}
404418
}
405-
*chars = tail.chars();
406419
}
407420

408421
/// Takes a contents of a string literal (without quotes) and produces a

0 commit comments

Comments
 (0)