Merge #4925

bors[bot] · ltentrup · web-flow · commit f061b2daeb86 · 2020-06-17T15:08:17.000Z
4925: Syntax highlighting for escape sequences in strings r=matklad a=ltentrup I have added a new semantic token type `ESCAPE_SEQUENCE` as the LSP specification does not seem to have an appropriate token type. This may actually be a regression for some users, as the TextMate Rust grammar has a scope `constant.character.escape.rust` which highlights escape sequences (which caused problems with semantic highlighting, see #4138). Fixes #2604. Co-authored-by: Leander Tentrup <leander.tentrup@gmail.com>
diff --git a/crates/ra_ide/src/snapshots/highlight_doctest.html b/crates/ra_ide/src/snapshots/highlight_doctest.html
@@ -26,6 +26,7 @@
 .format_specifier   { color: #CC696B; }
 .mutable            { text-decoration: underline; }
 .unresolved_reference { color: #FC5555; }
+.escape_sequence    { color: #94BFF3; }
 
 .keyword            { color: #F0DFAF; font-weight: bold; }
 .keyword.unsafe     { color: #BC8383; font-weight: bold; }
diff --git a/crates/ra_ide/src/snapshots/highlight_injection.html b/crates/ra_ide/src/snapshots/highlight_injection.html
@@ -26,6 +26,7 @@
 .format_specifier   { color: #CC696B; }
 .mutable            { text-decoration: underline; }
 .unresolved_reference { color: #FC5555; }
+.escape_sequence    { color: #94BFF3; }
 
 .keyword            { color: #F0DFAF; font-weight: bold; }
 .keyword.unsafe     { color: #BC8383; font-weight: bold; }
diff --git a/crates/ra_ide/src/snapshots/highlight_strings.html b/crates/ra_ide/src/snapshots/highlight_strings.html
@@ -26,6 +26,7 @@
 .format_specifier   { color: #CC696B; }
 .mutable            { text-decoration: underline; }
 .unresolved_reference { color: #FC5555; }
+.escape_sequence    { color: #94BFF3; }
 
 .keyword            { color: #F0DFAF; font-weight: bold; }
 .keyword.unsafe     { color: #BC8383; font-weight: bold; }
@@ -83,6 +84,10 @@
 
     <span class="macro">println!</span>(<span class="string_literal">r"Hello, </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">!"</span>, <span class="string_literal">"world"</span>);
 
-    <span class="macro">println!</span>(<span class="string_literal">"</span><span class="format_specifier">{</span><span class="variable">\x41</span><span class="format_specifier">}</span><span class="string_literal">"</span>, A = <span class="numeric_literal">92</span>);
+    <span class="comment">// escape sequences</span>
+    <span class="macro">println!</span>(<span class="string_literal">"Hello</span><span class="escape_sequence">\n</span><span class="string_literal">World"</span>);
+    <span class="macro">println!</span>(<span class="string_literal">"</span><span class="escape_sequence">\u{48}</span><span class="escape_sequence">\x65</span><span class="escape_sequence">\x6C</span><span class="escape_sequence">\x6C</span><span class="escape_sequence">\x6F</span><span class="string_literal"> World"</span>);
+
+    <span class="macro">println!</span>(<span class="string_literal">"</span><span class="format_specifier">{</span><span class="escape_sequence">\x41</span><span class="format_specifier">}</span><span class="string_literal">"</span>, A = <span class="numeric_literal">92</span>);
     <span class="macro">println!</span>(<span class="string_literal">"</span><span class="format_specifier">{</span><span class="variable">ничоси</span><span class="format_specifier">}</span><span class="string_literal">"</span>, ничоси = <span class="numeric_literal">92</span>);
 }</code></pre>
diff --git a/crates/ra_ide/src/snapshots/highlight_unsafe.html b/crates/ra_ide/src/snapshots/highlight_unsafe.html
@@ -26,6 +26,7 @@
 .format_specifier   { color: #CC696B; }
 .mutable            { text-decoration: underline; }
 .unresolved_reference { color: #FC5555; }
+.escape_sequence    { color: #94BFF3; }
 
 .keyword            { color: #F0DFAF; font-weight: bold; }
 .keyword.unsafe     { color: #BC8383; font-weight: bold; }
diff --git a/crates/ra_ide/src/snapshots/highlighting.html b/crates/ra_ide/src/snapshots/highlighting.html
@@ -26,6 +26,7 @@
 .format_specifier   { color: #CC696B; }
 .mutable            { text-decoration: underline; }
 .unresolved_reference { color: #FC5555; }
+.escape_sequence    { color: #94BFF3; }
 
 .keyword            { color: #F0DFAF; font-weight: bold; }
 .keyword.unsafe     { color: #BC8383; font-weight: bold; }
diff --git a/crates/ra_ide/src/snapshots/rainbow_highlighting.html b/crates/ra_ide/src/snapshots/rainbow_highlighting.html
@@ -26,6 +26,7 @@
 .format_specifier   { color: #CC696B; }
 .mutable            { text-decoration: underline; }
 .unresolved_reference { color: #FC5555; }
+.escape_sequence    { color: #94BFF3; }
 
 .keyword            { color: #F0DFAF; font-weight: bold; }
 .keyword.unsafe     { color: #BC8383; font-weight: bold; }
diff --git a/crates/ra_ide/src/syntax_highlighting.rs b/crates/ra_ide/src/syntax_highlighting.rs
@@ -212,8 +212,8 @@ pub(crate) fn highlight(
             if let Some(string) =
                 element_to_highlight.as_token().cloned().and_then(ast::String::cast)
             {
-                stack.push();
                 if is_format_string {
+                    stack.push();
                     string.lex_format_specifier(|piece_range, kind| {
                         if let Some(highlight) = highlight_format_specifier(kind) {
                             stack.add(HighlightedRange {
@@ -223,13 +223,27 @@ pub(crate) fn highlight(
                             });
                         }
                     });
+                    stack.pop();
+                }
+                // Highlight escape sequences
+                if let Some(char_ranges) = string.char_ranges() {
+                    stack.push();
+                    for (piece_range, _) in char_ranges.iter().filter(|(_, char)| char.is_ok()) {
+                        if string.text()[piece_range.start().into()..].starts_with('\\') {
+                            stack.add(HighlightedRange {
+                                range: piece_range + range.start(),
+                                highlight: HighlightTag::EscapeSequence.into(),
+                                binding_hash: None,
+                            });
+                        }
+                    }
+                    stack.pop_and_inject(false);
                 }
-                stack.pop();
             } else if let Some(string) =
                 element_to_highlight.as_token().cloned().and_then(ast::RawString::cast)
             {
-                stack.push();
                 if is_format_string {
+                    stack.push();
                     string.lex_format_specifier(|piece_range, kind| {
                         if let Some(highlight) = highlight_format_specifier(kind) {
                             stack.add(HighlightedRange {
@@ -239,8 +253,8 @@ pub(crate) fn highlight(
                             });
                         }
                     });
+                    stack.pop();
                 }
-                stack.pop();
             }
         }
     }
diff --git a/crates/ra_ide/src/syntax_highlighting/html.rs b/crates/ra_ide/src/syntax_highlighting/html.rs
@@ -85,6 +85,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .format_specifier   { color: #CC696B; }
 .mutable            { text-decoration: underline; }
 .unresolved_reference { color: #FC5555; }
+.escape_sequence    { color: #94BFF3; }
 
 .keyword            { color: #F0DFAF; font-weight: bold; }
 .keyword.unsafe     { color: #BC8383; font-weight: bold; }
diff --git a/crates/ra_ide/src/syntax_highlighting/tags.rs b/crates/ra_ide/src/syntax_highlighting/tags.rs
@@ -23,6 +23,7 @@ pub enum HighlightTag {
     Constant,
     Enum,
     EnumVariant,
+    EscapeSequence,
     Field,
     FormatSpecifier,
     Function,
@@ -71,6 +72,7 @@ impl HighlightTag {
             HighlightTag::Constant => "constant",
             HighlightTag::Enum => "enum",
             HighlightTag::EnumVariant => "enum_variant",
+            HighlightTag::EscapeSequence => "escape_sequence",
             HighlightTag::Field => "field",
             HighlightTag::FormatSpecifier => "format_specifier",
             HighlightTag::Function => "function",
diff --git a/crates/ra_ide/src/syntax_highlighting/tests.rs b/crates/ra_ide/src/syntax_highlighting/tests.rs
@@ -246,6 +246,10 @@ fn main() {
 
     println!(r"Hello, {}!", "world");
 
+    // escape sequences
+    println!("Hello\nWorld");
+    println!("\u{48}\x65\x6C\x6C\x6F World");
+
     println!("{\x41}", A = 92);
     println!("{ничоси}", ничоси = 92);
 }"#
diff --git a/crates/rust-analyzer/src/semantic_tokens.rs b/crates/rust-analyzer/src/semantic_tokens.rs
@@ -45,6 +45,7 @@ define_semantic_token_types![
     (UNION, "union"),
     (UNRESOLVED_REFERENCE, "unresolvedReference"),
     (FORMAT_SPECIFIER, "formatSpecifier"),
+    (ESCAPE_SEQUENCE, "escapeSequence"),
 ];
 
 macro_rules! define_semantic_token_modifiers {
diff --git a/crates/rust-analyzer/src/to_proto.rs b/crates/rust-analyzer/src/to_proto.rs
@@ -324,6 +324,7 @@ fn semantic_token_type_and_modifiers(
         HighlightTag::UnresolvedReference => semantic_tokens::UNRESOLVED_REFERENCE,
         HighlightTag::FormatSpecifier => semantic_tokens::FORMAT_SPECIFIER,
         HighlightTag::Operator => lsp_types::SemanticTokenType::OPERATOR,
+        HighlightTag::EscapeSequence => semantic_tokens::ESCAPE_SEQUENCE,
     };
 
     for modifier in highlight.modifiers.iter() {

Original file line number	Diff line number	Diff line change
`@@ -212,8 +212,8 @@ pub(crate) fn highlight(`
`212`	`212`	`if let Some(string) =`
`213`	`213`	`element_to_highlight.as_token().cloned().and_then(ast::String::cast)`
`214`	`214`	`{`
`215`		`- stack.push();`
`216`	`215`	`if is_format_string {`
	`216`	`+ stack.push();`
`217`	`217`	`string.lex_format_specifier(\|piece_range, kind\| {`
`218`	`218`	`if let Some(highlight) = highlight_format_specifier(kind) {`
`219`	`219`	`stack.add(HighlightedRange {`
`@@ -223,13 +223,27 @@ pub(crate) fn highlight(`
`223`	`223`	`});`
`224`	`224`	`}`
`225`	`225`	`});`
	`226`	`+ stack.pop();`
	`227`	`+ }`
	`228`	`+ // Highlight escape sequences`
	`229`	`+ if let Some(char_ranges) = string.char_ranges() {`
	`230`	`+ stack.push();`
	`231`	`+ for (piece_range, _) in char_ranges.iter().filter(\|(_, char)\| char.is_ok()) {`
	`232`	`+ if string.text()[piece_range.start().into()..].starts_with('\\') {`
	`233`	`+ stack.add(HighlightedRange {`
	`234`	`+ range: piece_range + range.start(),`
	`235`	`+ highlight: HighlightTag::EscapeSequence.into(),`
	`236`	`+ binding_hash: None,`
	`237`	`+ });`
	`238`	`+ }`
	`239`	`+ }`
	`240`	`+ stack.pop_and_inject(false);`
`226`	`241`	`}`
`227`		`- stack.pop();`
`228`	`242`	`} else if let Some(string) =`
`229`	`243`	`element_to_highlight.as_token().cloned().and_then(ast::RawString::cast)`
`230`	`244`	`{`
`231`		`- stack.push();`
`232`	`245`	`if is_format_string {`
	`246`	`+ stack.push();`
`233`	`247`	`string.lex_format_specifier(\|piece_range, kind\| {`
`234`	`248`	`if let Some(highlight) = highlight_format_specifier(kind) {`
`235`	`249`	`stack.add(HighlightedRange {`
`@@ -239,8 +253,8 @@ pub(crate) fn highlight(`
`239`	`253`	`});`
`240`	`254`	`}`
`241`	`255`	`});`
	`256`	`+ stack.pop();`
`242`	`257`	`}`
`243`		`- stack.pop();`
`244`	`258`	`}`
`245`	`259`	`}`
`246`	`260`	`}`