Skip to content

Commit b9c3885

Browse files
ltentrupltentrup
committed
Implement syntax highlighting for strings
Detailed changes: 1) Implement a lexer for string literals that divides the string in format specifier `{}` and escape sequences. 2) Adapt syntax highlighting to add ranges for the detected sequences. 3) Add a test case for the string syntax highlighting.
1 parent 29a8464 commit b9c3885

File tree

6 files changed

+259
-4
lines changed

6 files changed

+259
-4
lines changed

‎crates/ra_ide/src/snapshots/highlight_injection.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
<span class="function">fixture</span>(<span class="string_literal">r#"</span>
3333
<span class="keyword">trait</span> <span class="trait declaration">Foo</span> {
3434
<span class="keyword">fn</span> <span class="function declaration">foo</span>() {
35-
<span class="macro">println!</span>(<span class="string_literal">"2 + 2 = {}"</span>, <span class="numeric_literal">4</span>);
35+
<span class="macro">println!</span>(<span class="string_literal">"2 + 2 = </span><span class="attribute">{}</span><span class="string_literal">"</span>, <span class="numeric_literal">4</span>);
3636
}
3737
}<span class="string_literal">"#</span>
3838
);

‎crates/ra_ide/src/snapshots/highlight_strings.html

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
2+
<style>
3+
body { margin: 0; }
4+
pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padding: 0.4em; }
5+
6+
.lifetime { color: #DFAF8F; font-style: italic; }
7+
.comment { color: #7F9F7F; }
8+
.struct, .enum { color: #7CB8BB; }
9+
.enum_variant { color: #BDE0F3; }
10+
.string_literal { color: #CC9393; }
11+
.field { color: #94BFF3; }
12+
.function { color: #93E0E3; }
13+
.parameter { color: #94BFF3; }
14+
.text { color: #DCDCCC; }
15+
.type { color: #7CB8BB; }
16+
.builtin_type { color: #8CD0D3; }
17+
.type_param { color: #DFAF8F; }
18+
.attribute { color: #94BFF3; }
19+
.numeric_literal { color: #BFEBBF; }
20+
.macro { color: #94BFF3; }
21+
.module { color: #AFD8AF; }
22+
.variable { color: #DCDCCC; }
23+
.mutable { text-decoration: underline; }
24+
25+
.keyword { color: #F0DFAF; font-weight: bold; }
26+
.keyword.unsafe { color: #BC8383; font-weight: bold; }
27+
.control { font-style: italic; }
28+
</style>
29+
<pre><code><span class="keyword">fn</span> <span class="function declaration">main</span>() {
30+
<span class="macro">println!</span>(<span class="string_literal">"2 + </span><span class="attribute">{}</span><span class="string_literal"> = </span><span class="attribute">{}</span><span class="string_literal">"</span>, <span class="numeric_literal">2</span>, <span class="numeric_literal">4</span>);
31+
<span class="macro">format!</span>(<span class="string_literal">"2 + </span><span class="attribute">{:?}</span><span class="string_literal"> }}= </span><span class="attribute">{}</span><span class="string_literal">"</span>, <span class="numeric_literal">2</span>, <span class="numeric_literal">4</span>);
32+
<span class="macro">print!</span>(<span class="string_literal">"2 + {{ </span><span class="attribute">{:#?}</span><span class="string_literal"> = </span><span class="attribute">{}</span><span class="string_literal"> {{}}"</span>, <span class="numeric_literal">2</span>, <span class="numeric_literal">4</span>);
33+
<span class="macro">writeln!</span>(f, <span class="string_literal">"</span><span class="attribute">{}</span><span class="string_literal">"</span>, <span class="numeric_literal">4</span>);
34+
<span class="macro">assert_eq!</span>(<span class="numeric_literal">2</span>, <span class="numeric_literal">2</span>, <span class="string_literal">"2 == </span><span class="attribute">{}</span><span class="string_literal">"</span>, <span class="numeric_literal">2</span>);
35+
<span class="macro">assert_ne!</span>(<span class="numeric_literal">2</span>, <span class="numeric_literal">3</span>, <span class="string_literal">"2 != </span><span class="attribute">{}</span><span class="string_literal">"</span>, <span class="numeric_literal">3</span>);
36+
<span class="macro">panic!</span>(<span class="string_literal">"foo = </span><span class="attribute">{}</span><span class="string_literal">"</span>, <span class="string_literal">"bar"</span>);
37+
<span class="macro">unreachable!</span>(<span class="string_literal">"foo = </span><span class="attribute">{}</span><span class="string_literal">"</span>, <span class="string_literal">"bar"</span>);
38+
39+
<span class="macro">println!</span>(<span class="string_literal">r"2 + {} = {}"</span>, <span class="numeric_literal">2</span>, <span class="numeric_literal">4</span>);
40+
41+
<span class="keyword">let</span> <span class="variable declaration">non_fmt_str</span> = <span class="string_literal">"2 + {{ {:#?} = {} {{}}"</span>;
42+
<span class="keyword">let</span> <span class="variable declaration">ascii_escapes</span> = <span class="string_literal">"</span><span class="attribute">\x41</span><span class="attribute">\x52</span><span class="string_literal"> </span><span class="attribute">\n</span><span class="string_literal"> </span><span class="attribute">\r</span><span class="string_literal"> </span><span class="attribute">\t</span><span class="string_literal"> </span><span class="attribute">\\</span><span class="string_literal"> </span><span class="attribute">\0</span><span class="string_literal">"</span>;
43+
<span class="keyword">let</span> <span class="variable declaration">unicode_escapes</span> = <span class="string_literal">"</span><span class="attribute">\u{7FFF}</span><span class="attribute">\u{7FF}</span><span class="string_literal"> </span><span class="attribute">\u{7FF}</span><span class="string_literal">"</span>;
44+
<span class="keyword">let</span> <span class="variable declaration">quote_escapes</span> = <span class="string_literal">"</span><span class="attribute">\'</span><span class="string_literal"> </span><span class="attribute">\"</span><span class="string_literal">"</span>;
45+
}</code></pre>

‎crates/ra_ide/src/snapshots/highlighting.html

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@
4848

4949
<span class="comment">// comment</span>
5050
<span class="keyword">fn</span> <span class="function declaration">main</span>() {
51-
<span class="macro">println!</span>(<span class="string_literal">"Hello, {}!"</span>, <span class="numeric_literal">92</span>);
51+
<span class="macro">println!</span>(<span class="string_literal">"Hello, </span><span class="attribute">{}</span><span class="string_literal">!"</span>, <span class="numeric_literal">92</span>);
5252

5353
<span class="keyword">let</span> <span class="keyword">mut</span> <span class="variable declaration mutable">vec</span> = Vec::new();
5454
<span class="keyword control">if</span> <span class="keyword">true</span> {

‎crates/ra_ide/src/syntax_highlighting.rs

Lines changed: 52 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ use rustc_hash::FxHashMap;
2121

2222
use crate::{call_info::call_info_for_token, Analysis, FileId};
2323

24+
use ast::StringPiece;
2425
pub(crate) use html::highlight_as_html;
2526
pub use tags::{Highlight, HighlightModifier, HighlightModifiers, HighlightTag};
2627

@@ -95,7 +96,8 @@ impl HighlightedRangeStack {
9596
1,
9697
"after DFS traversal, the stack should only contain a single element"
9798
);
98-
let res = self.stack.pop().unwrap();
99+
let mut res = self.stack.pop().unwrap();
100+
res.sort_by_key(|range| range.range.start());
99101
// Check that ranges are sorted and disjoint
100102
assert!(res
101103
.iter()
@@ -134,6 +136,7 @@ pub(crate) fn highlight(
134136
let mut stack = HighlightedRangeStack::new();
135137

136138
let mut current_macro_call: Option<ast::MacroCall> = None;
139+
let mut format_string: Option<SyntaxElement> = None;
137140

138141
// Walk all nodes, keeping track of whether we are inside a macro or not.
139142
// If in macro, expand it first and highlight the expanded code.
@@ -169,6 +172,7 @@ pub(crate) fn highlight(
169172
WalkEvent::Leave(Some(mc)) => {
170173
assert!(current_macro_call == Some(mc));
171174
current_macro_call = None;
175+
format_string = None;
172176
continue;
173177
}
174178
_ => (),
@@ -189,6 +193,33 @@ pub(crate) fn highlight(
189193
};
190194
let token = sema.descend_into_macros(token.clone());
191195
let parent = token.parent();
196+
197+
// Check if macro takes a format string and remember the string
198+
if let Some(name) = current_macro_call
199+
.as_ref()
200+
.and_then(|m| m.path())
201+
.and_then(|p| p.segment())
202+
.and_then(|s| s.name_ref())
203+
{
204+
// Determine the index of format string child based on macro name
205+
let format_string_pos = match name.text().as_str() {
206+
"format" | "println" | "print" | "eprint" | "eprintln" | "panic"
207+
| "unreachable" => Some(1),
208+
"write" | "writeln" | "assert" => Some(3),
209+
"assert_eq" | "assert_ne" => Some(5),
210+
_ => None,
211+
};
212+
format_string = format_string_pos.and_then(|pos| {
213+
parent
214+
.children_with_tokens()
215+
.filter(|t| t.kind() != WHITESPACE)
216+
.nth(pos)
217+
.filter(|e| {
218+
ast::String::can_cast(e.kind()) || ast::RawString::can_cast(e.kind())
219+
})
220+
});
221+
}
222+
192223
// We only care Name and Name_ref
193224
match (token.kind(), parent.kind()) {
194225
(IDENT, NAME) | (IDENT, NAME_REF) => parent.into(),
@@ -205,10 +236,29 @@ pub(crate) fn highlight(
205236
}
206237
}
207238

239+
let is_format_string =
240+
format_string.as_ref().map(|fs| fs == &element_to_highlight).unwrap_or_default();
241+
208242
if let Some((highlight, binding_hash)) =
209-
highlight_element(&sema, &mut bindings_shadow_count, element_to_highlight)
243+
highlight_element(&sema, &mut bindings_shadow_count, element_to_highlight.clone())
210244
{
211245
stack.add(HighlightedRange { range, highlight, binding_hash });
246+
if let Some(string) =
247+
element_to_highlight.as_token().cloned().and_then(ast::String::cast)
248+
{
249+
stack.push();
250+
string.lex(is_format_string, &mut |piece_range, kind| match kind {
251+
StringPiece::FormatSpecifier | StringPiece::EscapeSequence => {
252+
stack.add(HighlightedRange {
253+
range: piece_range + range.start(),
254+
highlight: HighlightTag::Attribute.into(),
255+
binding_hash: None,
256+
});
257+
}
258+
_ => {}
259+
});
260+
stack.pop();
261+
}
212262
}
213263
}
214264

‎crates/ra_ide/src/syntax_highlighting/tests.rs

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,3 +168,34 @@ macro_rules! test {}
168168
);
169169
let _ = analysis.highlight(file_id).unwrap();
170170
}
171+
172+
#[test]
173+
fn test_string_highlighting() {
174+
let (analysis, file_id) = single_file(
175+
r#"
176+
fn main() {
177+
println!("2 + {} = {}", 2, 4);
178+
format!("2 + {:?} }}= {}", 2, 4);
179+
print!("2 + {{ {:#?} = {} {{}}", 2, 4);
180+
writeln!(f, "{}", 4);
181+
assert_eq!(2, 2, "2 == {}", 2);
182+
assert_ne!(2, 3, "2 != {}", 3);
183+
panic!("foo = {}", "bar");
184+
unreachable!("foo = {}", "bar");
185+
186+
println!(r"2 + {} = {}", 2, 4);
187+
188+
let non_fmt_str = "2 + {{ {:#?} = {} {{}}";
189+
let ascii_escapes = "\x41\x52 \n \r \t \\ \0";
190+
let unicode_escapes = "\u{7FFF}\u{7FF} \u{7FF}";
191+
let quote_escapes = "\' \"";
192+
}"#
193+
.trim(),
194+
);
195+
196+
let dst_file = project_dir().join("crates/ra_ide/src/snapshots/highlight_strings.html");
197+
let actual_html = &analysis.highlight_as_html(file_id, false).unwrap();
198+
let expected_html = &read_text(&dst_file);
199+
fs::write(dst_file, &actual_html).unwrap();
200+
assert_eq_text!(expected_html, actual_html);
201+
}

‎crates/ra_syntax/src/ast/tokens.rs

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,3 +172,132 @@ impl RawString {
172172
Some(range + contents_range.start())
173173
}
174174
}
175+
176+
#[derive(Debug)]
177+
pub enum StringPiece {
178+
Quote,
179+
LiteralSequence,
180+
EscapeSequence,
181+
FormatSpecifier,
182+
}
183+
184+
impl String {
185+
pub fn lex<F>(&self, is_fmt_str: bool, callback: &mut F)
186+
where
187+
F: FnMut(TextRange, StringPiece),
188+
{
189+
let src = self.text().as_str();
190+
let initial_len = src.len();
191+
let mut chars = src.chars();
192+
193+
while let Some(first_char) = chars.next() {
194+
let start = initial_len - chars.as_str().len() - first_char.len_utf8();
195+
let piece = match first_char {
196+
'\\' => {
197+
// escape secquence, see https://doc.rust-lang.org/reference/tokens.html
198+
match chars.clone().next() {
199+
Some('\\') | Some('\n') | Some('n') | Some('r') | Some('t') | Some('0')
200+
| Some('\'') | Some('\"') => {
201+
chars.next();
202+
StringPiece::EscapeSequence
203+
}
204+
Some('x') => {
205+
// ascii escape
206+
chars.next();
207+
let mut cloned = chars.clone().take(2);
208+
let mut piece = None;
209+
while let Some(next_char) = cloned.next() {
210+
match next_char {
211+
'0'..='F' => {
212+
chars.next();
213+
}
214+
_ => {
215+
piece = Some(StringPiece::LiteralSequence);
216+
break;
217+
}
218+
}
219+
}
220+
piece.unwrap_or(StringPiece::EscapeSequence)
221+
}
222+
Some('u') => {
223+
// unicode escape
224+
chars.next();
225+
let mut cloned = chars.clone().take(8); // up to 6 digits + opening `{` and closing `}`
226+
if let Some(next_char) = cloned.next() {
227+
if next_char != '{' {
228+
StringPiece::LiteralSequence
229+
} else {
230+
let mut piece = None;
231+
while let Some(next_char) = cloned.next() {
232+
match next_char {
233+
'}' => {
234+
chars.next();
235+
chars.next();
236+
break;
237+
}
238+
'0'..='F' => {
239+
chars.next();
240+
}
241+
_ => {
242+
piece = Some(StringPiece::LiteralSequence);
243+
break;
244+
}
245+
}
246+
}
247+
piece.unwrap_or(StringPiece::EscapeSequence)
248+
}
249+
} else {
250+
StringPiece::LiteralSequence
251+
}
252+
}
253+
_ => StringPiece::LiteralSequence,
254+
}
255+
}
256+
'{' if is_fmt_str => {
257+
// format specifier
258+
match chars.clone().next() {
259+
Some('{') => {
260+
// escaped format specifier, `{{`
261+
chars.next();
262+
StringPiece::LiteralSequence
263+
}
264+
_ => {
265+
// consume format specifier
266+
let mut piece = None;
267+
while let Some(char) = chars.next() {
268+
if char == '}' {
269+
if let Some(next_char) = chars.clone().next() {
270+
if next_char == '}' {
271+
chars.next();
272+
continue;
273+
} else {
274+
piece = Some(StringPiece::FormatSpecifier);
275+
break;
276+
}
277+
}
278+
}
279+
}
280+
piece.unwrap_or(StringPiece::LiteralSequence)
281+
}
282+
}
283+
}
284+
_ => {
285+
while let Some(next_char) = chars.clone().next() {
286+
match next_char {
287+
'\\' => break,
288+
'{' if is_fmt_str => break,
289+
_ => {}
290+
}
291+
chars.next();
292+
}
293+
StringPiece::LiteralSequence
294+
}
295+
};
296+
let end = initial_len - chars.as_str().len();
297+
callback(
298+
TextRange::from_to(TextUnit::from_usize(start), TextUnit::from_usize(end)),
299+
piece,
300+
);
301+
}
302+
}
303+
}

0 commit comments

Comments
 (0)