Treat tab as text, not whitespace (#312)

Add pub(crate) parser::matches_fluent_ws helper function

Author: eemeli

fluent-syntax/src/parser/mod.rs

@@ -79,6 +79,7 @@ mod slice;
 
 use crate::ast;
 pub use errors::{ErrorKind, ParserError};
+pub(crate) use slice::matches_fluent_ws;
 pub use slice::Slice;
 
 /// Parser result always returns an AST representation of the input,

fluent-syntax/src/parser/slice.rs

@@ -1,4 +1,9 @@
 use std::ops::Range;
+
+pub(crate) fn matches_fluent_ws(c: char) -> bool {
+    c == ' ' || c == '\r' || c == '\n'
+}
+
 pub trait Slice<'s>: AsRef<str> + Clone + PartialEq {
     fn slice(&self, range: Range<usize>) -> Self;
     fn trim(&mut self);
@@ -10,7 +15,7 @@ impl<'s> Slice<'s> for String {
     }
 
     fn trim(&mut self) {
-        *self = self.trim_end().to_string();
+        *self = self.trim_end_matches(matches_fluent_ws).to_string();
     }
 }
 
@@ -20,6 +25,6 @@ impl<'s> Slice<'s> for &'s str {
     }
 
     fn trim(&mut self) {
-        *self = self.trim_end();
+        *self = self.trim_end_matches(matches_fluent_ws);
     }
 }

fluent-syntax/src/serializer.rs

@@ -21,7 +21,7 @@
 //! assert_eq!(ftl, serialized);
 //! ```
 
-use crate::{ast::*, parser::Slice};
+use crate::{ast::*, parser::Slice, parser::matches_fluent_ws};
 use std::fmt::Write;
 
 /// Serializes an abstract syntax tree representing a Fluent Translation List into a
@@ -118,7 +118,11 @@ impl Serializer {
         for line in &comment.content {
             self.writer.write_literal(prefix);
 
-            if !line.as_ref().trim().is_empty() {
+            if !line
+                .as_ref()
+                .trim_matches(matches_fluent_ws)
+                .is_empty()
+            {
                 self.writer.write_literal(" ");
                 self.writer.write_literal(line.as_ref());
             }

fluent-syntax/tests/fixtures/tab.ftl

@@ -12,3 +12,10 @@ key03 =
 key04 =
     This line is indented by 4 spaces,
 	whereas this line by 1 tab.
+
+# OK (value is a single tab)
+key05 = 	
+
+# OK (attribute value is two tabs)
+key06 =
+  .attr = 		

fluent-syntax/tests/fixtures/tab.json

@@ -64,7 +64,58 @@
         {
             "type": "Junk",
             "annotations": [],
-            "content": "\twhereas this line by 1 tab.\n"
+            "content": "\twhereas this line by 1 tab.\n\n"
+        },
+        {
+            "type": "Message",
+            "id": {
+                "type": "Identifier",
+                "name": "key05"
+            },
+            "value": {
+                "type": "Pattern",
+                "elements": [
+                    {
+                        "type": "TextElement",
+                        "value": "\t"
+                    }
+                ]
+            },
+            "attributes": [],
+            "comment": {
+                "type": "Comment",
+                "content": "OK (value is a single tab)"
+            }
+        },
+        {
+            "type": "Message",
+            "id": {
+                "type": "Identifier",
+                "name": "key06"
+            },
+            "value": null,
+            "attributes": [
+                {
+                    "type": "Attribute",
+                    "id": {
+                        "type": "Identifier",
+                        "name": "attr"
+                    },
+                    "value": {
+                        "type": "Pattern",
+                        "elements": [
+                            {
+                                "type": "TextElement",
+                                "value": "\t\t"
+                            }
+                        ]
+                    }
+                }
+            ],
+            "comment": {
+                "type": "Comment",
+                "content": "OK (attribute value is two tabs)"
+            }
         }
     ]
 }