Merge pull request #1023 from kiwix/suggestions_with_control_characters

Control characters are escaped in suggestions JSON

Author: kelson42

src/tools/otherTools.cpp

@@ -330,17 +330,19 @@ std::string kiwix::render_template(const std::string& template_str, kainjow::mus
 namespace
 {
 
-std::string escapeBackslashes(const std::string& s)
+std::string escapeForJSON(const std::string& s)
 {
-  std::string es;
-  es.reserve(s.size());
+  std::ostringstream oss;
   for (char c : s) {
     if ( c == '\\' ) {
-      es.push_back('\\');
+      oss << "\\\\";
+    } else if ( unsigned(c) < 0x20U ) {
+      oss << "\\u" << std::setw(4) << std::setfill('0') << unsigned(c);
+    } else {
+      oss << c;
     }
-    es.push_back(c);
   }
-  return es;
+  return oss.str();
 }
 
 std::string makeFulltextSearchSuggestion(const std::string& lang,
@@ -368,10 +370,10 @@ void kiwix::Suggestions::add(const zim::SuggestionItem& suggestion)
                           ? suggestion.getSnippet()
                           : suggestion.getTitle();
 
-  result.set("label", escapeBackslashes(label));
-  result.set("value", escapeBackslashes(suggestion.getTitle()));
+  result.set("label", escapeForJSON(label));
+  result.set("value", escapeForJSON(suggestion.getTitle()));
   result.set("kind", "path");
-  result.set("path", escapeBackslashes(suggestion.getPath()));
+  result.set("path", escapeForJSON(suggestion.getPath()));
   result.set("first", m_data.is_empty_list());
   m_data.push_back(result);
 }
@@ -381,8 +383,8 @@ void kiwix::Suggestions::addFTSearchSuggestion(const std::string& uiLang,
 {
   kainjow::mustache::data result;
   const std::string label = makeFulltextSearchSuggestion(uiLang, queryString);
-  result.set("label", escapeBackslashes(label));
-  result.set("value", escapeBackslashes(queryString + " "));
+  result.set("label", escapeForJSON(label));
+  result.set("value", escapeForJSON(queryString + " "));
   result.set("kind", "pattern");
   result.set("first", m_data.is_empty_list());
   m_data.push_back(result);

test/otherTools.cpp

@@ -100,7 +100,7 @@ TEST(Suggestions, specialCharHandling)
 {
   // HTML special symbols (<, >, &, ", and ') must be HTML-escaped
   // Backslash symbols (\) must be duplicated.
-  const std::string SYMBOLS(R"(\<>&'"~!@#$%^*()_+`-=[]{}|:;,.?)");
+  const std::string SYMBOLS("\t\n\r" R"(\<>&'"~!@#$%^*()_+`-=[]{}|:;,.?)");
   {
     kiwix::Suggestions s;
     s.add(zim::SuggestionItem("Title with "   + SYMBOLS,
@@ -110,10 +110,10 @@ TEST(Suggestions, specialCharHandling)
     CHECK_SUGGESTIONS(s.getJSON(),
 R"EXPECTEDJSON([
   {
-    "value" : "Title with \\&lt;&gt;&amp;&apos;&quot;~!@#$%^*()_+`-=[]{}|:;,.?",
-    "label" : "Snippet with \\&lt;&gt;&amp;&apos;&quot;~!@#$%^*()_+`-=[]{}|:;,.?",
+    "value" : "Title with \u0009\u0010\u0013\\&lt;&gt;&amp;&apos;&quot;~!@#$%^*()_+`-=[]{}|:;,.?",
+    "label" : "Snippet with \u0009\u0010\u0013\\&lt;&gt;&amp;&apos;&quot;~!@#$%^*()_+`-=[]{}|:;,.?",
     "kind" : "path"
-      , "path" : "Path with \\&lt;&gt;&amp;&apos;&quot;~!@#$%^*()_+`-=[]{}|:;,.?"
+      , "path" : "Path with \u0009\u0010\u0013\\&lt;&gt;&amp;&apos;&quot;~!@#$%^*()_+`-=[]{}|:;,.?"
   }
 ]
 )EXPECTEDJSON"
@@ -128,10 +128,10 @@ R"EXPECTEDJSON([
     CHECK_SUGGESTIONS(s.getJSON(),
 R"EXPECTEDJSON([
   {
-    "value" : "Snippetless title with \\&lt;&gt;&amp;&apos;&quot;~!@#$%^*()_+`-=[]{}|:;,.?",
-    "label" : "Snippetless title with \\&lt;&gt;&amp;&apos;&quot;~!@#$%^*()_+`-=[]{}|:;,.?",
+    "value" : "Snippetless title with \u0009\u0010\u0013\\&lt;&gt;&amp;&apos;&quot;~!@#$%^*()_+`-=[]{}|:;,.?",
+    "label" : "Snippetless title with \u0009\u0010\u0013\\&lt;&gt;&amp;&apos;&quot;~!@#$%^*()_+`-=[]{}|:;,.?",
     "kind" : "path"
-      , "path" : "Path with \\&lt;&gt;&amp;&apos;&quot;~!@#$%^*()_+`-=[]{}|:;,.?"
+      , "path" : "Path with \u0009\u0010\u0013\\&lt;&gt;&amp;&apos;&quot;~!@#$%^*()_+`-=[]{}|:;,.?"
   }
 ]
 )EXPECTEDJSON"
@@ -145,8 +145,8 @@ R"EXPECTEDJSON([
     CHECK_SUGGESTIONS(s.getJSON(),
 R"EXPECTEDJSON([
   {
-    "value" : "text with \\&lt;&gt;&amp;&apos;&quot;~!@#$%^*()_+`-=[]{}|:;,.? ",
-    "label" : "containing &apos;text with \\&lt;&gt;&amp;&apos;&quot;~!@#$%^*()_+`-=[]{}|:;,.?&apos;...",
+    "value" : "text with \u0009\u0010\u0013\\&lt;&gt;&amp;&apos;&quot;~!@#$%^*()_+`-=[]{}|:;,.? ",
+    "label" : "containing &apos;text with \u0009\u0010\u0013\\&lt;&gt;&amp;&apos;&quot;~!@#$%^*()_+`-=[]{}|:;,.?&apos;...",
     "kind" : "pattern"
     //EOLWHITESPACEMARKER
   }