Fix processing of multibyte messages (#4519)

* Fix processing of multibyte messages

* Too many characters are read from back-to-back multibyte messages due to a
conversion from byte-length to character length

* Updated to immediately convert messages to unibyte when processing

* Update encoding logic to avoid copies

---------

Co-authored-by: weedjer <[email protected]>

Author: JeremyWeed

lsp-mode.el

@@ -7074,8 +7074,8 @@ server. WORKSPACE is the active workspace."
         leftovers body-length body chunk)
     (lambda (_proc input)
       (setf chunk (if (s-blank? leftovers)
-                      input
-                    (concat leftovers input)))
+                      (encode-coding-string input 'utf-8-unix t)
+                    (concat leftovers (encode-coding-string input 'utf-8-unix t))))
 
       (let (messages)
         (while (not (s-blank? chunk))

test/lsp-io-test.el

@@ -66,6 +66,19 @@
          (messages (funcall fn message-in)))
     (should (equal messages '("’")))))
 
+(ert-deftest lsp--parser-read--multiple-multibyte-messages ()
+  (let* ((fn (lsp--create-process-message))
+         (messages-in '("Content-Length: 3\r\n\r\n←"
+                        "Content-Length: 3\r\n\r\n←"
+                        "Content-Length:3\r\n\r\n←"
+                        "Content-Length: 3\r\n\r\n←"
+                        "Content-Length:3\r\n\r\n←"
+                        "Content-Length: 3\r\n\r\n←"
+                        "Content-Length: 3\r\n\r\n←"
+                        ))
+         (messages (funcall fn (string-join messages-in))))
+    (should (equal messages '("←" "←" "←" "←" "←" "←" "←")))))
+
 (ert-deftest lsp--parser-read--multibyte-nospace ()
   (let* ((fn (lsp--create-process-message))
          (message-in "Content-Length:3\r\n\r\n\xe2\x80\x99")