Tokenizer: Copy optional tokens prior to being set to null #3737 (#3910)

* Tokenizer: Copy optional tokens prior to being set to null #3737

* Add TODO comments, reminder to audit copying optional pattern.

Author: LakeByTheWoods

lib/std/json.zig

@@ -886,6 +886,7 @@ pub const TokenStream = struct {
 
     pub fn next(self: *TokenStream) Error!?Token {
         if (self.token) |token| {
+            // TODO: Audit this pattern once #2915 is closed
             const copy = token;
             self.token = null;
             return copy;

lib/std/zig/tokenizer.zig

@@ -411,8 +411,10 @@ pub const Tokenizer = struct {
 
     pub fn next(self: *Tokenizer) Token {
         if (self.pending_invalid_token) |token| {
+            // TODO: Audit this pattern once #2915 is closed
+            const copy = token;
             self.pending_invalid_token = null;
-            return token;
+            return copy;
         }
         const start_index = self.index;
         var state = State.Start;
@@ -1265,8 +1267,10 @@ pub const Tokenizer = struct {
 
         if (result.id == Token.Id.Eof) {
             if (self.pending_invalid_token) |token| {
+                // TODO: Audit this pattern once #2915 is closed
+                const copy = token;
                 self.pending_invalid_token = null;
-                return token;
+                return copy;
             }
         }