From ca14081173329d46bea8f6d3b9570c7fb4be896e Mon Sep 17 00:00:00 2001
From: Lachlan Easton <lachlan@lakebythewoods.xyz>
Date: Sat, 14 Dec 2019 14:22:17 +1100
Subject: [PATCH 1/2] Tokenizer: Copy optional tokens prior to being set to
 null #3737

---
 lib/std/zig/tokenizer.zig | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig
index eda152710007..990814c25031 100644
--- a/lib/std/zig/tokenizer.zig
+++ b/lib/std/zig/tokenizer.zig
@@ -411,8 +411,9 @@ pub const Tokenizer = struct {
 
     pub fn next(self: *Tokenizer) Token {
         if (self.pending_invalid_token) |token| {
+            const copy = token;
             self.pending_invalid_token = null;
-            return token;
+            return copy;
         }
         const start_index = self.index;
         var state = State.Start;
@@ -1265,8 +1266,9 @@ pub const Tokenizer = struct {
 
         if (result.id == Token.Id.Eof) {
             if (self.pending_invalid_token) |token| {
+                const copy = token;
                 self.pending_invalid_token = null;
-                return token;
+                return copy;
             }
         }
 

From 7ad51b0de08ad09a3e1cec39a5330662dba998ff Mon Sep 17 00:00:00 2001
From: Lachlan Easton <lachlan@lakebythewoods.xyz>
Date: Mon, 16 Dec 2019 19:11:35 +1100
Subject: [PATCH 2/2] Add TODO comments, reminder to audit copying optional
 pattern.

---
 lib/std/json.zig          | 1 +
 lib/std/zig/tokenizer.zig | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/lib/std/json.zig b/lib/std/json.zig
index c1a0b4200955..030b940276b2 100644
--- a/lib/std/json.zig
+++ b/lib/std/json.zig
@@ -886,6 +886,7 @@ pub const TokenStream = struct {
 
     pub fn next(self: *TokenStream) Error!?Token {
         if (self.token) |token| {
+            // TODO: Audit this pattern once #2915 is closed
             const copy = token;
             self.token = null;
             return copy;
diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig
index 990814c25031..d95fb47d41ac 100644
--- a/lib/std/zig/tokenizer.zig
+++ b/lib/std/zig/tokenizer.zig
@@ -411,6 +411,7 @@ pub const Tokenizer = struct {
 
     pub fn next(self: *Tokenizer) Token {
         if (self.pending_invalid_token) |token| {
+            // TODO: Audit this pattern once #2915 is closed
             const copy = token;
             self.pending_invalid_token = null;
             return copy;
@@ -1266,6 +1267,7 @@ pub const Tokenizer = struct {
 
         if (result.id == Token.Id.Eof) {
             if (self.pending_invalid_token) |token| {
+                // TODO: Audit this pattern once #2915 is closed
                 const copy = token;
                 self.pending_invalid_token = null;
                 return copy;