test_runner: enable testing panics in mainTerminal

The user can use std.testing.spawnExpectPanic() in a test to spawn a
child process, which must panic or the test fails.
Internally,
- 1. is_panic_parentproc is set from the cli arguments for simple
reproduction of both test spawn and panic behavior,
- 2. panic_msg is set as threadlocal, if comptime-detectable capabilities
exist, to enable multithreaded processing and user-customized messages,
- 3. error.SpawnZigTest is returned to the test_runner.zig
- 4. the test_runner spawns a child_process on correct usage
- 5. the child_process expected to panic executes only one test block

This means, that only one @Panic is possible within a test block and that
no follow-up code after the @Panic in the test block can be run.

This commit does not add the panic test capability to the server yet,
since there are open design questions how many processes should be
spawned at the same time and how to manage time quotas to prevent
unnecessary slowdowns.

Supersedes ziglang#14351.
Work on ziglang#1356.

Author: matu3ba

lib/std/child_process.zig

@@ -240,6 +240,9 @@ pub const ChildProcess = struct {
     }
 
     /// Blocks until child process terminates and then cleans up all resources.
+    /// In case of error, the caller is responsible to clean up the ressources
+    /// via calling `self.cleanupStreams()`.
+    /// TODO: This describes the current state. Is this intended?
     pub fn wait(self: *ChildProcess) !Term {
         const term = if (builtin.os.tag == .windows)
             try self.waitWindows()
@@ -312,7 +315,7 @@ pub const ChildProcess = struct {
     };
 
     /// Spawns a child process, waits for it, collecting stdout and stderr, and then returns.
-    /// If it succeeds, the caller owns result.stdout and result.stderr memory.
+    /// If spawning succeeds, then the caller owns result.stdout and result.stderr memory.
     pub fn exec(args: struct {
         allocator: mem.Allocator,
         argv: []const []const u8,
@@ -415,7 +418,7 @@ pub const ChildProcess = struct {
         self.term = self.cleanupAfterWait(status);
     }
 
-    fn cleanupStreams(self: *ChildProcess) void {
+    pub fn cleanupStreams(self: *ChildProcess) void {
         if (self.stdin) |*stdin| {
             stdin.close();
             self.stdin = null;

lib/std/std.zig

@@ -196,6 +196,9 @@ pub const options = struct {
         options_override.side_channels_mitigations
     else
         crypto.default_side_channels_mitigations;
+
+    /// Default thread-local storage panic message size used for panic tests.
+    pub const testing_max_panic_msg_size = 100;
 };
 
 // This forces the start.zig file to be imported, and the comptime logic inside that

lib/std/testing.zig

@@ -916,6 +916,37 @@ test "expectEqualDeep composite type" {
     }
 }
 
+pub const can_panic_test = builtin.is_test and !builtin.single_threaded and std.process.can_spawn;
+
+/// Static storage to support user-generated panic messages
+/// Parent process writes these, returns to the test execution loop and spawns,
+/// child process ignores these.
+const TestFn_iT = if (can_panic_test) ?[std.options.testing_max_panic_msg_size:0]u8 else void;
+pub threadlocal var panic_msg: TestFn_iT = if (can_panic_test) null else {};
+
+/// Distinguishes between parent and child, if panics are tested for.
+/// TODO: is_panic_parentproc and panic_msg feels like it belongs into test api to
+/// allow implementations providing their own way to prevent the necessity to use tls.
+pub var is_panic_parentproc: if (can_panic_test) bool else void = if (can_panic_test) true else {};
+
+/// To be used for panic tests after test block declaration.
+pub fn spawnExpectPanic(msg: []const u8) error{ SpawnZigTest, SkipZigTest }!void {
+    std.debug.assert(can_panic_test); // Caller is responsible to check.
+    if (is_panic_parentproc) {
+        if (panic_msg == null) {
+            panic_msg = .{undefined} ** std.options.testing_max_panic_msg_size;
+            @memcpy(panic_msg.?[0..msg.len], msg); // Message must be persistent, not stack-local.
+            panic_msg.?[msg.len] = 0; // 0-sentinel for the len without separate field
+            return error.SpawnZigTest; // Test will be run in separate process
+        } else {
+            @panic("std.testing.panic_msg must be only used in spawnExpectPanic");
+        }
+    } else {
+        std.debug.assert(panic_msg == null);
+        // panic runner continues running the test block
+    }
+}
+
 fn printIndicatorLine(source: []const u8, indicator_index: usize) void {
     const line_begin_index = if (std.mem.lastIndexOfScalar(u8, source[0..indicator_index], '\n')) |line_begin|
         line_begin + 1

lib/std/zig/Server.zig

@@ -42,7 +42,7 @@ pub const Message = struct {
     ///   - 0 means not async
     /// * expected_panic_msg: [tests_len]u32,
     ///   - null-terminated string_bytes index
-    ///   - 0 means does not expect pani
+    ///   - 0 means does not expect panic
     /// * string_bytes: [string_bytes_len]u8,
     pub const TestMetadata = extern struct {
         string_bytes_len: u32,

lib/test_runner.zig

@@ -11,6 +11,21 @@ var log_err_count: usize = 0;
 var cmdline_buffer: [4096]u8 = undefined;
 var fba = std.heap.FixedBufferAllocator.init(&cmdline_buffer);
 
+const Mode = enum {
+    listen,
+    terminal,
+    panic_test,
+};
+
+fn callError(args: [][:0]u8) noreturn {
+    std.debug.print("invalid cli arguments:\n", .{});
+    for (args) |arg| {
+        std.debug.print("{s} ", .{arg});
+    }
+    std.debug.print("\n", .{});
+    @panic("call error");
+}
+
 pub fn main() void {
     if (builtin.zig_backend == .stage2_aarch64) {
         return mainSimple() catch @panic("test failure");
@@ -19,20 +34,33 @@ pub fn main() void {
     const args = std.process.argsAlloc(fba.allocator()) catch
         @panic("unable to parse command line args");
 
-    var listen = false;
+    var i: u32 = 1;
+    var test_i: ?u64 = null;
+    var mode: Mode = .terminal;
 
-    for (args[1..]) |arg| {
-        if (std.mem.eql(u8, arg, "--listen=-")) {
-            listen = true;
+    while (i < args.len) : (i += 1) {
+        if (std.mem.eql(u8, args[i], "--listen=-")) {
+            mode = .listen;
+        } else if (std.mem.eql(u8, args[i], "--test_panic_index")) {
+            i += 1;
+            if (i < args.len) {
+                test_i = std.fmt.parseInt(u64, args[i], 10) catch {
+                    callError(args);
+                };
+                mode = .panic_test;
+                std.testing.is_panic_parentproc = false;
+            } else {
+                callError(args);
+            }
         } else {
-            @panic("unrecognized command line argument");
+            callError(args);
         }
     }
 
-    if (listen) {
-        return mainServer() catch @panic("internal test runner failure");
-    } else {
-        return mainTerminal();
+    switch (mode) {
+        .listen => return mainServer() catch @panic("internal test runner failure"),
+        .terminal => return mainTerminal(args),
+        .panic_test => return panicTest(test_i.?),
     }
 }
 
@@ -124,7 +152,18 @@ fn mainServer() !void {
     }
 }
 
-fn mainTerminal() void {
+// TODO
+// - [ ] has test_i:
+//   * spawn and compare specific function
+//   * compare result: if returning from execution => @panic("FoundNoPanicInTest");
+// - [ ] not test_i:
+//   * iterate through all functions
+//   * compare result: compare execution result with special case for panic msg "FoundNoPanicInTest"
+
+fn mainTerminal(args: [][:0]const u8) void {
+    var test_i_buf: [20]u8 = undefined;
+    // TODO make environment buffer size configurable and use a sane default
+    // Tradeoff: waste stack space or allocate on every panic test
     const test_fn_list = builtin.test_functions;
     var ok_count: usize = 0;
     var skip_count: usize = 0;
@@ -140,7 +179,6 @@ fn mainTerminal() void {
     // TODO this is on the next line (using `undefined` above) because otherwise zig incorrectly
     // ignores the alignment of the slice.
     async_frame_buffer = &[_]u8{};
-
     var leaks: usize = 0;
     for (test_fn_list, 0..) |test_fn, i| {
         std.testing.allocator_instance = .{};
@@ -183,9 +221,134 @@ fn mainTerminal() void {
                 progress.log("SKIP\n", .{});
                 test_node.end();
             },
+            error.SpawnZigTest => {
+                progress.log("error.SpawnZigTest\n", .{});
+                if (!std.testing.can_panic_test)
+                    @panic("Found error.SpawnZigTest without panic test capabilities.");
+                if (std.testing.panic_msg == null)
+                    @panic("Panic test expects `panic_msg` to be set. Use std.testing.spawnExpectPanic().");
+
+                const test_i_written = std.fmt.bufPrint(&test_i_buf, "{d}", .{i}) catch unreachable;
+                var child_proc = std.ChildProcess.init(
+                    &.{ args[0], "--test_panic_index", test_i_written },
+                    std.testing.allocator,
+                );
+                progress.log("spawning '{s} {s} {s}'\n", .{ args[0], "--test_panic_index", test_i_written });
+
+                child_proc.stdin_behavior = .Ignore;
+                child_proc.stdout_behavior = .Pipe;
+                child_proc.stderr_behavior = .Pipe;
+                child_proc.spawn() catch |spawn_err| {
+                    progress.log("FAIL spawn ({s})\n", .{@errorName(spawn_err)});
+                    fail_count += 1;
+                    test_node.end();
+                    continue;
+                };
+
+                var stdout = std.ArrayList(u8).init(std.testing.allocator);
+                defer stdout.deinit();
+                var stderr = std.ArrayList(u8).init(std.testing.allocator);
+                defer stderr.deinit();
+                // child_process.zig: max_output_bytes: usize = 50 * 1024,
+                child_proc.collectOutput(&stdout, &stderr, 50 * 1024) catch |collect_err| {
+                    progress.log("FAIL collect ({s})\n", .{@errorName(collect_err)});
+                    fail_count += 1;
+                    test_node.end();
+                    continue;
+                };
+                const term = child_proc.wait() catch |wait_err| {
+                    child_proc.cleanupStreams();
+                    progress.log("FAIL wait_error (exit_status: {d})\n", .{@errorName(wait_err)});
+                    fail_count += 1;
+                    test_node.end();
+                    continue;
+                };
+                switch (term) {
+                    .Exited => |code| {
+                        progress.log("FAIL term exited, status: {})\nstdout: ({s})\nstderr: ({s})\n", .{ code, stdout.items, stderr.items });
+                        fail_count += 1;
+                        test_node.end();
+                        continue;
+                    },
+                    .Signal => |code| {
+                        progress.log("Signal: {d}\n", .{code});
+                        // assert: panic message format: 'XYZ thread thread_id panic: msg'
+                        // Any signal can be returned on panic, if a custom signal
+                        // or panic handler was installed as part of the unit test.
+                        var pos_eol: usize = 0;
+                        var found_eol: bool = false;
+                        while (pos_eol < stderr.items.len) : (pos_eol += 1) {
+                            if (stderr.items[pos_eol] == '\n') {
+                                found_eol = true;
+                                break;
+                            }
+                        }
+
+                        if (!found_eol) {
+                            progress.log("FAIL no end of line in panic format\nstdout: ({s})\nstderr: ({s})\n", .{ stdout.items, stderr.items });
+                            fail_count += 1;
+                            test_node.end();
+                            continue;
+                        }
+
+                        var it = std.mem.tokenize(u8, stderr.items[0..pos_eol], " ");
+                        var parsed_panic_msg = false;
+                        while (it.next()) |word| { // 'thread thread_id panic: msg'
+                            if (!std.mem.eql(u8, word, "thread")) continue;
+                            const thread_id = it.next();
+                            if (thread_id == null) continue;
+                            _ = std.fmt.parseInt(u64, thread_id.?, 10) catch continue;
+                            const panic_txt = it.next();
+                            if (panic_txt == null) continue;
+                            if (!std.mem.eql(u8, panic_txt.?, "panic:")) continue;
+                            const panic_msg = it.next();
+                            if (panic_msg == null) continue;
+                            const panic_msg_start = it.index - panic_msg.?.len;
+                            const len_exp_panic_msg = std.mem.len(@as([*:0]u8, std.testing.panic_msg.?[0..]));
+                            const expected_panic_msg = std.testing.panic_msg.?[0..len_exp_panic_msg];
+                            const panic_msg_end = panic_msg_start + expected_panic_msg.len;
+                            if (panic_msg_end > pos_eol) break;
+
+                            parsed_panic_msg = true;
+                            const current_panic_msg = stderr.items[panic_msg_start..panic_msg_end];
+
+                            if (!std.mem.eql(u8, "SKIP (async test)", current_panic_msg) and !std.mem.eql(u8, expected_panic_msg, current_panic_msg)) {
+                                progress.log("FAIL expected_panic_msg: '{s}', got: '{s}'\n", .{ expected_panic_msg, current_panic_msg });
+                                std.testing.panic_msg = null;
+                                fail_count += 1;
+                                test_node.end();
+                                break;
+                            }
+                            std.testing.panic_msg = null;
+                            ok_count += 1;
+                            test_node.end();
+                            if (!have_tty) std.debug.print("OK\n", .{});
+                            break;
+                        }
+                        if (!parsed_panic_msg) {
+                            progress.log("FAIL invalid panic_msg format expect 'XYZ thread thread_id panic: msg'\nstdout: ({s})\nstderr: ({s})\n", .{ stdout.items, stderr.items });
+                            fail_count += 1;
+                            test_node.end();
+                            continue;
+                        }
+                    },
+                    .Stopped => |code| {
+                        fail_count += 1;
+                        progress.log("FAIL stopped, status: ({d})\nstdout: ({s})\nstderr: ({s})\n", .{ code, stdout.items, stderr.items });
+                        test_node.end();
+                        continue;
+                    },
+                    .Unknown => |code| {
+                        fail_count += 1;
+                        progress.log("FAIL unknown, status: ({d})\nstdout: ({s})\nstderr: ({s})\n", .{ code, stdout.items, stderr.items });
+                        test_node.end();
+                        continue;
+                    },
+                }
+            },
             else => {
                 fail_count += 1;
-                progress.log("FAIL ({s})\n", .{@errorName(err)});
+                progress.log("FAIL unexpected error ({s})\n", .{@errorName(err)});
                 if (@errorReturnTrace()) |trace| {
                     std.debug.dumpStackTrace(trace.*);
                 }
@@ -210,6 +373,45 @@ fn mainTerminal() void {
     }
 }
 
+fn panicTest(test_i: u64) void {
+    const test_fn_list = builtin.test_functions;
+    var async_frame_buffer: []align(std.Target.stack_align) u8 = undefined;
+    // TODO this is on the next line (using `undefined` above) because otherwise zig incorrectly
+    // ignores the alignment of the slice.
+    async_frame_buffer = &[_]u8{};
+    {
+        std.testing.allocator_instance = .{};
+        // custom panic handler to restore to save state and prevent memory
+        // leakage is out of scope, so ignore memory leaks
+        defer {
+            if (std.testing.allocator_instance.deinit() == .leak) {
+                @panic("internal test runner memory leak");
+            }
+        }
+        std.testing.log_level = .warn;
+        const result = if (test_fn_list[test_i].async_frame_size) |size| switch (std.options.io_mode) {
+            .evented => blk: {
+                if (async_frame_buffer.len < size) {
+                    std.heap.page_allocator.free(async_frame_buffer);
+                    async_frame_buffer = std.heap.page_allocator.alignedAlloc(u8, std.Target.stack_align, size) catch @panic("out of memory");
+                }
+                const casted_fn = @ptrCast(fn () callconv(.Async) anyerror!void, test_fn_list[test_i].func);
+                break :blk await @asyncCall(async_frame_buffer, {}, casted_fn, .{});
+            },
+            .blocking => @panic("SKIP (async test)"),
+        } else test_fn_list[test_i].func();
+
+        if (result) {
+            std.os.exit(0);
+        } else |err| {
+            std.debug.print("FAIL unexpected error ({s})\n", .{@errorName(err)});
+            if (@errorReturnTrace()) |trace| {
+                std.debug.dumpStackTrace(trace.*);
+            }
+        }
+    }
+}
+
 pub fn log(
     comptime message_level: std.log.Level,
     comptime scope: @Type(.EnumLiteral),