diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index 703571eb56a6..1b13881586ee 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -235,6 +235,10 @@ pub fn main() !void { prominent_compile_errors = true; } else if (mem.eql(u8, arg, "--watch")) { watch = true; + } else if (mem.eql(u8, arg, "-fincremental")) { + graph.incremental = true; + } else if (mem.eql(u8, arg, "-fno-incremental")) { + graph.incremental = false; } else if (mem.eql(u8, arg, "-fwine")) { builder.enable_wine = true; } else if (mem.eql(u8, arg, "-fno-wine")) { @@ -406,8 +410,8 @@ pub fn main() !void { // trigger a rebuild on all steps with modified inputs, as well as their // recursive dependants. var caption_buf: [std.Progress.Node.max_name_len]u8 = undefined; - const caption = std.fmt.bufPrint(&caption_buf, "Watching {d} Directories", .{ - w.dir_table.entries.len, + const caption = std.fmt.bufPrint(&caption_buf, "watching {d} directories, {d} processes", .{ + w.dir_table.entries.len, countSubProcesses(run.step_stack.keys()), }) catch &caption_buf; var debouncing_node = main_progress_node.start(caption, 0); var debounce_timeout: Watch.Timeout = .none; @@ -440,6 +444,14 @@ fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { }; } +fn countSubProcesses(all_steps: []const *Step) usize { + var count: usize = 0; + for (all_steps) |s| { + count += @intFromBool(s.getZigProcess() != null); + } + return count; +} + const Run = struct { max_rss: u64, max_rss_is_default: bool, @@ -1031,7 +1043,11 @@ fn workerMakeOneStep( const sub_prog_node = prog_node.start(s.name, 0); defer sub_prog_node.end(); - const make_result = s.make(sub_prog_node); + const make_result = s.make(.{ + .progress_node = sub_prog_node, + .thread_pool = thread_pool, + .watch = run.watch, + }); // No matter the result, we want to display error/warning messages. const show_compile_errors = !run.prominent_compile_errors and @@ -1212,6 +1228,8 @@ fn usage(b: *std.Build, out_stream: anytype) !void { \\ --fetch Exit after fetching dependency tree \\ --watch Continuously rebuild when source files are modified \\ --debounce Delay before rebuilding after changed file detected + \\ -fincremental Enable incremental compilation + \\ -fno-incremental Disable incremental compilation \\ \\Project-Specific Options: \\ diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 36f7396c8eb6..06de7aa6f45e 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -120,6 +120,7 @@ pub const Graph = struct { needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .{}, /// Information about the native target. Computed before build() is invoked. host: ResolvedTarget, + incremental: ?bool = null, }; const AvailableDeps = []const struct { []const u8, []const u8 }; @@ -1078,8 +1079,8 @@ pub fn getUninstallStep(b: *Build) *Step { return &b.uninstall_tls.step; } -fn makeUninstall(uninstall_step: *Step, prog_node: std.Progress.Node) anyerror!void { - _ = prog_node; +fn makeUninstall(uninstall_step: *Step, options: Step.MakeOptions) anyerror!void { + _ = options; const uninstall_tls: *TopLevelStep = @fieldParentPtr("step", uninstall_step); const b: *Build = @fieldParentPtr("uninstall_tls", uninstall_tls); diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 3c6cd660fff1..8f3236d867a5 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -68,7 +68,13 @@ pub const TestResults = struct { } }; -pub const MakeFn = *const fn (step: *Step, prog_node: std.Progress.Node) anyerror!void; +pub const MakeOptions = struct { + progress_node: std.Progress.Node, + thread_pool: *std.Thread.Pool, + watch: bool, +}; + +pub const MakeFn = *const fn (step: *Step, options: MakeOptions) anyerror!void; pub const State = enum { precheck_unstarted, @@ -219,10 +225,10 @@ pub fn init(options: StepOptions) Step { /// If the Step's `make` function reports `error.MakeFailed`, it indicates they /// have already reported the error. Otherwise, we add a simple error report /// here. -pub fn make(s: *Step, prog_node: std.Progress.Node) error{ MakeFailed, MakeSkipped }!void { +pub fn make(s: *Step, options: MakeOptions) error{ MakeFailed, MakeSkipped }!void { const arena = s.owner.allocator; - s.makeFn(s, prog_node) catch |err| switch (err) { + s.makeFn(s, options) catch |err| switch (err) { error.MakeFailed => return error.MakeFailed, error.MakeSkipped => return error.MakeSkipped, else => { @@ -260,8 +266,8 @@ pub fn getStackTrace(s: *Step) ?std.builtin.StackTrace { }; } -fn makeNoOp(step: *Step, prog_node: std.Progress.Node) anyerror!void { - _ = prog_node; +fn makeNoOp(step: *Step, options: MakeOptions) anyerror!void { + _ = options; var all_cached = true; @@ -352,13 +358,54 @@ pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutO try step.result_error_msgs.append(arena, msg); } +pub const ZigProcess = struct { + child: std.process.Child, + poller: std.io.Poller(StreamEnum), + progress_ipc_fd: if (std.Progress.have_ipc) ?std.posix.fd_t else void, + + pub const StreamEnum = enum { stdout, stderr }; +}; + /// Assumes that argv contains `--listen=-` and that the process being spawned /// is the zig compiler - the same version that compiled the build runner. pub fn evalZigProcess( s: *Step, argv: []const []const u8, prog_node: std.Progress.Node, + watch: bool, ) !?[]const u8 { + if (s.getZigProcess()) |zp| update: { + assert(watch); + if (std.Progress.have_ipc) if (zp.progress_ipc_fd) |fd| prog_node.setIpcFd(fd); + const result = zigProcessUpdate(s, zp, watch) catch |err| switch (err) { + error.BrokenPipe => { + // Process restart required. + const term = zp.child.wait() catch |e| { + return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(e) }); + }; + _ = term; + s.clearZigProcess(); + break :update; + }, + else => |e| return e, + }; + + if (s.result_error_bundle.errorMessageCount() > 0) + return s.fail("{d} compilation errors", .{s.result_error_bundle.errorMessageCount()}); + + if (s.result_error_msgs.items.len > 0 and result == null) { + // Crash detected. + const term = zp.child.wait() catch |e| { + return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(e) }); + }; + s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; + s.clearZigProcess(); + try handleChildProcessTerm(s, term, null, argv); + return error.MakeFailed; + } + + return result; + } assert(argv.len != 0); const b = s.owner; const arena = b.allocator; @@ -378,29 +425,79 @@ pub fn evalZigProcess( child.spawn() catch |err| return s.fail("unable to spawn {s}: {s}", .{ argv[0], @errorName(err), }); - var timer = try std.time.Timer.start(); - var poller = std.io.poll(gpa, enum { stdout, stderr }, .{ - .stdout = child.stdout.?, - .stderr = child.stderr.?, - }); - defer poller.deinit(); + const zp = try gpa.create(ZigProcess); + zp.* = .{ + .child = child, + .poller = std.io.poll(gpa, ZigProcess.StreamEnum, .{ + .stdout = child.stdout.?, + .stderr = child.stderr.?, + }), + .progress_ipc_fd = if (std.Progress.have_ipc) child.progress_node.getIpcFd() else {}, + }; + if (watch) s.setZigProcess(zp); + defer if (!watch) zp.poller.deinit(); + + const result = try zigProcessUpdate(s, zp, watch); + + if (!watch) { + // Send EOF to stdin. + zp.child.stdin.?.close(); + zp.child.stdin = null; + + const term = zp.child.wait() catch |err| { + return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(err) }); + }; + s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0; + + // Special handling for Compile step that is expecting compile errors. + if (s.cast(Compile)) |compile| switch (term) { + .Exited => { + // Note that the exit code may be 0 in this case due to the + // compiler server protocol. + if (compile.expect_errors != null) { + return error.NeedCompileErrorCheck; + } + }, + else => {}, + }; + + try handleChildProcessTerm(s, term, null, argv); + } + + // This is intentionally printed for failure on the first build but not for + // subsequent rebuilds. + if (s.result_error_bundle.errorMessageCount() > 0) { + return s.fail("the following command failed with {d} compilation errors:\n{s}", .{ + s.result_error_bundle.errorMessageCount(), + try allocPrintCmd(arena, null, argv), + }); + } - try sendMessage(child.stdin.?, .update); - try sendMessage(child.stdin.?, .exit); + return result; +} + +fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 { + const b = s.owner; + const arena = b.allocator; + + var timer = try std.time.Timer.start(); + + try sendMessage(zp.child.stdin.?, .update); + if (!watch) try sendMessage(zp.child.stdin.?, .exit); const Header = std.zig.Server.Message.Header; var result: ?[]const u8 = null; - const stdout = poller.fifo(.stdout); + const stdout = zp.poller.fifo(.stdout); poll: while (true) { while (stdout.readableLength() < @sizeOf(Header)) { - if (!(try poller.poll())) break :poll; + if (!(try zp.poller.poll())) break :poll; } const header = stdout.reader().readStruct(Header) catch unreachable; while (stdout.readableLength() < header.bytes_len) { - if (!(try poller.poll())) break :poll; + if (!(try zp.poller.poll())) break :poll; } const body = stdout.readableSliceOfLen(header.bytes_len); @@ -428,12 +525,22 @@ pub fn evalZigProcess( .string_bytes = try arena.dupe(u8, string_bytes), .extra = extra_array, }; + if (watch) { + // This message indicates the end of the update. + stdout.discard(body.len); + break; + } }, .emit_bin_path => { const EbpHdr = std.zig.Server.Message.EmitBinPath; const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body)); s.result_cached = ebp_hdr.flags.cache_hit; result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]); + if (watch) { + // This message indicates the end of the update. + stdout.discard(body.len); + break; + } }, .file_system_inputs => { s.clearWatchInputs(); @@ -470,6 +577,13 @@ pub fn evalZigProcess( }; try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); }, + .global_cache => { + const path: Build.Cache.Path = .{ + .root_dir = s.owner.graph.global_cache_root, + .sub_path = sub_path_dirname, + }; + try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path)); + }, } } }, @@ -479,43 +593,42 @@ pub fn evalZigProcess( stdout.discard(body.len); } - const stderr = poller.fifo(.stderr); + s.result_duration_ns = timer.read(); + + const stderr = zp.poller.fifo(.stderr); if (stderr.readableLength() > 0) { try s.result_error_msgs.append(arena, try stderr.toOwnedSlice()); } - // Send EOF to stdin. - child.stdin.?.close(); - child.stdin = null; + return result; +} - const term = child.wait() catch |err| { - return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(err) }); - }; - s.result_duration_ns = timer.read(); - s.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0; - - // Special handling for Compile step that is expecting compile errors. - if (s.cast(Compile)) |compile| switch (term) { - .Exited => { - // Note that the exit code may be 0 in this case due to the - // compiler server protocol. - if (compile.expect_errors != null) { - return error.NeedCompileErrorCheck; - } - }, - else => {}, +pub fn getZigProcess(s: *Step) ?*ZigProcess { + return switch (s.id) { + .compile => s.cast(Compile).?.zig_process, + else => null, }; +} - try handleChildProcessTerm(s, term, null, argv); - - if (s.result_error_bundle.errorMessageCount() > 0) { - return s.fail("the following command failed with {d} compilation errors:\n{s}", .{ - s.result_error_bundle.errorMessageCount(), - try allocPrintCmd(arena, null, argv), - }); +fn setZigProcess(s: *Step, zp: *ZigProcess) void { + switch (s.id) { + .compile => s.cast(Compile).?.zig_process = zp, + else => unreachable, } +} - return result; +fn clearZigProcess(s: *Step) void { + const gpa = s.owner.allocator; + switch (s.id) { + .compile => { + const compile = s.cast(Compile).?; + if (compile.zig_process) |zp| { + gpa.destroy(zp); + compile.zig_process = null; + } + }, + else => unreachable, + } } fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void { diff --git a/lib/std/Build/Step/CheckFile.zig b/lib/std/Build/Step/CheckFile.zig index c7a2046c1f63..699e6d2e9d3f 100644 --- a/lib/std/Build/Step/CheckFile.zig +++ b/lib/std/Build/Step/CheckFile.zig @@ -46,8 +46,8 @@ pub fn setName(check_file: *CheckFile, name: []const u8) void { check_file.step.name = name; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; const b = step.owner; const check_file: *CheckFile = @fieldParentPtr("step", step); try step.singleUnchangingWatchInput(check_file.source); diff --git a/lib/std/Build/Step/CheckObject.zig b/lib/std/Build/Step/CheckObject.zig index 93ee57e3b47b..ce3300311777 100644 --- a/lib/std/Build/Step/CheckObject.zig +++ b/lib/std/Build/Step/CheckObject.zig @@ -550,8 +550,8 @@ pub fn checkComputeCompare( check_object.checks.append(check) catch @panic("OOM"); } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn make(step: *Step, make_options: Step.MakeOptions) !void { + _ = make_options; const b = step.owner; const gpa = b.allocator; const check_object: *CheckObject = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 89862e750137..724656ce29f1 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -213,6 +213,10 @@ is_linking_libcpp: bool = false, no_builtin: bool = false, +/// Populated during the make phase when there is a long-lived compiler process. +/// Managed by the build runner, not user build script. +zig_process: ?*Step.ZigProcess, + pub const ExpectedCompileErrors = union(enum) { contains: []const u8, exact: []const []const u8, @@ -398,6 +402,8 @@ pub fn create(owner: *std.Build, options: Options) *Compile { .use_llvm = options.use_llvm, .use_lld = options.use_lld, + + .zig_process = null, }; compile.root_module.init(owner, options.root_module, compile); @@ -1673,6 +1679,8 @@ fn getZigArgs(compile: *Compile) ![][]const u8 { b.fmt("{}", .{err_limit}), }); + try addFlag(&zig_args, "incremental", b.graph.incremental); + try zig_args.append("--listen=-"); // Windows has an argument length limit of 32,766 characters, macOS 262,144 and Linux @@ -1735,13 +1743,17 @@ fn getZigArgs(compile: *Compile) ![][]const u8 { return try zig_args.toOwnedSlice(); } -fn make(step: *Step, prog_node: std.Progress.Node) !void { +fn make(step: *Step, options: Step.MakeOptions) !void { const b = step.owner; const compile: *Compile = @fieldParentPtr("step", step); const zig_args = try getZigArgs(compile); - const maybe_output_bin_path = step.evalZigProcess(zig_args, prog_node) catch |err| switch (err) { + const maybe_output_bin_path = step.evalZigProcess( + zig_args, + options.progress_node, + (b.graph.incremental == true) and options.watch, + ) catch |err| switch (err) { error.NeedCompileErrorCheck => { assert(compile.expect_errors != null); try checkCompileErrors(compile); diff --git a/lib/std/Build/Step/ConfigHeader.zig b/lib/std/Build/Step/ConfigHeader.zig index fd655125cf11..512460a53255 100644 --- a/lib/std/Build/Step/ConfigHeader.zig +++ b/lib/std/Build/Step/ConfigHeader.zig @@ -164,8 +164,8 @@ fn putValue(config_header: *ConfigHeader, field_name: []const u8, comptime T: ty } } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; const b = step.owner; const config_header: *ConfigHeader = @fieldParentPtr("step", step); if (config_header.style.getPath()) |lp| try step.singleUnchangingWatchInput(lp); diff --git a/lib/std/Build/Step/Fail.zig b/lib/std/Build/Step/Fail.zig index 09b8515f773e..9236c2ac7b61 100644 --- a/lib/std/Build/Step/Fail.zig +++ b/lib/std/Build/Step/Fail.zig @@ -24,8 +24,8 @@ pub fn create(owner: *std.Build, error_msg: []const u8) *Fail { return fail; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; // No progress to report. +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; // No progress to report. const fail: *Fail = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Step/Fmt.zig b/lib/std/Build/Step/Fmt.zig index 1d3850a7ec76..576aeb1d21db 100644 --- a/lib/std/Build/Step/Fmt.zig +++ b/lib/std/Build/Step/Fmt.zig @@ -36,7 +36,9 @@ pub fn create(owner: *std.Build, options: Options) *Fmt { return fmt; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { +fn make(step: *Step, options: Step.MakeOptions) !void { + const prog_node = options.progress_node; + // TODO: if check=false, this means we are modifying source files in place, which // is an operation that could race against other operations also modifying source files // in place. In this case, this step should obtain a write lock while making those diff --git a/lib/std/Build/Step/InstallArtifact.zig b/lib/std/Build/Step/InstallArtifact.zig index bd1d5db4a977..4e778d897cf8 100644 --- a/lib/std/Build/Step/InstallArtifact.zig +++ b/lib/std/Build/Step/InstallArtifact.zig @@ -115,8 +115,8 @@ pub fn create(owner: *std.Build, artifact: *Step.Compile, options: Options) *Ins return install_artifact; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; const install_artifact: *InstallArtifact = @fieldParentPtr("step", step); const b = step.owner; const cwd = fs.cwd(); diff --git a/lib/std/Build/Step/InstallDir.zig b/lib/std/Build/Step/InstallDir.zig index 78281e56d9df..4d4ff78cfc75 100644 --- a/lib/std/Build/Step/InstallDir.zig +++ b/lib/std/Build/Step/InstallDir.zig @@ -55,8 +55,8 @@ pub fn create(owner: *std.Build, options: Options) *InstallDir { return install_dir; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; const b = step.owner; const install_dir: *InstallDir = @fieldParentPtr("step", step); step.clearWatchInputs(); diff --git a/lib/std/Build/Step/InstallFile.zig b/lib/std/Build/Step/InstallFile.zig index d29ac21c1cd4..fb1c0ffc3405 100644 --- a/lib/std/Build/Step/InstallFile.zig +++ b/lib/std/Build/Step/InstallFile.zig @@ -35,8 +35,8 @@ pub fn create( return install_file; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; const b = step.owner; const install_file: *InstallFile = @fieldParentPtr("step", step); try step.singleUnchangingWatchInput(install_file.source); diff --git a/lib/std/Build/Step/ObjCopy.zig b/lib/std/Build/Step/ObjCopy.zig index d314550f601d..6ed23d3fb4c2 100644 --- a/lib/std/Build/Step/ObjCopy.zig +++ b/lib/std/Build/Step/ObjCopy.zig @@ -90,7 +90,8 @@ pub fn getOutputSeparatedDebug(objcopy: *const ObjCopy) ?std.Build.LazyPath { return if (objcopy.output_file_debug) |*file| .{ .generated = .{ .file = file } } else null; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { +fn make(step: *Step, options: Step.MakeOptions) !void { + const prog_node = options.progress_node; const b = step.owner; const objcopy: *ObjCopy = @fieldParentPtr("step", step); try step.singleUnchangingWatchInput(objcopy.input_file); @@ -158,7 +159,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { try argv.appendSlice(&.{ full_src_path, full_dest_path }); try argv.append("--listen=-"); - _ = try step.evalZigProcess(argv.items, prog_node); + _ = try step.evalZigProcess(argv.items, prog_node, false); objcopy.output_file.path = full_dest_path; if (objcopy.output_file_debug) |*file| file.path = full_dest_path_debug; diff --git a/lib/std/Build/Step/Options.zig b/lib/std/Build/Step/Options.zig index b67acd408697..7c872db170ce 100644 --- a/lib/std/Build/Step/Options.zig +++ b/lib/std/Build/Step/Options.zig @@ -410,9 +410,9 @@ pub fn getOutput(options: *Options) LazyPath { return .{ .generated = .{ .file = &options.generated_file } }; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - // This step completes so quickly that no progress is necessary. - _ = prog_node; +fn make(step: *Step, make_options: Step.MakeOptions) !void { + // This step completes so quickly that no progress reporting is necessary. + _ = make_options; const b = step.owner; const options: *Options = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Step/RemoveDir.zig b/lib/std/Build/Step/RemoveDir.zig index 1b7dc7feb8be..e2d4c02abca8 100644 --- a/lib/std/Build/Step/RemoveDir.zig +++ b/lib/std/Build/Step/RemoveDir.zig @@ -23,10 +23,8 @@ pub fn create(owner: *std.Build, doomed_path: LazyPath) *RemoveDir { return remove_dir; } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - // TODO update progress node while walking file system. - // Should the standard library support this use case?? - _ = prog_node; +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; const b = step.owner; const remove_dir: *RemoveDir = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index bc89d1a12fe5..d79d4647b187 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -595,7 +595,8 @@ const IndexedOutput = struct { tag: @typeInfo(Arg).Union.tag_type.?, output: *Output, }; -fn make(step: *Step, prog_node: std.Progress.Node) !void { +fn make(step: *Step, options: Step.MakeOptions) !void { + const prog_node = options.progress_node; const b = step.owner; const arena = b.allocator; const run: *Run = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Step/TranslateC.zig b/lib/std/Build/Step/TranslateC.zig index d7c62ed1948e..ac4729abd0cc 100644 --- a/lib/std/Build/Step/TranslateC.zig +++ b/lib/std/Build/Step/TranslateC.zig @@ -116,7 +116,8 @@ pub fn defineCMacroRaw(translate_c: *TranslateC, name_and_value: []const u8) voi translate_c.c_macros.append(translate_c.step.owner.dupe(name_and_value)) catch @panic("OOM"); } -fn make(step: *Step, prog_node: std.Progress.Node) !void { +fn make(step: *Step, options: Step.MakeOptions) !void { + const prog_node = options.progress_node; const b = step.owner; const translate_c: *TranslateC = @fieldParentPtr("step", step); @@ -154,7 +155,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void { try argv_list.append(translate_c.source.getPath2(b, step)); - const output_path = try step.evalZigProcess(argv_list.items, prog_node); + const output_path = try step.evalZigProcess(argv_list.items, prog_node, false); translate_c.out_basename = fs.path.basename(output_path.?); const output_dir = fs.path.dirname(output_path.?).?; diff --git a/lib/std/Build/Step/UpdateSourceFiles.zig b/lib/std/Build/Step/UpdateSourceFiles.zig index 9d1c8e20fe62..d4a9565083fa 100644 --- a/lib/std/Build/Step/UpdateSourceFiles.zig +++ b/lib/std/Build/Step/UpdateSourceFiles.zig @@ -67,8 +67,8 @@ pub fn addBytesToSource(usf: *UpdateSourceFiles, bytes: []const u8, sub_path: [] }) catch @panic("OOM"); } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; const b = step.owner; const usf: *UpdateSourceFiles = @fieldParentPtr("step", step); diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig index c1488a23d2ed..29fba1c871cd 100644 --- a/lib/std/Build/Step/WriteFile.zig +++ b/lib/std/Build/Step/WriteFile.zig @@ -171,8 +171,8 @@ fn maybeUpdateName(write_file: *WriteFile) void { } } -fn make(step: *Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn make(step: *Step, options: Step.MakeOptions) !void { + _ = options; const b = step.owner; const arena = b.allocator; const gpa = arena; diff --git a/lib/std/Progress.zig b/lib/std/Progress.zig index bb5adcefd1e0..190f66789756 100644 --- a/lib/std/Progress.zig +++ b/lib/std/Progress.zig @@ -269,6 +269,19 @@ pub const Node = struct { storageByIndex(index).setIpcFd(fd); } + /// Posix-only. Thread-safe. Assumes the node is storing an IPC file + /// descriptor. + pub fn getIpcFd(node: Node) ?posix.fd_t { + const index = node.index.unwrap() orelse return null; + const storage = storageByIndex(index); + const int = @atomicLoad(u32, &storage.completed_count, .monotonic); + return switch (@typeInfo(posix.fd_t)) { + .Int => @bitCast(int), + .Pointer => @ptrFromInt(int), + else => @compileError("unsupported fd_t of " ++ @typeName(posix.fd_t)), + }; + } + fn storageByIndex(index: Node.Index) *Node.Storage { return &global_progress.node_storage[@intFromEnum(index)]; } @@ -329,6 +342,11 @@ var default_draw_buffer: [4096]u8 = undefined; var debug_start_trace = std.debug.Trace.init; +pub const have_ipc = switch (builtin.os.tag) { + .wasi, .freestanding, .windows => false, + else => true, +}; + const noop_impl = builtin.single_threaded or switch (builtin.os.tag) { .wasi, .freestanding => true, else => false, diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index f1d3bc7b61d3..2d66ee628d6f 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -36,6 +36,7 @@ pub const Message = struct { cwd, zig_lib, local_cache, + global_cache, }; /// Trailing: diff --git a/src/Compilation.zig b/src/Compilation.zig index a785351df554..76dc42fc64a9 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -169,7 +169,7 @@ time_report: bool, stack_report: bool, debug_compiler_runtime_libs: bool, debug_compile_errors: bool, -debug_incremental: bool, +incremental: bool, job_queued_compiler_rt_lib: bool = false, job_queued_compiler_rt_obj: bool = false, job_queued_update_builtin_zig: bool, @@ -1134,7 +1134,7 @@ pub const CreateOptions = struct { verbose_llvm_cpu_features: bool = false, debug_compiler_runtime_libs: bool = false, debug_compile_errors: bool = false, - debug_incremental: bool = false, + incremental: bool = false, /// Normally when you create a `Compilation`, Zig will automatically build /// and link in required dependencies, such as compiler-rt and libc. When /// building such dependencies themselves, this flag must be set to avoid @@ -1363,6 +1363,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() }); cache.addPrefix(options.zig_lib_directory); cache.addPrefix(options.local_cache_directory); + cache.addPrefix(options.global_cache_directory); errdefer cache.manifest_dir.close(); // This is shared hasher state common to zig source and all C source files. @@ -1515,7 +1516,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil .test_name_prefix = options.test_name_prefix, .debug_compiler_runtime_libs = options.debug_compiler_runtime_libs, .debug_compile_errors = options.debug_compile_errors, - .debug_incremental = options.debug_incremental, + .incremental = options.incremental, .libcxx_abi_version = options.libcxx_abi_version, .root_name = root_name, .sysroot = sysroot, @@ -2358,7 +2359,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { } } -fn appendFileSystemInput( +pub fn appendFileSystemInput( comp: *Compilation, file_system_inputs: *std.ArrayListUnmanaged(u8), root: Cache.Path, diff --git a/src/Sema.zig b/src/Sema.zig index f5ee909caf8d..36694eef47a9 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -2726,7 +2726,7 @@ fn maybeRemoveOutdatedType(sema: *Sema, ty: InternPool.Index) !bool { const pt = sema.pt; const zcu = pt.zcu; - if (!zcu.comp.debug_incremental) return false; + if (!zcu.comp.incremental) return false; const decl_index = Type.fromInterned(ty).getOwnerDecl(zcu); const decl_as_depender = AnalUnit.wrap(.{ .decl = decl_index }); @@ -2826,7 +2826,7 @@ fn zirStructDecl( mod.declPtr(new_decl_index).owns_tv = true; errdefer pt.abortAnonDecl(new_decl_index); - if (pt.zcu.comp.debug_incremental) { + if (pt.zcu.comp.incremental) { try ip.addDependency( sema.gpa, AnalUnit.wrap(.{ .decl = new_decl_index }), @@ -3064,7 +3064,7 @@ fn zirEnumDecl( new_decl.owns_tv = true; errdefer if (!done) pt.abortAnonDecl(new_decl_index); - if (pt.zcu.comp.debug_incremental) { + if (pt.zcu.comp.incremental) { try mod.intern_pool.addDependency( gpa, AnalUnit.wrap(.{ .decl = new_decl_index }), @@ -3331,7 +3331,7 @@ fn zirUnionDecl( mod.declPtr(new_decl_index).owns_tv = true; errdefer pt.abortAnonDecl(new_decl_index); - if (pt.zcu.comp.debug_incremental) { + if (pt.zcu.comp.incremental) { try mod.intern_pool.addDependency( gpa, AnalUnit.wrap(.{ .decl = new_decl_index }), @@ -3421,7 +3421,7 @@ fn zirOpaqueDecl( mod.declPtr(new_decl_index).owns_tv = true; errdefer pt.abortAnonDecl(new_decl_index); - if (pt.zcu.comp.debug_incremental) { + if (pt.zcu.comp.incremental) { try ip.addDependency( gpa, AnalUnit.wrap(.{ .decl = new_decl_index }), @@ -38098,7 +38098,7 @@ fn isKnownZigType(sema: *Sema, ref: Air.Inst.Ref, tag: std.builtin.TypeId) bool pub fn declareDependency(sema: *Sema, dependee: InternPool.Dependee) !void { const zcu = sema.pt.zcu; - if (!zcu.comp.debug_incremental) return; + if (!zcu.comp.incremental) return; // Avoid creating dependencies on ourselves. This situation can arise when we analyze the fields // of a type and they use `@This()`. This dependency would be unnecessary, and in fact would diff --git a/src/Zcu.zig b/src/Zcu.zig index 15f418c6fe7f..36b4c955718f 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -2679,7 +2679,7 @@ fn markTransitiveDependersPotentiallyOutdated(zcu: *Zcu, maybe_outdated: AnalUni } pub fn findOutdatedToAnalyze(zcu: *Zcu) Allocator.Error!?AnalUnit { - if (!zcu.comp.debug_incremental) return null; + if (!zcu.comp.incremental) return null; if (zcu.outdated.count() == 0 and zcu.potentially_outdated.count() == 0) { log.debug("findOutdatedToAnalyze: no outdated depender", .{}); diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index 59b6b6bf0bb5..c3f569cc7d06 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -888,7 +888,7 @@ fn getFileRootStruct( }; errdefer wip_ty.cancel(ip, pt.tid); - if (zcu.comp.debug_incremental) { + if (zcu.comp.incremental) { try ip.addDependency( gpa, InternPool.AnalUnit.wrap(.{ .decl = decl_index }), @@ -1418,6 +1418,10 @@ pub fn importPkg(pt: Zcu.PerThread, mod: *Module) !Zcu.ImportFileResult { const sub_file_path = try gpa.dupe(u8, mod.root_src_path); errdefer gpa.free(sub_file_path); + const comp = zcu.comp; + if (comp.file_system_inputs) |fsi| + try comp.appendFileSystemInput(fsi, mod.root, sub_file_path); + const new_file = try gpa.create(Zcu.File); errdefer gpa.destroy(new_file); @@ -1527,6 +1531,10 @@ pub fn importFile( resolved_root_path, resolved_path, sub_file_path, import_string, }); + const comp = zcu.comp; + if (comp.file_system_inputs) |fsi| + try comp.appendFileSystemInput(fsi, mod.root, sub_file_path); + const path_digest = zcu.computePathDigest(mod, sub_file_path); const new_file_index = try ip.createFile(gpa, pt.tid, .{ .bin_digest = path_digest, diff --git a/src/main.zig b/src/main.zig index 06c20d514c61..2b36c31865a5 100644 --- a/src/main.zig +++ b/src/main.zig @@ -404,6 +404,8 @@ const usage_build_generic = \\ -h, --help Print this help and exit \\ --color [auto|off|on] Enable or disable colored error messages \\ -j Limit concurrent jobs (default is to use all CPU cores) + \\ -fincremental Enable incremental compilation + \\ -fno-incremental Disable incremental compilation \\ -femit-bin[=path] (default) Output machine code \\ -fno-emit-bin Do not output machine code \\ -femit-asm[=path] Output .s (assembly code) @@ -642,7 +644,6 @@ const usage_build_generic = \\ --debug-log [scope] Enable printing debug/info log messages for scope \\ --debug-compile-errors Crash with helpful diagnostics at the first compile error \\ --debug-link-snapshot Enable dumping of the linker's state in JSON format - \\ --debug-incremental Enable experimental feature: incremental compilation \\ ; @@ -904,7 +905,7 @@ fn buildOutputType( var minor_subsystem_version: ?u16 = null; var mingw_unicode_entry_point: bool = false; var enable_link_snapshots: bool = false; - var debug_incremental: bool = false; + var opt_incremental: ?bool = null; var install_name: ?[]const u8 = null; var hash_style: link.File.Elf.HashStyle = .both; var entitlements: ?[]const u8 = null; @@ -1357,8 +1358,10 @@ fn buildOutputType( } else { enable_link_snapshots = true; } - } else if (mem.eql(u8, arg, "--debug-incremental")) { - debug_incremental = true; + } else if (mem.eql(u8, arg, "-fincremental")) { + opt_incremental = true; + } else if (mem.eql(u8, arg, "-fno-incremental")) { + opt_incremental = false; } else if (mem.eql(u8, arg, "--entitlements")) { entitlements = args_iter.nextOrFatal(); } else if (mem.eql(u8, arg, "-fcompiler-rt")) { @@ -3225,6 +3228,8 @@ fn buildOutputType( break :b .incremental; }; + const incremental = opt_incremental orelse false; + process.raiseFileDescriptorLimit(); var file_system_inputs: std.ArrayListUnmanaged(u8) = .{}; @@ -3336,7 +3341,7 @@ fn buildOutputType( .cache_mode = cache_mode, .subsystem = subsystem, .debug_compile_errors = debug_compile_errors, - .debug_incremental = debug_incremental, + .incremental = incremental, .enable_link_snapshots = enable_link_snapshots, .install_name = install_name, .entitlements = entitlements, @@ -3443,7 +3448,7 @@ fn buildOutputType( updateModule(comp, color, root_prog_node) catch |err| switch (err) { error.SemanticAnalyzeFail => { assert(listen == .none); - saveState(comp, debug_incremental); + saveState(comp, incremental); process.exit(1); }, else => |e| return e, @@ -3451,7 +3456,7 @@ fn buildOutputType( } if (build_options.only_c) return cleanExit(); try comp.makeBinFileExecutable(); - saveState(comp, debug_incremental); + saveState(comp, incremental); if (test_exec_args.items.len == 0 and target.ofmt == .c) default_exec_args: { // Default to using `zig run` to execute the produced .c code from `zig test`. @@ -4032,8 +4037,8 @@ fn createModule( return mod; } -fn saveState(comp: *Compilation, debug_incremental: bool) void { - if (debug_incremental) { +fn saveState(comp: *Compilation, incremental: bool) void { + if (incremental) { comp.saveState() catch |err| { warn("unable to save incremental compilation state: {s}", .{@errorName(err)}); }; diff --git a/test/standalone/cmakedefine/build.zig b/test/standalone/cmakedefine/build.zig index 3c57523373e8..50df9d0b7a2d 100644 --- a/test/standalone/cmakedefine/build.zig +++ b/test/standalone/cmakedefine/build.zig @@ -80,8 +80,8 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&wrapper_header.step); } -fn compare_headers(step: *std.Build.Step, prog_node: std.Progress.Node) !void { - _ = prog_node; +fn compare_headers(step: *std.Build.Step, options: std.Build.Step.MakeOptions) !void { + _ = options; const allocator = step.owner.allocator; const expected_fmt = "expected_{s}";