compiler_rt: use single cache for libcompiler_rt.a static lib

kubkon · andrewrk · commit 8330a95b66af · 2022-06-15T13:00:02.000-07:00
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -490,6 +490,7 @@ set(ZIG_STAGE2_SOURCES
     "${CMAKE_SOURCE_DIR}/lib/compiler_rt/ceil.zig"
     "${CMAKE_SOURCE_DIR}/lib/compiler_rt/clear_cache.zig"
     "${CMAKE_SOURCE_DIR}/lib/compiler_rt/cmp.zig"
+    "${CMAKE_SOURCE_DIR}/lib/compiler_rt/common.zig"
     "${CMAKE_SOURCE_DIR}/lib/compiler_rt/compareXf2.zig"
     "${CMAKE_SOURCE_DIR}/lib/compiler_rt/cos.zig"
     "${CMAKE_SOURCE_DIR}/lib/compiler_rt/count0bits.zig"
diff --git a/lib/compiler_rt/os_version_check.zig b/lib/compiler_rt/os_version_check.zig
@@ -6,10 +6,7 @@ pub const panic = @import("common.zig").panic;
 
 comptime {
     if (builtin.os.tag.isDarwin()) {
-        @export(IsPlatformVersionAtLeast.__isPlatformVersionAtLeast, .{
-            .name = "__isPlatformVersionAtLeast",
-            .linkage = linkage,
-        });
+        @export(__isPlatformVersionAtLeast, .{ .name = "__isPlatformVersionAtLeast", .linkage = linkage });
     }
 }
 
@@ -28,7 +25,7 @@ comptime {
 // the newer codepath, which merely calls out to the Darwin _availability_version_check API which is
 // available on macOS 10.15+, iOS 13+, tvOS 13+ and watchOS 6+.
 
-const IsPlatformVersionAtLeast = struct {
+const __isPlatformVersionAtLeast = if (builtin.os.tag.isDarwin()) struct {
     inline fn constructVersion(major: u32, minor: u32, subminor: u32) u32 {
         return ((major & 0xffff) << 16) | ((minor & 0xff) << 8) | (subminor & 0xff);
     }
@@ -50,14 +47,14 @@ const IsPlatformVersionAtLeast = struct {
     };
     // Darwin-only
     extern "c" fn _availability_version_check(count: u32, versions: [*c]const dyld_build_version_t) bool;
-};
+}.__isPlatformVersionAtLeast else struct {};
 
 test "isPlatformVersionAtLeast" {
     if (!comptime builtin.os.tag.isDarwin()) return error.SkipZigTest;
 
     // Note: this test depends on the actual host OS version since it is merely calling into the
     // native Darwin API.
     const macos_platform_constant = 1;
-    try testing.expect(IsPlatformVersionAtLeast.__isPlatformVersionAtLeast(macos_platform_constant, 10, 0, 15) == 1);
-    try testing.expect(IsPlatformVersionAtLeast.__isPlatformVersionAtLeast(macos_platform_constant, 99, 0, 0) == 0);
+    try testing.expect(__isPlatformVersionAtLeast(macos_platform_constant, 10, 0, 15) == 1);
+    try testing.expect(__isPlatformVersionAtLeast(macos_platform_constant, 99, 0, 0) == 0);
 }
diff --git a/src/Compilation.zig b/src/Compilation.zig
@@ -132,7 +132,7 @@ libssp_static_lib: ?CRTFile = null,
 libc_static_lib: ?CRTFile = null,
 /// Populated when we build the libcompiler_rt static library. A Job to build this is placed in the queue
 /// and resolved before calling linker.flush().
-compiler_rt_static_lib: compiler_rt.CompilerRtLib = .{},
+compiler_rt_lib: ?CRTFile = null,
 /// Populated when we build the compiler_rt_obj object. A Job to build this is placed in the queue
 /// and resolved before calling linker.flush().
 compiler_rt_obj: ?CRTFile = null,
@@ -1979,7 +1979,9 @@ pub fn destroy(self: *Compilation) void {
     if (self.libcxxabi_static_lib) |*crt_file| {
         crt_file.deinit(gpa);
     }
-    self.compiler_rt_static_lib.deinit(gpa);
+    if (self.compiler_rt_lib) |*crt_file| {
+        crt_file.deinit(gpa);
+    }
     if (self.compiler_rt_obj) |*crt_file| {
         crt_file.deinit(gpa);
     }
@@ -3139,7 +3141,7 @@ fn processOneJob(comp: *Compilation, job: Job) !void {
 
             compiler_rt.buildCompilerRtLib(
                 comp,
-                &comp.compiler_rt_static_lib,
+                &comp.compiler_rt_lib,
             ) catch |err| switch (err) {
                 error.OutOfMemory => return error.OutOfMemory,
                 error.SubCompilationFailed => return, // error reported already
diff --git a/src/compiler_rt.zig b/src/compiler_rt.zig
@@ -1,123 +1,169 @@
 const std = @import("std");
 const builtin = @import("builtin");
+const build_options = @import("build_options");
 const Allocator = std.mem.Allocator;
+const assert = std.debug.assert;
 const mem = std.mem;
 const tracy = @import("tracy.zig");
 const trace = tracy.trace;
 
+const Cache = @import("Cache.zig");
 const Compilation = @import("Compilation.zig");
 const CRTFile = Compilation.CRTFile;
 const LinkObject = Compilation.LinkObject;
 const Package = @import("Package.zig");
 
-pub const CompilerRtLib = struct {
-    crt_object_files: [sources.len]?CRTFile = undefined,
-    crt_lib_file: ?CRTFile = null,
-
-    pub fn deinit(crt_lib: *CompilerRtLib, gpa: Allocator) void {
-        for (crt_lib.crt_object_files) |*crt_file| {
-            if (crt_file.*) |*cf| {
-                cf.deinit(gpa);
-            }
-        }
-        if (crt_lib.crt_lib_file) |*crt_file| {
-            crt_file.deinit(gpa);
-        }
-    }
-};
-
-pub fn buildCompilerRtLib(comp: *Compilation, compiler_rt_lib: *CompilerRtLib) !void {
+pub fn buildCompilerRtLib(comp: *Compilation, compiler_rt_lib: *?CRTFile) !void {
     const tracy_trace = trace(@src());
     defer tracy_trace.end();
 
-    var progress: std.Progress = .{ .dont_print_on_dumb = true };
-    var progress_node = progress.start("Compile Compiler-RT", sources.len + 1);
-    defer progress_node.end();
-    if (comp.color == .off) progress.terminal = null;
+    var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
+    defer arena_allocator.deinit();
+    const arena = arena_allocator.allocator();
 
-    progress_node.activate();
+    const target = comp.getTarget();
 
-    var link_objects: [sources.len]LinkObject = undefined;
-    for (sources) |source, i| {
-        var obj_progress_node = progress_node.start(source, 0);
-        obj_progress_node.activate();
-        defer obj_progress_node.end();
+    // Use the global cache directory.
+    var cache_parent: Cache = .{
+        .gpa = comp.gpa,
+        .manifest_dir = try comp.global_cache_directory.handle.makeOpenPath("h", .{}),
+    };
+    defer cache_parent.manifest_dir.close();
 
-        try comp.buildOutputFromZig(source, .Obj, &compiler_rt_lib.crt_object_files[i], .compiler_rt);
-        link_objects[i] = .{
-            .path = compiler_rt_lib.crt_object_files[i].?.full_object_path,
-            .must_link = true,
-        };
+    var cache = cache_parent.obtain();
+    defer cache.deinit();
+
+    cache.hash.add(sources.len);
+    for (sources) |source| {
+        const full_path = try comp.zig_lib_directory.join(arena, &[_][]const u8{source});
+        _ = try cache.addFile(full_path, null);
     }
 
-    const root_name = "compiler_rt";
+    cache.hash.addBytes(build_options.version);
+    cache.hash.addBytes(comp.zig_lib_directory.path orelse ".");
+    cache.hash.add(target.cpu.arch);
+    cache.hash.add(target.os.tag);
+    cache.hash.add(target.abi);
 
-    var lib_progress_node = progress_node.start(root_name, 0);
-    lib_progress_node.activate();
-    defer lib_progress_node.end();
+    const hit = try cache.hit();
+    const digest = cache.final();
+    const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest });
 
-    const target = comp.getTarget();
-    const basename = try std.zig.binNameAlloc(comp.gpa, .{
+    var o_directory: Compilation.Directory = .{
+        .handle = try comp.global_cache_directory.handle.makeOpenPath(o_sub_path, .{}),
+        .path = try std.fs.path.join(arena, &[_][]const u8{ comp.global_cache_directory.path.?, o_sub_path }),
+    };
+    defer o_directory.handle.close();
+
+    const ok_basename = "ok";
+    const actual_hit = if (hit) blk: {
+        o_directory.handle.access(ok_basename, .{}) catch |err| switch (err) {
+            error.FileNotFound => break :blk false,
+            else => |e| return e,
+        };
+        break :blk true;
+    } else false;
+
+    const root_name = "compiler_rt";
+    const basename = try std.zig.binNameAlloc(arena, .{
         .root_name = root_name,
         .target = target,
         .output_mode = .Lib,
     });
-    errdefer comp.gpa.free(basename);
 
-    // TODO: This is extracted into a local variable to work around a stage1 miscompilation.
-    const emit_bin = Compilation.EmitLoc{
-        .directory = null, // Put it in the cache directory.
-        .basename = basename,
-    };
-    const sub_compilation = try Compilation.create(comp.gpa, .{
-        .local_cache_directory = comp.global_cache_directory,
-        .global_cache_directory = comp.global_cache_directory,
-        .zig_lib_directory = comp.zig_lib_directory,
-        .cache_mode = .whole,
-        .target = target,
-        .root_name = root_name,
-        .main_pkg = null,
-        .output_mode = .Lib,
-        .link_mode = .Static,
-        .function_sections = true,
-        .thread_pool = comp.thread_pool,
-        .libc_installation = comp.bin_file.options.libc_installation,
-        .emit_bin = emit_bin,
-        .optimize_mode = comp.compilerRtOptMode(),
-        .want_sanitize_c = false,
-        .want_stack_check = false,
-        .want_red_zone = comp.bin_file.options.red_zone,
-        .omit_frame_pointer = comp.bin_file.options.omit_frame_pointer,
-        .want_valgrind = false,
-        .want_tsan = false,
-        .want_pic = comp.bin_file.options.pic,
-        .want_pie = comp.bin_file.options.pie,
-        .want_lto = comp.bin_file.options.lto,
-        .emit_h = null,
-        .strip = comp.compilerRtStrip(),
-        .is_native_os = comp.bin_file.options.is_native_os,
-        .is_native_abi = comp.bin_file.options.is_native_abi,
-        .self_exe_path = comp.self_exe_path,
-        .link_objects = &link_objects,
-        .verbose_cc = comp.verbose_cc,
-        .verbose_link = comp.bin_file.options.verbose_link,
-        .verbose_air = comp.verbose_air,
-        .verbose_llvm_ir = comp.verbose_llvm_ir,
-        .verbose_cimport = comp.verbose_cimport,
-        .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
-        .clang_passthrough_mode = comp.clang_passthrough_mode,
-        .skip_linker_dependencies = true,
-        .parent_compilation_link_libc = comp.bin_file.options.link_libc,
-    });
-    defer sub_compilation.destroy();
+    if (!actual_hit) {
+        var progress: std.Progress = .{ .dont_print_on_dumb = true };
+        var progress_node = progress.start("Compile Compiler-RT", sources.len + 1);
+        defer progress_node.end();
+        if (comp.color == .off) progress.terminal = null;
+
+        progress_node.activate();
+
+        var link_objects: [sources.len]LinkObject = undefined;
+        for (sources) |source, i| {
+            var obj_progress_node = progress_node.start(source, 0);
+            obj_progress_node.activate();
+            defer obj_progress_node.end();
+
+            var tmp_crt_file: ?CRTFile = null;
+            defer if (tmp_crt_file) |*crt| crt.deinit(comp.gpa);
+            try comp.buildOutputFromZig(source, .Obj, &tmp_crt_file, .compiler_rt);
+            link_objects[i] = .{
+                .path = try arena.dupe(u8, tmp_crt_file.?.full_object_path),
+                .must_link = true,
+            };
+        }
+
+        var lib_progress_node = progress_node.start(root_name, 0);
+        lib_progress_node.activate();
+        defer lib_progress_node.end();
+
+        // TODO: This is extracted into a local variable to work around a stage1 miscompilation.
+        const emit_bin = Compilation.EmitLoc{
+            .directory = o_directory, // Put it in the cache directory.
+            .basename = basename,
+        };
+        const sub_compilation = try Compilation.create(comp.gpa, .{
+            .local_cache_directory = comp.global_cache_directory,
+            .global_cache_directory = comp.global_cache_directory,
+            .zig_lib_directory = comp.zig_lib_directory,
+            .cache_mode = .whole,
+            .target = target,
+            .root_name = root_name,
+            .main_pkg = null,
+            .output_mode = .Lib,
+            .link_mode = .Static,
+            .function_sections = true,
+            .thread_pool = comp.thread_pool,
+            .libc_installation = comp.bin_file.options.libc_installation,
+            .emit_bin = emit_bin,
+            .optimize_mode = comp.compilerRtOptMode(),
+            .want_sanitize_c = false,
+            .want_stack_check = false,
+            .want_red_zone = comp.bin_file.options.red_zone,
+            .omit_frame_pointer = comp.bin_file.options.omit_frame_pointer,
+            .want_valgrind = false,
+            .want_tsan = false,
+            .want_pic = comp.bin_file.options.pic,
+            .want_pie = comp.bin_file.options.pie,
+            .want_lto = comp.bin_file.options.lto,
+            .emit_h = null,
+            .strip = comp.compilerRtStrip(),
+            .is_native_os = comp.bin_file.options.is_native_os,
+            .is_native_abi = comp.bin_file.options.is_native_abi,
+            .self_exe_path = comp.self_exe_path,
+            .link_objects = &link_objects,
+            .verbose_cc = comp.verbose_cc,
+            .verbose_link = comp.bin_file.options.verbose_link,
+            .verbose_air = comp.verbose_air,
+            .verbose_llvm_ir = comp.verbose_llvm_ir,
+            .verbose_cimport = comp.verbose_cimport,
+            .verbose_llvm_cpu_features = comp.verbose_llvm_cpu_features,
+            .clang_passthrough_mode = comp.clang_passthrough_mode,
+            .skip_linker_dependencies = true,
+            .parent_compilation_link_libc = comp.bin_file.options.link_libc,
+        });
+        defer sub_compilation.destroy();
+
+        try sub_compilation.updateSubCompilation();
+
+        if (o_directory.handle.createFile(ok_basename, .{})) |file| {
+            file.close();
+        } else |err| {
+            std.log.warn("compiler-rt lib: failed to mark completion: {s}", .{@errorName(err)});
+        }
+    }
 
-    try sub_compilation.updateSubCompilation();
+    try cache.writeManifest();
 
-    compiler_rt_lib.crt_lib_file = .{
-        .full_object_path = try sub_compilation.bin_file.options.emit.?.directory.join(comp.gpa, &[_][]const u8{
-            sub_compilation.bin_file.options.emit.?.sub_path,
+    assert(compiler_rt_lib.* == null);
+    compiler_rt_lib.* = .{
+        .full_object_path = try std.fs.path.join(comp.gpa, &[_][]const u8{
+            comp.global_cache_directory.path.?,
+            o_sub_path,
+            basename,
         }),
-        .lock = sub_compilation.bin_file.toOwnedLock(),
+        .lock = cache.toOwnedLock(),
     };
 }
 
diff --git a/src/link/Coff.zig b/src/link/Coff.zig
@@ -1354,7 +1354,7 @@ fn linkWithLLD(self: *Coff, comp: *Compilation, prog_node: *std.Progress.Node) !
             }
             // MSVC compiler_rt is missing some stuff, so we build it unconditionally but
             // and rely on weak linkage to allow MSVC compiler_rt functions to override ours.
-            if (comp.compiler_rt_static_lib.crt_lib_file) |lib| {
+            if (comp.compiler_rt_lib) |lib| {
                 try argv.append(lib.full_object_path);
             }
         }
diff --git a/src/link/Elf.zig b/src/link/Elf.zig
@@ -1272,7 +1272,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v
     const stack_size = self.base.options.stack_size_override orelse 16777216;
     const allow_shlib_undefined = self.base.options.allow_shlib_undefined orelse !self.base.options.is_native_os;
     const compiler_rt_path: ?[]const u8 = blk: {
-        if (comp.compiler_rt_static_lib.crt_lib_file) |x| break :blk x.full_object_path;
+        if (comp.compiler_rt_lib) |x| break :blk x.full_object_path;
         if (comp.compiler_rt_obj) |x| break :blk x.full_object_path;
         break :blk null;
     };
diff --git a/src/link/MachO.zig b/src/link/MachO.zig
@@ -738,7 +738,7 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No
                 try positionals.append(p);
             }
 
-            if (comp.compiler_rt_static_lib.crt_lib_file) |lib| {
+            if (comp.compiler_rt_lib) |lib| {
                 try positionals.append(lib.full_object_path);
             }
 
diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig
@@ -2255,7 +2255,7 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) !
     const is_obj = self.base.options.output_mode == .Obj;
 
     const compiler_rt_path: ?[]const u8 = if (self.base.options.include_compiler_rt and !is_obj)
-        comp.compiler_rt_static_lib.crt_lib_file.?.full_object_path
+        comp.compiler_rt_lib.?.full_object_path
     else
         null;
 

Original file line number	Diff line number	Diff line change
`@@ -1354,7 +1354,7 @@ fn linkWithLLD(self: Coff, comp: Compilation, prog_node: *std.Progress.Node) !`
`1354`	`1354`	`}`
`1355`	`1355`	`// MSVC compiler_rt is missing some stuff, so we build it unconditionally but`
`1356`	`1356`	`// and rely on weak linkage to allow MSVC compiler_rt functions to override ours.`
`1357`		`- if (comp.compiler_rt_static_lib.crt_lib_file) \|lib\| {`
	`1357`	`+ if (comp.compiler_rt_lib) \|lib\| {`
`1358`	`1358`	`try argv.append(lib.full_object_path);`
`1359`	`1359`	`}`
`1360`	`1360`	`}`
Original file line number	Diff line number	Diff line change
`@@ -738,7 +738,7 @@ pub fn flushModule(self: MachO, comp: Compilation, prog_node: *std.Progress.No`
`738`	`738`	`try positionals.append(p);`
`739`	`739`	`}`
`740`	`740`
`741`		`- if (comp.compiler_rt_static_lib.crt_lib_file) \|lib\| {`
	`741`	`+ if (comp.compiler_rt_lib) \|lib\| {`
`742`	`742`	`try positionals.append(lib.full_object_path);`
`743`	`743`	`}`
`744`	`744`