support building static self hosted compiler on macos

 * add a --system-linker-hack command line parameter to work around
   poor LLD macho code. See #1535
 * build.zig correctly handles static as well as dynamic dependencies
   when building the self hosted compiler.
   - no more unnecessary libxml2 dependency
   - a static build on macos produces a completely static self-hosted
     compiler for macos (except for libSystem as intended).

Author: andrewrk

build.zig

@@ -121,12 +121,23 @@ pub fn build(b: *Builder) !void {
     test_step.dependOn(docs_step);
 }
 
-fn dependOnLib(lib_exe_obj: var, dep: LibraryDep) void {
+fn dependOnLib(b: *Builder, lib_exe_obj: var, dep: LibraryDep) void {
     for (dep.libdirs.toSliceConst()) |lib_dir| {
         lib_exe_obj.addLibPath(lib_dir);
     }
+    const lib_dir = os.path.join(b.allocator, dep.prefix, "lib") catch unreachable;
     for (dep.system_libs.toSliceConst()) |lib| {
-        lib_exe_obj.linkSystemLibrary(lib);
+        const static_bare_name = if (mem.eql(u8, lib, "curses"))
+            ([]const u8)("libncurses.a")
+        else
+            b.fmt("lib{}.a", lib);
+        const static_lib_name = os.path.join(b.allocator, lib_dir, static_bare_name) catch unreachable;
+        const have_static = fileExists(static_lib_name) catch unreachable;
+        if (have_static) {
+            lib_exe_obj.addObjectFile(static_lib_name);
+        } else {
+            lib_exe_obj.linkSystemLibrary(lib);
+        }
     }
     for (dep.libs.toSliceConst()) |lib| {
         lib_exe_obj.addObjectFile(lib);
@@ -136,34 +147,49 @@ fn dependOnLib(lib_exe_obj: var, dep: LibraryDep) void {
     }
 }
 
+fn fileExists(filename: []const u8) !bool {
+    os.File.access(filename) catch |err| switch (err) {
+        error.PermissionDenied,
+        error.FileNotFound,
+        => return false,
+        else => return err,
+    };
+    return true;
+}
+
 fn addCppLib(b: *Builder, lib_exe_obj: var, cmake_binary_dir: []const u8, lib_name: []const u8) void {
     const lib_prefix = if (lib_exe_obj.target.isWindows()) "" else "lib";
     lib_exe_obj.addObjectFile(os.path.join(b.allocator, cmake_binary_dir, "zig_cpp", b.fmt("{}{}{}", lib_prefix, lib_name, lib_exe_obj.target.libFileExt())) catch unreachable);
 }
 
 const LibraryDep = struct.{
+    prefix: []const u8,
     libdirs: ArrayList([]const u8),
     libs: ArrayList([]const u8),
     system_libs: ArrayList([]const u8),
     includes: ArrayList([]const u8),
 };
 
 fn findLLVM(b: *Builder, llvm_config_exe: []const u8) !LibraryDep {
-    const libs_output = try b.exec([][]const u8.{
-        llvm_config_exe,
-        "--libs",
-        "--system-libs",
-    });
-    const includes_output = try b.exec([][]const u8.{
-        llvm_config_exe,
-        "--includedir",
-    });
-    const libdir_output = try b.exec([][]const u8.{
-        llvm_config_exe,
-        "--libdir",
-    });
+    const shared_mode = try b.exec([][]const u8.{ llvm_config_exe, "--shared-mode" });
+    const is_static = mem.startsWith(u8, shared_mode, "static");
+    const libs_output = if (is_static)
+        try b.exec([][]const u8.{
+            llvm_config_exe,
+            "--libfiles",
+            "--system-libs",
+        })
+    else
+        try b.exec([][]const u8.{
+            llvm_config_exe,
+            "--libs",
+        });
+    const includes_output = try b.exec([][]const u8.{ llvm_config_exe, "--includedir" });
+    const libdir_output = try b.exec([][]const u8.{ llvm_config_exe, "--libdir" });
+    const prefix_output = try b.exec([][]const u8.{ llvm_config_exe, "--prefix" });
 
     var result = LibraryDep.{
+        .prefix = mem.split(prefix_output, " \r\n").next().?,
         .libs = ArrayList([]const u8).init(b.allocator),
         .system_libs = ArrayList([]const u8).init(b.allocator),
         .includes = ArrayList([]const u8).init(b.allocator),
@@ -244,10 +270,6 @@ fn nextValue(index: *usize, build_info: []const u8) []const u8 {
 }
 
 fn configureStage2(b: *Builder, exe: var, ctx: Context) !void {
-    // This is for finding /lib/libz.a on alpine linux.
-    // TODO turn this into -Dextra-lib-path=/lib option
-    exe.addLibPath("/lib");
-
     exe.setNoRoSegment(ctx.no_rosegment);
 
     exe.addIncludeDir("src");
@@ -265,39 +287,63 @@ fn configureStage2(b: *Builder, exe: var, ctx: Context) !void {
         addCppLib(b, exe, ctx.cmake_binary_dir, "embedded_lld_coff");
         addCppLib(b, exe, ctx.cmake_binary_dir, "embedded_lld_lib");
     }
-    dependOnLib(exe, ctx.llvm);
+    dependOnLib(b, exe, ctx.llvm);
 
     if (exe.target.getOs() == builtin.Os.linux) {
-        const libstdcxx_path_padded = try b.exec([][]const u8.{
-            ctx.cxx_compiler,
-            "-print-file-name=libstdc++.a",
-        });
-        const libstdcxx_path = mem.split(libstdcxx_path_padded, "\r\n").next().?;
-        if (mem.eql(u8, libstdcxx_path, "libstdc++.a")) {
-            warn(
-                \\Unable to determine path to libstdc++.a
-                \\On Fedora, install libstdc++-static and try again.
-                \\
-            );
-            return error.RequiredLibraryNotFound;
-        }
-        exe.addObjectFile(libstdcxx_path);
+        try addCxxKnownPath(b, ctx, exe, "libstdc++.a",
+            \\Unable to determine path to libstdc++.a
+            \\On Fedora, install libstdc++-static and try again.
+            \\
+        );
 
         exe.linkSystemLibrary("pthread");
     } else if (exe.target.isDarwin()) {
-        exe.linkSystemLibrary("c++");
+        if (addCxxKnownPath(b, ctx, exe, "libgcc_eh.a", "")) {
+            // Compiler is GCC.
+            try addCxxKnownPath(b, ctx, exe, "libstdc++.a", null);
+            exe.linkSystemLibrary("pthread");
+            // TODO LLD cannot perform this link.
+            // See https://github.com/ziglang/zig/issues/1535
+            exe.enableSystemLinkerHack();
+        } else |err| switch (err) {
+            error.RequiredLibraryNotFound => {
+                // System compiler, not gcc.
+                exe.linkSystemLibrary("c++");
+            },
+            else => return err,
+        }
     }
 
     if (ctx.dia_guids_lib.len != 0) {
         exe.addObjectFile(ctx.dia_guids_lib);
     }
 
-    if (exe.target.getOs() != builtin.Os.windows) {
-        exe.linkSystemLibrary("xml2");
-    }
     exe.linkSystemLibrary("c");
 }
 
+fn addCxxKnownPath(
+    b: *Builder,
+    ctx: Context,
+    exe: var,
+    objname: []const u8,
+    errtxt: ?[]const u8,
+) !void {
+    const path_padded = try b.exec([][]const u8.{
+        ctx.cxx_compiler,
+        b.fmt("-print-file-name={}", objname),
+    });
+    const path_unpadded = mem.split(path_padded, "\r\n").next().?;
+    if (mem.eql(u8, path_unpadded, objname)) {
+        if (errtxt) |msg| {
+            warn("{}", msg);
+        } else {
+            warn("Unable to determine path to {}\n", objname);
+        }
+        return error.RequiredLibraryNotFound;
+    }
+    exe.addObjectFile(path_unpadded);
+}
+
 const Context = struct.{
     cmake_binary_dir: []const u8,
     cxx_compiler: []const u8,

src/all_types.hpp

@@ -1731,6 +1731,7 @@ struct CodeGen {
     bool generate_error_name_table;
     bool enable_cache;
     bool enable_time_report;
+    bool system_linker_hack;
 
     //////////////////////////// Participates in Input Parameter Cache Hash
     ZigList<LinkLib *> link_libs_list;

src/link.cpp

@@ -778,7 +778,8 @@ static bool darwin_version_lt(DarwinPlatform *platform, int major, int minor) {
 static void construct_linker_job_macho(LinkJob *lj) {
     CodeGen *g = lj->codegen;
 
-    lj->args.append("-error-limit=0");
+    // LLD MACH-O has no error limit option.
+    //lj->args.append("-error-limit=0");
     lj->args.append("-demangle");
 
     if (g->linker_rdynamic) {
@@ -1007,7 +1008,17 @@ void codegen_link(CodeGen *g) {
     Buf diag = BUF_INIT;
 
     codegen_add_time_event(g, "LLVM Link");
-    if (!zig_lld_link(g->zig_target.oformat, lj.args.items, lj.args.length, &diag)) {
+    if (g->system_linker_hack && g->zig_target.os == OsMacOSX) {
+        Termination term;
+        ZigList<const char *> args = {};
+        for (size_t i = 1; i < lj.args.length; i += 1) {
+            args.append(lj.args.at(i));
+        }
+        os_spawn_process("ld", args, &term);
+        if (term.how != TerminationIdClean || term.code != 0) {
+            exit(1);
+        }
+    } else if (!zig_lld_link(g->zig_target.oformat, lj.args.items, lj.args.length, &diag)) {
         fprintf(stderr, "%s\n", buf_ptr(&diag));
         exit(1);
     }

src/main.cpp

@@ -394,6 +394,7 @@ int main(int argc, char **argv) {
     ZigList<const char *> test_exec_args = {0};
     int runtime_args_start = -1;
     bool no_rosegment_workaround = false;
+    bool system_linker_hack = false;
 
     if (argc >= 2 && strcmp(argv[1], "build") == 0) {
         Buf zig_exe_path_buf = BUF_INIT;
@@ -560,6 +561,8 @@ int main(int argc, char **argv) {
                 timing_info = true;
             } else if (strcmp(arg, "--disable-pic") == 0) {
                 disable_pic = true;
+            } else if (strcmp(arg, "--system-linker-hack") == 0) {
+                system_linker_hack = true;
             } else if (strcmp(arg, "--test-cmd-bin") == 0) {
                 test_exec_args.append(nullptr);
             } else if (arg[1] == 'L' && arg[2] != 0) {
@@ -893,6 +896,7 @@ int main(int argc, char **argv) {
             g->verbose_llvm_ir = verbose_llvm_ir;
             g->verbose_cimport = verbose_cimport;
             codegen_set_errmsg_color(g, color);
+            g->system_linker_hack = system_linker_hack;
 
             for (size_t i = 0; i < lib_dirs.length; i += 1) {
                 codegen_add_lib_dir(g, lib_dirs.at(i));

src/os.cpp

@@ -103,7 +103,7 @@ static void os_spawn_process_posix(const char *exe, ZigList<const char *> &args,
     }
 
     pid_t pid;
-    int rc = posix_spawn(&pid, exe, nullptr, nullptr, const_cast<char *const*>(argv), environ);
+    int rc = posix_spawnp(&pid, exe, nullptr, nullptr, const_cast<char *const*>(argv), environ);
     if (rc != 0) {
         zig_panic("posix_spawn failed: %s", strerror(rc));
     }

std/build.zig

@@ -836,6 +836,7 @@ pub const LibExeObjStep = struct.{
     assembly_files: ArrayList([]const u8),
     packages: ArrayList(Pkg),
     build_options_contents: std.Buffer,
+    system_linker_hack: bool,
 
     // C only stuff
     source_files: ArrayList([]const u8),
@@ -930,6 +931,7 @@ pub const LibExeObjStep = struct.{
             .disable_libc = true,
             .build_options_contents = std.Buffer.initSize(builder.allocator, 0) catch unreachable,
             .c_std = Builder.CStd.C99,
+            .system_linker_hack = false,
         };
         self.computeOutFileNames();
         return self;
@@ -965,6 +967,7 @@ pub const LibExeObjStep = struct.{
             .is_zig = false,
             .linker_script = null,
             .c_std = Builder.CStd.C99,
+            .system_linker_hack = false,
 
             .root_src = undefined,
             .verbose_link = false,
@@ -1162,6 +1165,10 @@ pub const LibExeObjStep = struct.{
         self.disable_libc = disable;
     }
 
+    pub fn enableSystemLinkerHack(self: *LibExeObjStep) void {
+        self.system_linker_hack = true;
+    }
+
     fn make(step: *Step) !void {
         const self = @fieldParentPtr(LibExeObjStep, "step", step);
         return if (self.is_zig) self.makeZig() else self.makeC();
@@ -1338,6 +1345,9 @@ pub const LibExeObjStep = struct.{
         if (self.no_rosegment) {
             try zig_args.append("--no-rosegment");
         }
+        if (self.system_linker_hack) {
+            try zig_args.append("--system-linker-hack");
+        }
 
         try builder.spawnChild(zig_args.toSliceConst());
 
@@ -1646,6 +1656,7 @@ pub const TestStep = struct.{
     object_files: ArrayList([]const u8),
     no_rosegment: bool,
     output_path: ?[]const u8,
+    system_linker_hack: bool,
 
     pub fn init(builder: *Builder, root_src: []const u8) TestStep {
         const step_name = builder.fmt("test {}", root_src);
@@ -1665,6 +1676,7 @@ pub const TestStep = struct.{
             .object_files = ArrayList([]const u8).init(builder.allocator),
             .no_rosegment = false,
             .output_path = null,
+            .system_linker_hack = false,
         };
     }
 
@@ -1747,6 +1759,10 @@ pub const TestStep = struct.{
         self.exec_cmd_args = args;
     }
 
+    pub fn enableSystemLinkerHack(self: *TestStep) void {
+        self.system_linker_hack = true;
+    }
+
     fn make(step: *Step) !void {
         const self = @fieldParentPtr(TestStep, "step", step);
         const builder = self.builder;
@@ -1851,6 +1867,9 @@ pub const TestStep = struct.{
         if (self.no_rosegment) {
             try zig_args.append("--no-rosegment");
         }
+        if (self.system_linker_hack) {
+            try zig_args.append("--system-linker-hack");
+        }
 
         try builder.spawnChild(zig_args.toSliceConst());
     }