Merge pull request #872 from zig-lang/runtime-libc

find libc and zig std lib at runtime

Author: andrewrk

CMakeLists.txt

@@ -30,11 +30,6 @@ if(GIT_EXE)
 endif()
 message("Configuring zig version ${ZIG_VERSION}")
 
-set(ZIG_LIBC_LIB_DIR "" CACHE STRING "Default native target libc directory where crt1.o can be found")
-set(ZIG_LIBC_STATIC_LIB_DIR "" CACHE STRING "Default native target libc directory where crtbeginT.o can be found")
-set(ZIG_LIBC_INCLUDE_DIR "/usr/include" CACHE STRING "Default native target libc include directory")
-set(ZIG_DYNAMIC_LINKER "" CACHE STRING "Override dynamic linker for native target")
-set(ZIG_EACH_LIB_RPATH off CACHE BOOL "Add each dynamic library to rpath for native target")
 set(ZIG_STATIC off CACHE BOOL "Attempt to build a static zig executable (not compatible with glibc)")
 
 string(REGEX REPLACE "\\\\" "\\\\\\\\" ZIG_LIBC_LIB_DIR_ESCAPED "${ZIG_LIBC_LIB_DIR}")

README.md

@@ -138,23 +138,17 @@ libc. Create demo games using Zig.
 
 ##### POSIX
 
-If you have gcc or clang installed, you can find out what `ZIG_LIBC_LIB_DIR`,
-`ZIG_LIBC_STATIC_LIB_DIR`, and `ZIG_LIBC_INCLUDE_DIR` should be set to
-(example below).
-
 ```
 mkdir build
 cd build
-cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) -DZIG_LIBC_LIB_DIR=$(dirname $(cc -print-file-name=crt1.o)) -DZIG_LIBC_INCLUDE_DIR=$(echo -n | cc -E -x c - -v 2>&1 | grep -B1 "End of search list." | head -n1 | cut -c 2- | sed "s/ .*//") -DZIG_LIBC_STATIC_LIB_DIR=$(dirname $(cc -print-file-name=crtbegin.o))
+cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd)
 make
 make install
 ./zig build --build-file ../build.zig test
 ```
 
 ##### MacOS
 
-`ZIG_LIBC_LIB_DIR` and `ZIG_LIBC_STATIC_LIB_DIR` are unused.
-
 ```
 brew install cmake llvm@6
 brew outdated llvm@6 || brew upgrade llvm@6

ci/appveyor/build_script.bat

@@ -20,9 +20,7 @@ call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x86_
 
 mkdir %ZIGBUILDDIR%
 cd %ZIGBUILDDIR%
-cmake.exe .. -Thost=x64 -G"Visual Studio 14 2015 Win64" "-DCMAKE_INSTALL_PREFIX=%ZIGBUILDDIR%" "-DCMAKE_PREFIX_PATH=%ZIGPREFIXPATH%" -DCMAKE_BUILD_TYPE=Release "-DZIG_LIBC_INCLUDE_DIR=C:\Program Files (x86)\Windows Kits\10\Include\10.0.10240.0\ucrt" "-DZIG_LIBC_LIB_DIR=C:\Program Files (x86)\Windows Kits\10\bin\x64\ucrt" "-DZIG_LIBC_STATIC_LIB_DIR=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64" || exit /b
+cmake.exe .. -Thost=x64 -G"Visual Studio 14 2015 Win64" "-DCMAKE_INSTALL_PREFIX=%ZIGBUILDDIR%" "-DCMAKE_PREFIX_PATH=%ZIGPREFIXPATH%" -DCMAKE_BUILD_TYPE=Release || exit /b
 msbuild /p:Configuration=Release INSTALL.vcxproj || exit /b
 
 bin\zig.exe build --build-file ..\build.zig test || exit /b
-
-@echo "MSVC build succeeded"

ci/travis_linux_script

@@ -8,7 +8,7 @@ export CXX=clang++-6.0
 echo $PATH
 mkdir build
 cd build
-cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd) -DZIG_LIBC_LIB_DIR=$(dirname $($CC -print-file-name=crt1.o)) -DZIG_LIBC_INCLUDE_DIR=$(echo -n | $CC -E -x c - -v 2>&1 | grep -B1 "End of search list." | head -n1 | cut -c 2- | sed "s/ .*//") -DZIG_LIBC_STATIC_LIB_DIR=$(dirname $($CC -print-file-name=crtbegin.o))
+cmake .. -DCMAKE_INSTALL_PREFIX=$(pwd)
 make VERBOSE=1
 make install
 ./zig build --build-file ../build.zig test

src/analyze.cpp

@@ -4285,24 +4285,118 @@ static ZigWindowsSDK *get_windows_sdk(CodeGen *g) {
     return g->win_sdk;
 }
 
+
+Buf *get_linux_libc_lib_path(const char *o_file) {
+    const char *cc_exe = getenv("CC");
+    cc_exe = (cc_exe == nullptr) ? "cc" : cc_exe;
+    ZigList<const char *> args = {};
+    args.append(buf_ptr(buf_sprintf("-print-file-name=%s", o_file)));
+    Termination term;
+    Buf *out_stderr = buf_alloc();
+    Buf *out_stdout = buf_alloc();
+    int err;
+    if ((err = os_exec_process(cc_exe, args, &term, out_stderr, out_stdout))) {
+        zig_panic("unable to determine libc lib path: executing C compiler: %s", err_str(err));
+    }
+    if (term.how != TerminationIdClean || term.code != 0) {
+        zig_panic("unable to determine libc lib path: executing C compiler command failed");
+    }
+    if (buf_ends_with_str(out_stdout, "\n")) {
+        buf_resize(out_stdout, buf_len(out_stdout) - 1);
+    }
+    if (buf_len(out_stdout) == 0 || buf_eql_str(out_stdout, o_file)) {
+        zig_panic("unable to determine libc lib path: C compiler could not find %s", o_file);
+    }
+    Buf *result = buf_alloc();
+    os_path_dirname(out_stdout, result);
+    return result;
+}
+
+Buf *get_linux_libc_include_path(void) {
+    const char *cc_exe = getenv("CC");
+    cc_exe = (cc_exe == nullptr) ? "cc" : cc_exe;
+    ZigList<const char *> args = {};
+    args.append("-E");
+    args.append("-Wp,-v");
+    args.append("-xc");
+    args.append("/dev/null");
+    Termination term;
+    Buf *out_stderr = buf_alloc();
+    Buf *out_stdout = buf_alloc();
+    int err;
+    if ((err = os_exec_process(cc_exe, args, &term, out_stderr, out_stdout))) {
+        zig_panic("unable to determine libc include path: executing C compiler: %s", err_str(err));
+    }
+    if (term.how != TerminationIdClean || term.code != 0) {
+        zig_panic("unable to determine libc include path: executing C compiler command failed");
+    }
+    char *prev_newline = buf_ptr(out_stderr);
+    ZigList<const char *> search_paths = {};
+    bool found_search_paths = false;
+    for (;;) {
+        char *newline = strchr(prev_newline, '\n');
+        if (newline == nullptr) {
+            zig_panic("unable to determine libc include path: bad output from C compiler command");
+        }
+        *newline = 0;
+        if (found_search_paths) {
+            if (strcmp(prev_newline, "End of search list.") == 0) {
+                break;
+            }
+            search_paths.append(prev_newline);
+        } else {
+            if (strcmp(prev_newline, "#include <...> search starts here:") == 0) {
+                found_search_paths = true;
+            }
+        }
+        prev_newline = newline + 1;
+    }
+    if (search_paths.length == 0) {
+        zig_panic("unable to determine libc include path: even C compiler does not know where libc headers are");
+    }
+    for (size_t i = 0; i < search_paths.length; i += 1) {
+        // search in reverse order
+        const char *search_path = search_paths.items[search_paths.length - i - 1];
+        // cut off spaces
+        while (*search_path == ' ') {
+            search_path += 1;
+        }
+        Buf *stdlib_path = buf_sprintf("%s/stdlib.h", search_path);
+        bool exists;
+        if ((err = os_file_exists(stdlib_path, &exists))) {
+            exists = false;
+        }
+        if (exists) {
+            return buf_create_from_str(search_path);
+        }
+    }
+    zig_panic("unable to determine libc include path: stdlib.h not found in C compiler search paths");
+}
+
 void find_libc_include_path(CodeGen *g) {
-    if (!g->libc_include_dir || buf_len(g->libc_include_dir) == 0) {
+    if (g->libc_include_dir == nullptr) {
 
         if (g->zig_target.os == OsWindows) {
             ZigWindowsSDK *sdk = get_windows_sdk(g);
+            g->libc_include_dir = buf_alloc();
             if (os_get_win32_ucrt_include_path(sdk, g->libc_include_dir)) {
                 zig_panic("Unable to determine libc include path.");
             }
+        } else if (g->zig_target.os == OsLinux) {
+            g->libc_include_dir = get_linux_libc_include_path();
+        } else if (g->zig_target.os == OsMacOSX) {
+            g->libc_include_dir = buf_create_from_str("/usr/include");
+        } else {
+            // TODO find libc at runtime for other operating systems
+            zig_panic("Unable to determine libc include path.");
         }
-
-        // TODO find libc at runtime for other operating systems
-        zig_panic("Unable to determine libc include path.");
     }
+    assert(buf_len(g->libc_include_dir) != 0);
 }
 
 void find_libc_lib_path(CodeGen *g) {
     // later we can handle this better by reporting an error via the normal mechanism
-    if (!g->libc_lib_dir || buf_len(g->libc_lib_dir) == 0 ||
+    if (g->libc_lib_dir == nullptr ||
         (g->zig_target.os == OsWindows && (g->msvc_lib_dir == nullptr || g->kernel32_lib_dir == nullptr)))
     {
         if (g->zig_target.os == OsWindows) {
@@ -4326,18 +4420,25 @@ void find_libc_lib_path(CodeGen *g) {
             g->msvc_lib_dir = vc_lib_dir;
             g->libc_lib_dir = ucrt_lib_path;
             g->kernel32_lib_dir = kern_lib_path;
+        } else if (g->zig_target.os == OsLinux) {
+            g->libc_lib_dir = get_linux_libc_lib_path("crt1.o");
         } else {
             zig_panic("Unable to determine libc lib path.");
         }
+    } else {
+        assert(buf_len(g->libc_lib_dir) != 0);
     }
 
-    if (!g->libc_static_lib_dir || buf_len(g->libc_static_lib_dir) == 0) {
+    if (g->libc_static_lib_dir == nullptr) {
         if ((g->zig_target.os == OsWindows) && (g->msvc_lib_dir != NULL)) {
             return;
-        }
-        else {
+        } else if (g->zig_target.os == OsLinux) {
+            g->libc_static_lib_dir = get_linux_libc_lib_path("crtbegin.o");
+        } else {
             zig_panic("Unable to determine libc static lib path.");
         }
+    } else {
+        assert(buf_len(g->libc_static_lib_dir) != 0);
     }
 }
 

src/codegen.cpp

@@ -112,27 +112,24 @@ CodeGen *codegen_create(Buf *root_src_path, const ZigTarget *target, OutType out
         // that's for native compilation
         g->zig_target = *target;
         resolve_target_object_format(&g->zig_target);
-        g->dynamic_linker = buf_create_from_str("");
-        g->libc_lib_dir = buf_create_from_str("");
-        g->libc_static_lib_dir = buf_create_from_str("");
-        g->libc_include_dir = buf_create_from_str("");
+        g->dynamic_linker = nullptr;
+        g->libc_lib_dir = nullptr;
+        g->libc_static_lib_dir = nullptr;
+        g->libc_include_dir = nullptr;
         g->msvc_lib_dir = nullptr;
         g->kernel32_lib_dir = nullptr;
         g->each_lib_rpath = false;
     } else {
         // native compilation, we can rely on the configuration stuff
         g->is_native_target = true;
         get_native_target(&g->zig_target);
-        g->dynamic_linker = buf_create_from_str(ZIG_DYNAMIC_LINKER);
-        g->libc_lib_dir = buf_create_from_str(ZIG_LIBC_LIB_DIR);
-        g->libc_static_lib_dir = buf_create_from_str(ZIG_LIBC_STATIC_LIB_DIR);
-        g->libc_include_dir = buf_create_from_str(ZIG_LIBC_INCLUDE_DIR);
+        g->dynamic_linker = nullptr; // find it at runtime
+        g->libc_lib_dir = nullptr; // find it at runtime
+        g->libc_static_lib_dir = nullptr; // find it at runtime
+        g->libc_include_dir = nullptr; // find it at runtime
         g->msvc_lib_dir = nullptr; // find it at runtime
         g->kernel32_lib_dir = nullptr; // find it at runtime
-
-#ifdef ZIG_EACH_LIB_RPATH
         g->each_lib_rpath = true;
-#endif
 
         if (g->zig_target.os == OsMacOSX ||
             g->zig_target.os == OsIOS)

src/config.h.in

@@ -13,14 +13,6 @@
 #define ZIG_VERSION_PATCH @ZIG_VERSION_PATCH@
 #define ZIG_VERSION_STRING "@ZIG_VERSION@"
 
-#define ZIG_INSTALL_PREFIX "@CMAKE_INSTALL_PREFIX@"
-#define ZIG_LIBC_INCLUDE_DIR "@ZIG_LIBC_INCLUDE_DIR_ESCAPED@"
-#define ZIG_LIBC_LIB_DIR "@ZIG_LIBC_LIB_DIR_ESCAPED@"
-#define ZIG_LIBC_STATIC_LIB_DIR "@ZIG_LIBC_STATIC_LIB_DIR_ESCAPED@"
-#define ZIG_DYNAMIC_LINKER "@ZIG_DYNAMIC_LINKER@"
-
-#cmakedefine ZIG_EACH_LIB_RPATH
-
 // Only used for running tests before installing.
 #define ZIG_TEST_DIR "@CMAKE_SOURCE_DIR@/test"
 

src/link.cpp

@@ -164,6 +164,34 @@ static void add_rpath(LinkJob *lj, Buf *rpath) {
     lj->rpath_table.put(rpath, true);
 }
 
+static Buf *get_dynamic_linker_path(CodeGen *g) {
+    if (g->is_native_target && g->zig_target.arch.arch == ZigLLVM_x86_64) {
+        const char *cc_exe = getenv("CC");
+        cc_exe = (cc_exe == nullptr) ? "cc" : cc_exe;
+        ZigList<const char *> args = {};
+        args.append("-print-file-name=ld-linux-x86-64.so.2");
+        Termination term;
+        Buf *out_stderr = buf_alloc();
+        Buf *out_stdout = buf_alloc();
+        int err;
+        if ((err = os_exec_process(cc_exe, args, &term, out_stderr, out_stdout))) {
+            return target_dynamic_linker(&g->zig_target);
+        }
+        if (term.how != TerminationIdClean || term.code != 0) {
+            return target_dynamic_linker(&g->zig_target);
+        }
+        if (buf_ends_with_str(out_stdout, "\n")) {
+            buf_resize(out_stdout, buf_len(out_stdout) - 1);
+        }
+        if (buf_len(out_stdout) == 0 || buf_eql_str(out_stdout, "ld-linux-x86-64.so.2")) {
+            return target_dynamic_linker(&g->zig_target);
+        }
+        return out_stdout;
+    } else {
+        return target_dynamic_linker(&g->zig_target);
+    }
+}
+
 static void construct_linker_job_elf(LinkJob *lj) {
     CodeGen *g = lj->codegen;
 
@@ -259,12 +287,16 @@ static void construct_linker_job_elf(LinkJob *lj) {
         lj->args.append(buf_ptr(g->libc_static_lib_dir));
     }
 
-    if (g->dynamic_linker && buf_len(g->dynamic_linker) > 0) {
-        lj->args.append("-dynamic-linker");
-        lj->args.append(buf_ptr(g->dynamic_linker));
-    } else {
-        lj->args.append("-dynamic-linker");
-        lj->args.append(buf_ptr(target_dynamic_linker(&g->zig_target)));
+    if (!g->is_static) {
+        if (g->dynamic_linker != nullptr) {
+            assert(buf_len(g->dynamic_linker) != 0);
+            lj->args.append("-dynamic-linker");
+            lj->args.append(buf_ptr(g->dynamic_linker));
+        } else {
+            Buf *resolved_dynamic_linker = get_dynamic_linker_path(g);
+            lj->args.append("-dynamic-linker");
+            lj->args.append(buf_ptr(resolved_dynamic_linker));
+        }
     }
 
     if (shared) {
@@ -423,7 +455,9 @@ static void construct_linker_job_coff(LinkJob *lj) {
         lj->args.append(buf_ptr(buf_sprintf("-LIBPATH:%s", buf_ptr(g->kernel32_lib_dir))));
 
         lj->args.append(buf_ptr(buf_sprintf("-LIBPATH:%s", buf_ptr(g->libc_lib_dir))));
-        lj->args.append(buf_ptr(buf_sprintf("-LIBPATH:%s", buf_ptr(g->libc_static_lib_dir))));
+        if (g->libc_static_lib_dir != nullptr) {
+            lj->args.append(buf_ptr(buf_sprintf("-LIBPATH:%s", buf_ptr(g->libc_static_lib_dir))));
+        }
     }
 
     if (lj->link_in_crt) {

src/main.cpp

@@ -195,13 +195,6 @@ static int find_zig_lib_dir(Buf *out_path) {
         }
     }
 
-    if (ZIG_INSTALL_PREFIX != nullptr) {
-        if (test_zig_install_prefix(buf_create_from_str(ZIG_INSTALL_PREFIX), out_path)) {
-            return 0;
-        }
-    }
-
-
     return ErrorFileNotFound;
 }
 

src/os.cpp

@@ -45,6 +45,7 @@ typedef SSIZE_T ssize_t;
 #if defined(__MACH__)
 #include <mach/clock.h>
 #include <mach/mach.h>
+#include <mach-o/dyld.h>
 #endif
 
 #if defined(ZIG_OS_WINDOWS)
@@ -57,10 +58,6 @@ static clock_serv_t cclock;
 #include <errno.h>
 #include <time.h>
 
-// these implementations are lazy. But who cares, we'll make a robust
-// implementation in the zig standard library and then this code all gets
-// deleted when we self-host. it works for now.
-
 #if defined(ZIG_OS_POSIX)
 static void populate_termination(Termination *term, int status) {
     if (WIFEXITED(status)) {
@@ -927,9 +924,26 @@ int os_self_exe_path(Buf *out_path) {
     }
 
 #elif defined(ZIG_OS_DARWIN)
-    return ErrorFileNotFound;
+    uint32_t u32_len = 0;
+    int ret1 = _NSGetExecutablePath(nullptr, &u32_len);
+    assert(ret1 != 0);
+    buf_resize(out_path, u32_len);
+    int ret2 = _NSGetExecutablePath(buf_ptr(out_path), &u32_len);
+    assert(ret2 == 0);
+    return 0;
 #elif defined(ZIG_OS_LINUX)
-    return ErrorFileNotFound;
+    buf_resize(out_path, 256);
+    for (;;) {
+        ssize_t amt = readlink("/proc/self/exe", buf_ptr(out_path), buf_len(out_path));
+        if (amt == -1) {
+            return ErrorUnexpected;
+        }
+        if (amt == (ssize_t)buf_len(out_path)) {
+            buf_resize(out_path, buf_len(out_path) * 2);
+            continue;
+        }
+        return 0;
+    }
 #endif
     return ErrorFileNotFound;
 }