self-hosted libc and dynamic linker detection

CMakeLists.txt

@@ -434,8 +434,8 @@ find_package(Threads)
 # CMake doesn't let us create an empty executable, so we hang on to this one separately.
 set(ZIG_MAIN_SRC "${CMAKE_SOURCE_DIR}/src/main.cpp")
 
-# This is our shim which will be replaced by libuserland written in Zig.
-set(ZIG0_SHIM_SRC "${CMAKE_SOURCE_DIR}/src/userland.cpp")
+# This is our shim which will be replaced by libstage2 written in Zig.
+set(ZIG0_SHIM_SRC "${CMAKE_SOURCE_DIR}/src/stage2.cpp")
 
 if(ZIG_ENABLE_MEM_PROFILE)
     set(ZIG_SOURCES_MEM_PROFILE "${CMAKE_SOURCE_DIR}/src/mem_profile.cpp")
@@ -457,7 +457,6 @@ set(ZIG_SOURCES
     "${CMAKE_SOURCE_DIR}/src/heap.cpp"
     "${CMAKE_SOURCE_DIR}/src/ir.cpp"
     "${CMAKE_SOURCE_DIR}/src/ir_print.cpp"
-    "${CMAKE_SOURCE_DIR}/src/libc_installation.cpp"
     "${CMAKE_SOURCE_DIR}/src/link.cpp"
     "${CMAKE_SOURCE_DIR}/src/mem.cpp"
     "${CMAKE_SOURCE_DIR}/src/os.cpp"
@@ -566,12 +565,12 @@ set_target_properties(opt_c_util PROPERTIES
     COMPILE_FLAGS "${OPTIMIZED_C_FLAGS}"
 )
 
-add_library(compiler STATIC ${ZIG_SOURCES})
-set_target_properties(compiler PROPERTIES
+add_library(zigcompiler STATIC ${ZIG_SOURCES})
+set_target_properties(zigcompiler PROPERTIES
     COMPILE_FLAGS ${EXE_CFLAGS}
     LINK_FLAGS ${EXE_LDFLAGS}
 )
-target_link_libraries(compiler LINK_PUBLIC
+target_link_libraries(zigcompiler LINK_PUBLIC
     zig_cpp
     opt_c_util
     ${SOFTFLOAT_LIBRARIES}
@@ -581,56 +580,58 @@ target_link_libraries(compiler LINK_PUBLIC
     ${CMAKE_THREAD_LIBS_INIT}
 )
 if(NOT MSVC)
-    target_link_libraries(compiler LINK_PUBLIC ${LIBXML2})
+    target_link_libraries(zigcompiler LINK_PUBLIC ${LIBXML2})
 endif()
 
 if(ZIG_DIA_GUIDS_LIB)
-    target_link_libraries(compiler LINK_PUBLIC ${ZIG_DIA_GUIDS_LIB})
+    target_link_libraries(zigcompiler LINK_PUBLIC ${ZIG_DIA_GUIDS_LIB})
 endif()
 
 if(MSVC OR MINGW)
-    target_link_libraries(compiler LINK_PUBLIC version)
+    target_link_libraries(zigcompiler LINK_PUBLIC version)
 endif()
 
 add_executable(zig0 "${ZIG_MAIN_SRC}" "${ZIG0_SHIM_SRC}")
 set_target_properties(zig0 PROPERTIES
     COMPILE_FLAGS ${EXE_CFLAGS}
     LINK_FLAGS ${EXE_LDFLAGS}
 )
-target_link_libraries(zig0 compiler)
+target_link_libraries(zig0 zigcompiler)
 
 if(MSVC)
-    set(LIBUSERLAND "${CMAKE_BINARY_DIR}/userland.lib")
+    set(LIBSTAGE2 "${CMAKE_BINARY_DIR}/zigstage2.lib")
 else()
-    set(LIBUSERLAND "${CMAKE_BINARY_DIR}/libuserland.a")
+    set(LIBSTAGE2 "${CMAKE_BINARY_DIR}/libzigstage2.a")
 endif()
 if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
-    set(LIBUSERLAND_RELEASE_MODE "false")
+    set(LIBSTAGE2_RELEASE_ARG "")
 else()
-    set(LIBUSERLAND_RELEASE_MODE "true")
+    set(LIBSTAGE2_RELEASE_ARG --release-fast --strip)
+endif()
+if(WIN32)
+    set(LIBSTAGE2_WINDOWS_ARGS "-lntdll")
+else()
+    set(LIBSTAGE2_WINDOWS_ARGS "")
 endif()
 
-set(BUILD_LIBUSERLAND_ARGS "build"
+set(BUILD_LIBSTAGE2_ARGS "build-lib"
+    "src-self-hosted/stage2.zig"
+    --name zigstage2
     --override-lib-dir "${CMAKE_SOURCE_DIR}/lib"
-    "-Doutput-dir=${CMAKE_BINARY_DIR}"
-    "-Drelease=${LIBUSERLAND_RELEASE_MODE}"
-    "-Dlib-files-only"
-    --prefix "${CMAKE_INSTALL_PREFIX}"
-    libuserland
+    --cache on
+    --output-dir "${CMAKE_BINARY_DIR}"
+    ${LIBSTAGE2_RELEASE_ARG}
+    --disable-gen-h
+    --bundle-compiler-rt
+    -fPIC
+    -lc
+    ${LIBSTAGE2_WINDOWS_ARGS}
 )
 
-# When using Visual Studio build system generator we default to libuserland install.
-if(MSVC)
-    set(ZIG_SKIP_INSTALL_LIB_FILES off CACHE BOOL "Disable copying lib/ files to install prefix")
-    if(NOT ZIG_SKIP_INSTALL_LIB_FILES)
-        set(BUILD_LIBUSERLAND_ARGS ${BUILD_LIBUSERLAND_ARGS} install)
-    endif()
-endif()
-
-add_custom_target(zig_build_libuserland ALL
-    COMMAND zig0 ${BUILD_LIBUSERLAND_ARGS}
+add_custom_target(zig_build_libstage2 ALL
+    COMMAND zig0 ${BUILD_LIBSTAGE2_ARGS}
     DEPENDS zig0
-    BYPRODUCTS "${LIBUSERLAND}"
+    BYPRODUCTS "${LIBSTAGE2}"
     WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
 )
 add_executable(zig "${ZIG_MAIN_SRC}")
@@ -639,22 +640,40 @@ set_target_properties(zig PROPERTIES
     COMPILE_FLAGS ${EXE_CFLAGS}
     LINK_FLAGS ${EXE_LDFLAGS}
 )
-target_link_libraries(zig compiler "${LIBUSERLAND}")
+target_link_libraries(zig zigcompiler "${LIBSTAGE2}")
 if(MSVC)
   target_link_libraries(zig ntdll.lib)
 elseif(MINGW) 
   target_link_libraries(zig ntdll)
 endif()
-add_dependencies(zig zig_build_libuserland)
+add_dependencies(zig zig_build_libstage2)
 
 install(TARGETS zig DESTINATION bin)
 
-# CODE has no effect with Visual Studio build system generator.
-if(NOT MSVC)
-    get_target_property(zig0_BINARY_DIR zig0 BINARY_DIR)
-    install(CODE "set(zig0_EXE \"${zig0_BINARY_DIR}/zig0\")")
-    install(CODE "set(INSTALL_LIBUSERLAND_ARGS \"${BUILD_LIBUSERLAND_ARGS}\" install)")
-    install(CODE "set(BUILD_LIBUSERLAND_ARGS \"${BUILD_LIBUSERLAND_ARGS}\")")
+set(ZIG_INSTALL_ARGS "build"
+    --override-lib-dir "${CMAKE_SOURCE_DIR}/lib"
+    "-Dlib-files-only"
+    --prefix "${CMAKE_INSTALL_PREFIX}"
+    install
+)
+
+# CODE has no effect with Visual Studio build system generator, therefore
+# when using Visual Studio build system generator we resort to running
+# `zig build install` during the build phase.
+if(MSVC)
+    set(ZIG_SKIP_INSTALL_LIB_FILES off CACHE BOOL
+        "Windows-only: Disable copying lib/ files to install prefix during the build phase")
+    if(NOT ZIG_SKIP_INSTALL_LIB_FILES)
+        add_custom_target(zig_install_lib_files ALL
+            COMMAND zig ${ZIG_INSTALL_ARGS}
+            DEPENDS zig
+            WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+        )
+    endif()
+else()
+    get_target_property(zig_BINARY_DIR zig BINARY_DIR)
+    install(CODE "set(zig_EXE \"${zig_BINARY_DIR}/zig\")")
+    install(CODE "set(ZIG_INSTALL_ARGS \"${ZIG_INSTALL_ARGS}\")")
     install(CODE "set(CMAKE_SOURCE_DIR \"${CMAKE_SOURCE_DIR}\")")
     install(SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/cmake/install.cmake)
 endif()

build.zig

@@ -64,8 +64,6 @@ pub fn build(b: *Builder) !void {
     try configureStage2(b, test_stage2, ctx);
     try configureStage2(b, exe, ctx);
 
-    addLibUserlandStep(b, mode);
-
     const skip_release = b.option(bool, "skip-release", "Main test suite skips release builds") orelse false;
     const skip_release_small = b.option(bool, "skip-release-small", "Main test suite skips release-small builds") orelse skip_release;
     const skip_release_fast = b.option(bool, "skip-release-fast", "Main test suite skips release-fast builds") orelse skip_release;
@@ -175,7 +173,7 @@ fn dependOnLib(b: *Builder, lib_exe_obj: var, dep: LibraryDep) void {
 }
 
 fn fileExists(filename: []const u8) !bool {
-    fs.File.access(filename) catch |err| switch (err) {
+    fs.cwd().access(filename, .{}) catch |err| switch (err) {
         error.FileNotFound => return false,
         else => return err,
     };
@@ -366,28 +364,3 @@ const Context = struct {
     dia_guids_lib: []const u8,
     llvm: LibraryDep,
 };
-
-fn addLibUserlandStep(b: *Builder, mode: builtin.Mode) void {
-    const artifact = b.addStaticLibrary("userland", "src-self-hosted/stage1.zig");
-    artifact.disable_gen_h = true;
-    artifact.bundle_compiler_rt = true;
-    artifact.setTarget(builtin.arch, builtin.os, builtin.abi);
-    artifact.setBuildMode(mode);
-    artifact.force_pic = true;
-    if (mode != .Debug) {
-        artifact.strip = true;
-    }
-    artifact.linkSystemLibrary("c");
-    if (builtin.os == .windows) {
-        artifact.linkSystemLibrary("ntdll");
-    }
-    const libuserland_step = b.step("libuserland", "Build the userland compiler library for use in stage1");
-    libuserland_step.dependOn(&artifact.step);
-
-    const output_dir = b.option(
-        []const u8,
-        "output-dir",
-        "For libuserland step, where to put the output",
-    ) orelse return;
-    artifact.setOutputDir(output_dir);
-}

cmake/install.cmake

@@ -1,16 +1,16 @@
 message("-- Installing: ${CMAKE_INSTALL_PREFIX}/lib")
 
-if(NOT EXISTS ${zig0_EXE})
+if(NOT EXISTS ${zig_EXE})
     message("::")
     message(":: ERROR: Executable not found")
     message(":: (execute_process)")
     message("::")
-    message(":: executable: ${zig0_EXE}")
+    message(":: executable: ${zig_EXE}")
     message("::")
     message(FATAL_ERROR)
 endif()
 
-execute_process(COMMAND ${zig0_EXE} ${INSTALL_LIBUSERLAND_ARGS}
+execute_process(COMMAND ${zig_EXE} ${ZIG_INSTALL_ARGS}
     WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
     RESULT_VARIABLE _result
 )
@@ -19,11 +19,11 @@ if(_result)
     message(":: ERROR: ${_result}")
     message(":: (execute_process)")
 
-    string(REPLACE ";" " " s_INSTALL_LIBUSERLAND_ARGS "${INSTALL_LIBUSERLAND_ARGS}")
+    string(REPLACE ";" " " s_INSTALL_LIBSTAGE2_ARGS "${ZIG_INSTALL_ARGS}")
     message("::")
-    message(":: argv: ${zig0_EXE} ${s_INSTALL_LIBUSERLAND_ARGS} install")
+    message(":: argv: ${zig_EXE} ${s_INSTALL_LIBSTAGE2_ARGS}")
 
-    set(_args ${zig0_EXE} ${INSTALL_LIBUSERLAND_ARGS})
+    set(_args ${zig_EXE} ${ZIG_INSTALL_ARGS})
     list(LENGTH _args _len)
     math(EXPR _len "${_len} - 1")
     message("::")

lib/std/c.zig

@@ -96,6 +96,7 @@ pub extern "c" fn getcwd(buf: [*]u8, size: usize) ?[*]u8;
 pub extern "c" fn waitpid(pid: c_int, stat_loc: *c_uint, options: c_uint) c_int;
 pub extern "c" fn fork() c_int;
 pub extern "c" fn access(path: [*:0]const u8, mode: c_uint) c_int;
+pub extern "c" fn faccessat(dirfd: fd_t, path: [*:0]const u8, mode: c_uint, flags: c_uint) c_int;
 pub extern "c" fn pipe(fds: *[2]fd_t) c_int;
 pub extern "c" fn pipe2(fds: *[2]fd_t, flags: u32) c_int;
 pub extern "c" fn mkdir(path: [*:0]const u8, mode: c_uint) c_int;

lib/std/child_process.zig

@@ -48,7 +48,10 @@ pub const ChildProcess = struct {
     cwd: ?[]const u8,
 
     err_pipe: if (builtin.os == .windows) void else [2]os.fd_t,
-    llnode: if (builtin.os == .windows) void else TailQueue(*ChildProcess).Node,
+
+    expand_arg0: Arg0Expand,
+
+    pub const Arg0Expand = os.Arg0Expand;
 
     pub const SpawnError = error{
         OutOfMemory,
@@ -90,7 +93,6 @@ pub const ChildProcess = struct {
             .handle = undefined,
             .thread_handle = undefined,
             .err_pipe = undefined,
-            .llnode = undefined,
             .term = null,
             .env_map = null,
             .cwd = null,
@@ -102,6 +104,7 @@ pub const ChildProcess = struct {
             .stdin_behavior = StdIo.Inherit,
             .stdout_behavior = StdIo.Inherit,
             .stderr_behavior = StdIo.Inherit,
+            .expand_arg0 = .no_expand,
         };
         errdefer allocator.destroy(child);
         return child;
@@ -174,34 +177,56 @@ pub const ChildProcess = struct {
 
     /// Spawns a child process, waits for it, collecting stdout and stderr, and then returns.
     /// If it succeeds, the caller owns result.stdout and result.stderr memory.
+    /// TODO deprecate in favor of exec2
     pub fn exec(
         allocator: *mem.Allocator,
         argv: []const []const u8,
         cwd: ?[]const u8,
         env_map: ?*const BufMap,
-        max_output_size: usize,
+        max_output_bytes: usize,
     ) !ExecResult {
-        const child = try ChildProcess.init(argv, allocator);
+        return exec2(.{
+            .allocator = allocator,
+            .argv = argv,
+            .cwd = cwd,
+            .env_map = env_map,
+            .max_output_bytes = max_output_bytes,
+        });
+    }
+
+    /// Spawns a child process, waits for it, collecting stdout and stderr, and then returns.
+    /// If it succeeds, the caller owns result.stdout and result.stderr memory.
+    /// TODO rename to exec
+    pub fn exec2(args: struct {
+        allocator: *mem.Allocator,
+        argv: []const []const u8,
+        cwd: ?[]const u8 = null,
+        env_map: ?*const BufMap = null,
+        max_output_bytes: usize = 50 * 1024,
+        expand_arg0: Arg0Expand = .no_expand,
+    }) !ExecResult {
+        const child = try ChildProcess.init(args.argv, args.allocator);
         defer child.deinit();
 
-        child.stdin_behavior = ChildProcess.StdIo.Ignore;
-        child.stdout_behavior = ChildProcess.StdIo.Pipe;
-        child.stderr_behavior = ChildProcess.StdIo.Pipe;
-        child.cwd = cwd;
-        child.env_map = env_map;
+        child.stdin_behavior = .Ignore;
+        child.stdout_behavior = .Pipe;
+        child.stderr_behavior = .Pipe;
+        child.cwd = args.cwd;
+        child.env_map = args.env_map;
+        child.expand_arg0 = args.expand_arg0;
 
         try child.spawn();
 
-        var stdout = Buffer.initNull(allocator);
-        var stderr = Buffer.initNull(allocator);
+        var stdout = Buffer.initNull(args.allocator);
+        var stderr = Buffer.initNull(args.allocator);
         defer Buffer.deinit(&stdout);
         defer Buffer.deinit(&stderr);
 
         var stdout_file_in_stream = child.stdout.?.inStream();
         var stderr_file_in_stream = child.stderr.?.inStream();
 
-        try stdout_file_in_stream.stream.readAllBuffer(&stdout, max_output_size);
-        try stderr_file_in_stream.stream.readAllBuffer(&stderr, max_output_size);
+        try stdout_file_in_stream.stream.readAllBuffer(&stdout, args.max_output_bytes);
+        try stderr_file_in_stream.stream.readAllBuffer(&stderr, args.max_output_bytes);
 
         return ExecResult{
             .term = try child.wait(),
@@ -420,7 +445,7 @@ pub const ChildProcess = struct {
                 os.setreuid(uid, uid) catch |err| forkChildErrReport(err_pipe[1], err);
             }
 
-            const err = os.execvpe(self.allocator, self.argv, env_map);
+            const err = os.execvpe_expandArg0(self.allocator, self.expand_arg0, self.argv, env_map);
             forkChildErrReport(err_pipe[1], err);
         }
 
@@ -453,7 +478,6 @@ pub const ChildProcess = struct {
 
         self.pid = pid;
         self.err_pipe = err_pipe;
-        self.llnode = TailQueue(*ChildProcess).Node.init(self);
         self.term = null;
 
         if (self.stdin_behavior == StdIo.Pipe) {

lib/std/event.zig

@@ -1,6 +1,7 @@
 pub const Channel = @import("event/channel.zig").Channel;
 pub const Future = @import("event/future.zig").Future;
 pub const Group = @import("event/group.zig").Group;
+pub const Batch = @import("event/batch.zig").Batch;
 pub const Lock = @import("event/lock.zig").Lock;
 pub const Locked = @import("event/locked.zig").Locked;
 pub const RwLock = @import("event/rwlock.zig").RwLock;
@@ -11,6 +12,7 @@ test "import event tests" {
     _ = @import("event/channel.zig");
     _ = @import("event/future.zig");
     _ = @import("event/group.zig");
+    _ = @import("event/batch.zig");
     _ = @import("event/lock.zig");
     _ = @import("event/locked.zig");
     _ = @import("event/rwlock.zig");