Merge pull request #13560 from ziglang/wasi-bootstrap

Nuke the C++ implementation of Zig from orbit using WASI

Author: andrewrk

CMakeLists.txt

@@ -14,13 +14,25 @@ const zig_version = std.builtin.Version{ .major = 0, .minor = 11, .patch = 0 };
 const stack_size = 32 * 1024 * 1024;
 
 pub fn build(b: *Builder) !void {
-    b.setPreferredReleaseMode(.ReleaseFast);
-    const test_step = b.step("test", "Run all the tests");
-    const mode = b.standardReleaseOptions();
-    var target = b.standardTargetOptions(.{});
+    const release = b.option(bool, "release", "Build in release mode") orelse false;
+    const only_c = b.option(bool, "only-c", "Translate the Zig compiler to C code, with only the C backend enabled") orelse false;
+    const target = t: {
+        var default_target: std.zig.CrossTarget = .{};
+        if (only_c) {
+            default_target.ofmt = .c;
+        }
+        break :t b.standardTargetOptions(.{ .default_target = default_target });
+    };
+    const mode: std.builtin.Mode = if (release) switch (target.getCpuArch()) {
+        .wasm32 => .ReleaseSmall,
+        else => .ReleaseFast,
+    } else .Debug;
+
     const single_threaded = b.option(bool, "single-threaded", "Build artifacts that run in single threaded mode");
     const use_zig_libcxx = b.option(bool, "use-zig-libcxx", "If libc++ is needed, use zig's bundled version, don't try to integrate with the system") orelse false;
 
+    const test_step = b.step("test", "Run all the tests");
+
     const docgen_exe = b.addExecutable("docgen", "doc/docgen.zig");
     docgen_exe.single_threaded = single_threaded;
 
@@ -48,8 +60,6 @@ pub fn build(b: *Builder) !void {
 
     const fmt_build_zig = b.addFmt(&[_][]const u8{"build.zig"});
 
-    const only_c = b.option(bool, "only-c", "Translate the Zig compiler to C code, with only the C backend enabled") orelse false;
-
     const skip_debug = b.option(bool, "skip-debug", "Main test suite skips debug builds") orelse false;
     const skip_release = b.option(bool, "skip-release", "Main test suite skips release builds") orelse false;
     const skip_release_small = b.option(bool, "skip-release-small", "Main test suite skips release-small builds") orelse skip_release;
@@ -69,9 +79,8 @@ pub fn build(b: *Builder) !void {
 
     const only_install_lib_files = b.option(bool, "lib-files-only", "Only install library files") orelse false;
 
-    const have_stage1 = b.option(bool, "enable-stage1", "Include the stage1 compiler behind a feature flag") orelse false;
     const static_llvm = b.option(bool, "static-llvm", "Disable integration with system-installed LLVM, Clang, LLD, and libc++") orelse false;
-    const enable_llvm = b.option(bool, "enable-llvm", "Build self-hosted compiler with LLVM backend enabled") orelse (have_stage1 or static_llvm);
+    const enable_llvm = b.option(bool, "enable-llvm", "Build self-hosted compiler with LLVM backend enabled") orelse static_llvm;
     const llvm_has_m68k = b.option(
         bool,
         "llvm-has-m68k",
@@ -132,38 +141,26 @@ pub fn build(b: *Builder) !void {
     const force_gpa = b.option(bool, "force-gpa", "Force the compiler to use GeneralPurposeAllocator") orelse false;
     const link_libc = b.option(bool, "force-link-libc", "Force self-hosted compiler to link libc") orelse (enable_llvm or only_c);
     const sanitize_thread = b.option(bool, "sanitize-thread", "Enable thread-sanitization") orelse false;
-    const strip = b.option(bool, "strip", "Omit debug information") orelse false;
-    const use_zig0 = b.option(bool, "zig0", "Bootstrap using zig0") orelse false;
+    const strip = b.option(bool, "strip", "Omit debug information");
     const value_tracing = b.option(bool, "value-tracing", "Enable extra state tracking to help troubleshoot bugs in the compiler (using the std.debug.Trace API)") orelse false;
 
     const mem_leak_frames: u32 = b.option(u32, "mem-leak-frames", "How many stack frames to print when a memory leak occurs. Tests get 2x this amount.") orelse blk: {
-        if (strip) break :blk @as(u32, 0);
+        if (strip == true) break :blk @as(u32, 0);
         if (mode != .Debug) break :blk 0;
         break :blk 4;
     };
 
-    if (only_c) {
-        target.ofmt = .c;
-    }
-
-    const main_file: ?[]const u8 = mf: {
-        if (!have_stage1) break :mf "src/main.zig";
-        if (use_zig0) break :mf null;
-        break :mf "src/stage1.zig";
-    };
-
-    const exe = b.addExecutable("zig", main_file);
-
-    const compile_step = b.step("compile", "Build the self-hosted compiler");
-    compile_step.dependOn(&exe.step);
-
-    exe.stack_size = stack_size;
+    const exe = addCompilerStep(b);
     exe.strip = strip;
     exe.sanitize_thread = sanitize_thread;
     exe.build_id = b.option(bool, "build-id", "Include a build id note") orelse false;
     exe.install();
     exe.setBuildMode(mode);
     exe.setTarget(target);
+
+    const compile_step = b.step("compile", "Build the self-hosted compiler");
+    compile_step.dependOn(&exe.step);
+
     if (!skip_stage2_tests) {
         test_step.dependOn(&exe.step);
     }
@@ -199,7 +196,7 @@ pub fn build(b: *Builder) !void {
     const enable_link_snapshots = b.option(bool, "link-snapshot", "Whether to enable linker state snapshots") orelse false;
 
     const opt_version_string = b.option([]const u8, "version-string", "Override Zig version string. Default is to find out with git.");
-    const version = if (opt_version_string) |version| version else v: {
+    const version_slice = if (opt_version_string) |version| version else v: {
         if (!std.process.can_spawn) {
             std.debug.print("error: version info cannot be retrieved from git. Zig version must be provided using -Dversion-string\n", .{});
             std.process.exit(1);
@@ -251,7 +248,8 @@ pub fn build(b: *Builder) !void {
             },
         }
     };
-    exe_options.addOption([:0]const u8, "version", try b.allocator.dupeZ(u8, version));
+    const version = try b.allocator.dupeZ(u8, version_slice);
+    exe_options.addOption([:0]const u8, "version", version);
 
     if (enable_llvm) {
         const cmake_cfg = if (static_llvm) null else blk: {
@@ -264,92 +262,6 @@ pub fn build(b: *Builder) !void {
             }
         };
 
-        if (have_stage1) {
-            const softfloat = b.addStaticLibrary("softfloat", null);
-            softfloat.setBuildMode(.ReleaseFast);
-            softfloat.setTarget(target);
-            softfloat.addIncludePath("deps/SoftFloat-3e-prebuilt");
-            softfloat.addIncludePath("deps/SoftFloat-3e/source/8086");
-            softfloat.addIncludePath("deps/SoftFloat-3e/source/include");
-            softfloat.addCSourceFiles(&softfloat_sources, &[_][]const u8{ "-std=c99", "-O3" });
-            softfloat.single_threaded = single_threaded;
-
-            const zig0 = b.addExecutable("zig0", null);
-            zig0.addCSourceFiles(&.{"src/stage1/zig0.cpp"}, &exe_cflags);
-            zig0.addIncludePath("zig-cache/tmp"); // for config.h
-            zig0.defineCMacro("ZIG_VERSION_MAJOR", b.fmt("{d}", .{zig_version.major}));
-            zig0.defineCMacro("ZIG_VERSION_MINOR", b.fmt("{d}", .{zig_version.minor}));
-            zig0.defineCMacro("ZIG_VERSION_PATCH", b.fmt("{d}", .{zig_version.patch}));
-            zig0.defineCMacro("ZIG_VERSION_STRING", b.fmt("\"{s}\"", .{version}));
-
-            for ([_]*std.build.LibExeObjStep{ zig0, exe, test_cases }) |artifact| {
-                artifact.addIncludePath("src");
-                artifact.addIncludePath("deps/SoftFloat-3e/source/include");
-                artifact.addIncludePath("deps/SoftFloat-3e-prebuilt");
-
-                artifact.defineCMacro("ZIG_LINK_MODE", "Static");
-
-                artifact.addCSourceFiles(&stage1_sources, &exe_cflags);
-                artifact.addCSourceFiles(&optimized_c_sources, &[_][]const u8{ "-std=c99", "-O3" });
-
-                artifact.linkLibrary(softfloat);
-                artifact.linkLibCpp();
-            }
-
-            try addStaticLlvmOptionsToExe(zig0);
-
-            const zig1_obj_ext = target.getObjectFormat().fileExt(target.getCpuArch());
-            const zig1_obj_path = b.pathJoin(&.{ "zig-cache", "tmp", b.fmt("zig1{s}", .{zig1_obj_ext}) });
-            const zig1_compiler_rt_path = b.pathJoin(&.{ b.pathFromRoot("lib"), "std", "special", "compiler_rt.zig" });
-
-            const zig1_obj = zig0.run();
-            zig1_obj.addArgs(&.{
-                "src/stage1.zig",
-                "-target",
-                try target.zigTriple(b.allocator),
-                "-mcpu=baseline",
-                "--name",
-                "zig1",
-                "--zig-lib-dir",
-                b.pathFromRoot("lib"),
-                b.fmt("-femit-bin={s}", .{b.pathFromRoot(zig1_obj_path)}),
-                "-fcompiler-rt",
-                "-lc",
-            });
-            {
-                zig1_obj.addArgs(&.{ "--pkg-begin", "build_options" });
-                zig1_obj.addFileSourceArg(exe_options.getSource());
-                zig1_obj.addArgs(&.{ "--pkg-end", "--pkg-begin", "compiler_rt", zig1_compiler_rt_path, "--pkg-end" });
-            }
-            switch (mode) {
-                .Debug => {},
-                .ReleaseFast => {
-                    zig1_obj.addArg("-OReleaseFast");
-                    zig1_obj.addArg("-fstrip");
-                },
-                .ReleaseSafe => {
-                    zig1_obj.addArg("-OReleaseSafe");
-                    zig1_obj.addArg("-fstrip");
-                },
-                .ReleaseSmall => {
-                    zig1_obj.addArg("-OReleaseSmall");
-                    zig1_obj.addArg("-fstrip");
-                },
-            }
-            if (single_threaded orelse false) {
-                zig1_obj.addArg("-fsingle-threaded");
-            }
-
-            if (use_zig0) {
-                exe.step.dependOn(&zig1_obj.step);
-                exe.addObjectFile(zig1_obj_path);
-            }
-
-            // This is intentionally a dummy path. stage1.zig tries to @import("compiler_rt") in case
-            // of being built by cmake. But when built by zig it's gonna get a compiler_rt so that
-            // is pointless.
-            exe.addPackagePath("compiler_rt", "src/empty.zig");
-        }
         if (cmake_cfg) |cfg| {
             // Inside this code path, we have to coordinate with system packaged LLVM, Clang, and LLD.
             // That means we also have to rely on stage1 compiled c++ files. We parse config.h to find
@@ -379,7 +291,6 @@ pub fn build(b: *Builder) !void {
     exe_options.addOption(bool, "enable_tracy_callstack", tracy_callstack);
     exe_options.addOption(bool, "enable_tracy_allocation", tracy_allocation);
     exe_options.addOption(bool, "value_tracing", value_tracing);
-    exe_options.addOption(bool, "have_stage1", have_stage1);
     if (tracy) |tracy_path| {
         const client_cpp = fs.path.join(
             b.allocator,
@@ -414,7 +325,6 @@ pub fn build(b: *Builder) !void {
     test_cases_options.addOption(bool, "enable_link_snapshots", enable_link_snapshots);
     test_cases_options.addOption(bool, "skip_non_native", skip_non_native);
     test_cases_options.addOption(bool, "skip_stage1", skip_stage1);
-    test_cases_options.addOption(bool, "have_stage1", have_stage1);
     test_cases_options.addOption(bool, "have_llvm", enable_llvm);
     test_cases_options.addOption(bool, "llvm_has_m68k", llvm_has_m68k);
     test_cases_options.addOption(bool, "llvm_has_csky", llvm_has_csky);
@@ -429,7 +339,7 @@ pub fn build(b: *Builder) !void {
     test_cases_options.addOption(u32, "mem_leak_frames", mem_leak_frames * 2);
     test_cases_options.addOption(bool, "value_tracing", value_tracing);
     test_cases_options.addOption(?[]const u8, "glibc_runtimes_dir", b.glibc_runtimes_dir);
-    test_cases_options.addOption([:0]const u8, "version", try b.allocator.dupeZ(u8, version));
+    test_cases_options.addOption([:0]const u8, "version", version);
     test_cases_options.addOption(std.SemanticVersion, "semver", semver);
     test_cases_options.addOption(?[]const u8, "test_filter", test_filter);
 
@@ -547,6 +457,61 @@ pub fn build(b: *Builder) !void {
         skip_stage1,
         true, // TODO get these all passing
     ));
+
+    try addWasiUpdateStep(b, version);
+}
+
+fn addWasiUpdateStep(b: *Builder, version: [:0]const u8) !void {
+    const semver = try std.SemanticVersion.parse(version);
+
+    var target: std.zig.CrossTarget = .{
+        .cpu_arch = .wasm32,
+        .os_tag = .wasi,
+    };
+    target.cpu_features_add.addFeature(@enumToInt(std.Target.wasm.Feature.bulk_memory));
+
+    const exe = addCompilerStep(b);
+    exe.setBuildMode(.ReleaseSmall);
+    exe.setTarget(target);
+
+    const exe_options = b.addOptions();
+    exe.addOptions("build_options", exe_options);
+
+    exe_options.addOption(u32, "mem_leak_frames", 0);
+    exe_options.addOption(bool, "have_llvm", false);
+    exe_options.addOption(bool, "force_gpa", false);
+    exe_options.addOption(bool, "only_c", true);
+    exe_options.addOption([:0]const u8, "version", version);
+    exe_options.addOption(std.SemanticVersion, "semver", semver);
+    exe_options.addOption(bool, "enable_logging", false);
+    exe_options.addOption(bool, "enable_link_snapshots", false);
+    exe_options.addOption(bool, "enable_tracy", false);
+    exe_options.addOption(bool, "enable_tracy_callstack", false);
+    exe_options.addOption(bool, "enable_tracy_allocation", false);
+    exe_options.addOption(bool, "value_tracing", false);
+
+    const run_opt = b.addSystemCommand(&.{ "wasm-opt", "-Oz", "--enable-bulk-memory" });
+    run_opt.addArtifactArg(exe);
+    run_opt.addArg("-o");
+    run_opt.addFileSourceArg(.{ .path = "stage1/zig1.wasm" });
+
+    const run_zstd = b.addSystemCommand(&.{ "zstd", "-19", "-f" });
+    run_zstd.step.dependOn(&run_opt.step);
+    run_zstd.addFileSourceArg(.{ .path = "stage1/zig1.wasm" });
+    run_zstd.addArg("-o");
+    run_zstd.addFileSourceArg(.{ .path = "stage1/zig1.wasm.zst" });
+
+    const cleanup = b.addRemoveDirTree("stage1/zig1.wasm");
+    cleanup.step.dependOn(&run_zstd.step);
+
+    const update_zig1_step = b.step("update-zig1", "Update stage1/zig1.wasm.zst");
+    update_zig1_step.dependOn(&cleanup.step);
+}
+
+fn addCompilerStep(b: *Builder) *std.build.LibExeObjStep {
+    const exe = b.addExecutable("zig", "src/main.zig");
+    exe.stack_size = stack_size;
+    return exe;
 }
 
 const exe_cflags = [_][]const u8{
@@ -1010,31 +975,6 @@ const softfloat_sources = [_][]const u8{
     "deps/SoftFloat-3e/source/ui64_to_extF80M.c",
 };
 
-const stage1_sources = [_][]const u8{
-    "src/stage1/analyze.cpp",
-    "src/stage1/astgen.cpp",
-    "src/stage1/bigfloat.cpp",
-    "src/stage1/bigint.cpp",
-    "src/stage1/buffer.cpp",
-    "src/stage1/codegen.cpp",
-    "src/stage1/errmsg.cpp",
-    "src/stage1/error.cpp",
-    "src/stage1/heap.cpp",
-    "src/stage1/ir.cpp",
-    "src/stage1/ir_print.cpp",
-    "src/stage1/mem.cpp",
-    "src/stage1/os.cpp",
-    "src/stage1/parser.cpp",
-    "src/stage1/range_set.cpp",
-    "src/stage1/stage1.cpp",
-    "src/stage1/target.cpp",
-    "src/stage1/tokenizer.cpp",
-    "src/stage1/util.cpp",
-    "src/stage1/softfloat_ext.cpp",
-};
-const optimized_c_sources = [_][]const u8{
-    "src/stage1/parse_f128.c",
-};
 const zig_cpp_sources = [_][]const u8{
     // These are planned to stay even when we are self-hosted.
     "src/zig_llvm.cpp",

build.zig

@@ -8,7 +8,7 @@ set -e
 ARCH="$(uname -m)"
 TARGET="$ARCH-linux-musl"
 MCPU="baseline"
-CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.11.0-dev.448+e6e459e9e"
+CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.11.0-dev.534+b0b1cc356"
 PREFIX="$HOME/deps/$CACHE_BASENAME"
 ZIG="$PREFIX/bin/zig"
 

ci/aarch64-linux-debug.sh

@@ -8,7 +8,7 @@ set -e
 ARCH="$(uname -m)"
 TARGET="$ARCH-linux-musl"
 MCPU="baseline"
-CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.11.0-dev.448+e6e459e9e"
+CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.11.0-dev.534+b0b1cc356"
 PREFIX="$HOME/deps/$CACHE_BASENAME"
 ZIG="$PREFIX/bin/zig" 
 

ci/aarch64-linux-release.sh

@@ -9,7 +9,7 @@ set -e
 ZIGDIR="$(pwd)"
 TARGET="$ARCH-macos-none"
 MCPU="baseline"
-CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.11.0-dev.448+e6e459e9e"
+CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.11.0-dev.534+b0b1cc356-1"
 PREFIX="$HOME/$CACHE_BASENAME"
 ZIG="$PREFIX/bin/zig"
 

ci/aarch64-macos.sh

@@ -75,7 +75,6 @@ stage3-release/bin/zig test ../lib/std/std.zig -femit-docs -fno-emit-bin --zig-l
 stage3-release/bin/zig build \
   --prefix stage4-release \
   -Denable-llvm \
-  -Denable-stage1 \
   -Dno-lib \
   -Drelease \
   -Dstrip \

ci/x86_64-linux-release.sh

@@ -9,7 +9,7 @@ set -e
 ZIGDIR="$(pwd)"
 TARGET="$ARCH-macos-none"
 MCPU="baseline"
-CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.11.0-dev.448+e6e459e9e"
+CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.11.0-dev.534+b0b1cc356"
 PREFIX="$HOME/$CACHE_BASENAME"
 JOBS="-j3"
 

ci/x86_64-macos.sh

@@ -8,13 +8,13 @@ Invoke-WebRequest -Uri "$ZIG_LLVM_CLANG_LLD_URL" -OutFile "$ZIG_LLVM_CLANG_LLD_N
 
 Write-Output "Extracting..."
 
-Add-Type -AssemblyName System.IO.Compression.FileSystem ; 
+Add-Type -AssemblyName System.IO.Compression.FileSystem ;
 [System.IO.Compression.ZipFile]::ExtractToDirectory("$PWD/$ZIG_LLVM_CLANG_LLD_NAME.zip", "$PWD")
 
 Set-Variable -Name ZIGLIBDIR -Value "$(Get-Location)\lib"
 Set-Variable -Name ZIGINSTALLDIR -Value "$(Get-Location)\stage3-release"
 Set-Variable -Name ZIGPREFIXPATH -Value "$(Get-Location)\$ZIG_LLVM_CLANG_LLD_NAME"
-      
+
 function CheckLastExitCode {
     if (!$?) {
         exit 1
@@ -37,7 +37,6 @@ Write-Output "Building Zig..."
     --prefix "$ZIGINSTALLDIR" `
     --search-prefix "$ZIGPREFIXPATH" `
     --zig-lib-dir "$ZIGLIBDIR" `
-    -Denable-stage1 `
     -Dstatic-llvm `
     -Drelease `
     -Duse-zig-libcxx `