self-hosted translate-c emits a hello world AST

andrewrk · andrewrk · commit 28071ac637a7 · 2019-04-26T15:43:36.000-04:00
Also breaking std lib API change: the return value of std.zig.parse returns `*ast.Tree` rather than `ast.Tree`. See #1964
diff --git a/src-self-hosted/compilation.zig b/src-self-hosted/compilation.zig
@@ -569,9 +569,9 @@ pub const Compilation = struct {
                 'i', 'u' => blk: {
                     for (name[1..]) |byte|
                         switch (byte) {
-                        '0'...'9' => {},
-                        else => break :blk,
-                    };
+                            '0'...'9' => {},
+                            else => break :blk,
+                        };
                     const is_signed = name[0] == 'i';
                     const bit_count = std.fmt.parseUnsigned(u32, name[1..], 10) catch |err| switch (err) {
                         error.Overflow => return error.Overflow,
@@ -841,11 +841,9 @@ pub const Compilation = struct {
             };
             errdefer self.gpa().free(source_code);
 
-            const tree = try self.gpa().create(ast.Tree);
-            tree.* = try std.zig.parse(self.gpa(), source_code);
+            const tree = try std.zig.parse(self.gpa(), source_code);
             errdefer {
                 tree.deinit();
-                self.gpa().destroy(tree);
             }
 
             break :blk try Scope.AstTree.create(self, tree, root_scope);
diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig
@@ -625,15 +625,15 @@ fn cmdFmt(allocator: *Allocator, args: []const []const u8) !void {
         const source_code = try stdin.stream.readAllAlloc(allocator, max_src_size);
         defer allocator.free(source_code);
 
-        var tree = std.zig.parse(allocator, source_code) catch |err| {
+        const tree = std.zig.parse(allocator, source_code) catch |err| {
             try stderr.print("error parsing stdin: {}\n", err);
             os.exit(1);
         };
         defer tree.deinit();
 
         var error_it = tree.errors.iterator(0);
         while (error_it.next()) |parse_error| {
-            const msg = try errmsg.Msg.createFromParseError(allocator, parse_error, &tree, "<stdin>");
+            const msg = try errmsg.Msg.createFromParseError(allocator, parse_error, tree, "<stdin>");
             defer msg.destroy();
 
             try msg.printToFile(stderr_file, color);
@@ -642,12 +642,12 @@ fn cmdFmt(allocator: *Allocator, args: []const []const u8) !void {
             os.exit(1);
         }
         if (flags.present("check")) {
-            const anything_changed = try std.zig.render(allocator, io.null_out_stream, &tree);
+            const anything_changed = try std.zig.render(allocator, io.null_out_stream, tree);
             const code = if (anything_changed) u8(1) else u8(0);
             os.exit(code);
         }
 
-        _ = try std.zig.render(allocator, stdout, &tree);
+        _ = try std.zig.render(allocator, stdout, tree);
         return;
     }
 
@@ -768,7 +768,7 @@ async fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtErro
     };
     defer fmt.loop.allocator.free(source_code);
 
-    var tree = std.zig.parse(fmt.loop.allocator, source_code) catch |err| {
+    const tree = std.zig.parse(fmt.loop.allocator, source_code) catch |err| {
         try stderr.print("error parsing file '{}': {}\n", file_path, err);
         fmt.any_error = true;
         return;
@@ -777,7 +777,7 @@ async fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtErro
 
     var error_it = tree.errors.iterator(0);
     while (error_it.next()) |parse_error| {
-        const msg = try errmsg.Msg.createFromParseError(fmt.loop.allocator, parse_error, &tree, file_path);
+        const msg = try errmsg.Msg.createFromParseError(fmt.loop.allocator, parse_error, tree, file_path);
         defer fmt.loop.allocator.destroy(msg);
 
         try msg.printToFile(stderr_file, fmt.color);
@@ -788,7 +788,7 @@ async fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtErro
     }
 
     if (check_mode) {
-        const anything_changed = try std.zig.render(fmt.loop.allocator, io.null_out_stream, &tree);
+        const anything_changed = try std.zig.render(fmt.loop.allocator, io.null_out_stream, tree);
         if (anything_changed) {
             try stderr.print("{}\n", file_path);
             fmt.any_error = true;
@@ -798,7 +798,7 @@ async fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtErro
         const baf = try io.BufferedAtomicFile.create(fmt.loop.allocator, file_path);
         defer baf.destroy();
 
-        const anything_changed = try std.zig.render(fmt.loop.allocator, baf.stream(), &tree);
+        const anything_changed = try std.zig.render(fmt.loop.allocator, baf.stream(), tree);
         if (anything_changed) {
             try stderr.print("{}\n", file_path);
             try baf.finish();
diff --git a/src-self-hosted/scope.zig b/src-self-hosted/scope.zig
@@ -163,7 +163,6 @@ pub const Scope = struct {
         pub fn destroy(self: *AstTree, comp: *Compilation) void {
             comp.gpa().free(self.tree.source);
             self.tree.deinit();
-            comp.gpa().destroy(self.tree);
             comp.gpa().destroy(self);
         }
 
diff --git a/src-self-hosted/stage1.zig b/src-self-hosted/stage1.zig
@@ -85,11 +85,10 @@ export fn stage2_translate_c(
     resources_path: [*]const u8,
 ) Error {
     var errors: []translate_c.ClangErrMsg = undefined;
-    out_ast.* = translate_c.translate(args_begin, args_end, switch (mode) {
+    out_ast.* = translate_c.translate(std.heap.c_allocator, args_begin, args_end, switch (mode) {
         .import => translate_c.Mode.import,
         .translate => translate_c.Mode.translate,
     }, &errors, resources_path) catch |err| switch (err) {
-        error.Unimplemented => return Error.Unimplemented,
         error.SemanticAnalyzeFail => {
             out_errors_ptr.* = errors.ptr;
             out_errors_len.* = errors.len;
diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig
@@ -3,6 +3,7 @@
 
 const std = @import("std");
 const ast = std.zig.ast;
+const Token = std.zig.Token;
 use @import("clang.zig");
 
 pub const Mode = enum {
@@ -13,6 +14,7 @@ pub const Mode = enum {
 pub const ClangErrMsg = Stage2ErrorMsg;
 
 pub fn translate(
+    backing_allocator: *std.mem.Allocator,
     args_begin: [*]?[*]const u8,
     args_end: [*]?[*]const u8,
     mode: Mode,
@@ -29,8 +31,49 @@ pub fn translate(
         if (errors.len == 0) return error.OutOfMemory;
         return error.SemanticAnalyzeFail;
     };
+    defer ZigClangASTUnit_delete(ast_unit);
 
-    return error.Unimplemented;
+    var tree_arena = std.heap.ArenaAllocator.init(backing_allocator);
+    errdefer tree_arena.deinit();
+    const arena = &tree_arena.allocator;
+
+    const root_node = try arena.create(ast.Node.Root);
+    root_node.* = ast.Node.Root{
+        .base = ast.Node{ .id = ast.Node.Id.Root },
+        .decls = ast.Node.Root.DeclList.init(arena),
+        .doc_comments = null,
+        // initialized with the eof token at the end
+        .eof_token = undefined,
+    };
+
+    const tree = try arena.create(ast.Tree);
+    tree.* = ast.Tree{
+        .source = undefined, // need to use Buffer.toOwnedSlice later
+        .root_node = root_node,
+        .arena_allocator = tree_arena,
+        .tokens = ast.Tree.TokenList.init(arena),
+        .errors = ast.Tree.ErrorList.init(arena),
+    };
+
+    var source_buffer = try std.Buffer.initSize(arena, 0);
+
+    try appendToken(tree, &source_buffer, "// TODO: implement more than just an empty source file", .LineComment);
+
+    try appendToken(tree, &source_buffer, "", .Eof);
+    tree.source = source_buffer.toOwnedSlice();
+    return tree;
+}
+
+fn appendToken(tree: *ast.Tree, source_buffer: *std.Buffer, src_text: []const u8, token_id: Token.Id) !void {
+    const start_index = source_buffer.len();
+    try source_buffer.append(src_text);
+    const end_index = source_buffer.len();
+    const new_token = try tree.tokens.addOne();
+    new_token.* = Token{
+        .id = token_id,
+        .start = start_index,
+        .end = end_index,
+    };
 }
 
 pub fn freeErrors(errors: []ClangErrMsg) void {
diff --git a/std/special/fmt_runner.zig b/std/special/fmt_runner.zig
@@ -71,26 +71,26 @@ pub fn main() !void {
         const source_code = try stdin.stream.readAllAlloc(allocator, self_hosted_main.max_src_size);
         defer allocator.free(source_code);
 
-        var tree = std.zig.parse(allocator, source_code) catch |err| {
+        const tree = std.zig.parse(allocator, source_code) catch |err| {
             try stderr.print("error parsing stdin: {}\n", err);
             os.exit(1);
         };
         defer tree.deinit();
 
         var error_it = tree.errors.iterator(0);
         while (error_it.next()) |parse_error| {
-            try printErrMsgToFile(allocator, parse_error, &tree, "<stdin>", stderr_file, color);
+            try printErrMsgToFile(allocator, parse_error, tree, "<stdin>", stderr_file, color);
         }
         if (tree.errors.len != 0) {
             os.exit(1);
         }
         if (flags.present("check")) {
-            const anything_changed = try std.zig.render(allocator, io.null_out_stream, &tree);
+            const anything_changed = try std.zig.render(allocator, io.null_out_stream, tree);
             const code = if (anything_changed) u8(1) else u8(0);
             os.exit(code);
         }
 
-        _ = try std.zig.render(allocator, stdout, &tree);
+        _ = try std.zig.render(allocator, stdout, tree);
         return;
     }
 
@@ -166,7 +166,7 @@ fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtError!void
     };
     defer fmt.allocator.free(source_code);
 
-    var tree = std.zig.parse(fmt.allocator, source_code) catch |err| {
+    const tree = std.zig.parse(fmt.allocator, source_code) catch |err| {
         try stderr.print("error parsing file '{}': {}\n", file_path, err);
         fmt.any_error = true;
         return;
@@ -175,15 +175,15 @@ fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtError!void
 
     var error_it = tree.errors.iterator(0);
     while (error_it.next()) |parse_error| {
-        try printErrMsgToFile(fmt.allocator, parse_error, &tree, file_path, stderr_file, fmt.color);
+        try printErrMsgToFile(fmt.allocator, parse_error, tree, file_path, stderr_file, fmt.color);
     }
     if (tree.errors.len != 0) {
         fmt.any_error = true;
         return;
     }
 
     if (check_mode) {
-        const anything_changed = try std.zig.render(fmt.allocator, io.null_out_stream, &tree);
+        const anything_changed = try std.zig.render(fmt.allocator, io.null_out_stream, tree);
         if (anything_changed) {
             try stderr.print("{}\n", file_path);
             fmt.any_error = true;
@@ -193,7 +193,7 @@ fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtError!void
         const baf = try io.BufferedAtomicFile.create(fmt.allocator, file_path);
         defer baf.destroy();
 
-        const anything_changed = try std.zig.render(fmt.allocator, baf.stream(), &tree);
+        const anything_changed = try std.zig.render(fmt.allocator, baf.stream(), tree);
         if (anything_changed) {
             try stderr.print("{}\n", file_path);
             try baf.finish();
@@ -210,9 +210,14 @@ const Fmt = struct {
     const SeenMap = std.HashMap([]const u8, void, mem.hash_slice_u8, mem.eql_slice_u8);
 };
 
-fn printErrMsgToFile(allocator: *mem.Allocator, parse_error: *const ast.Error, tree: *ast.Tree,
-    path: []const u8, file: os.File, color: errmsg.Color,) !void
-{
+fn printErrMsgToFile(
+    allocator: *mem.Allocator,
+    parse_error: *const ast.Error,
+    tree: *ast.Tree,
+    path: []const u8,
+    file: os.File,
+    color: errmsg.Color,
+) !void {
     const color_on = switch (color) {
         errmsg.Color.Auto => file.isTty(),
         errmsg.Color.On => true,
diff --git a/std/zig/ast.zig b/std/zig/ast.zig
@@ -18,7 +18,11 @@ pub const Tree = struct {
     pub const ErrorList = SegmentedList(Error, 0);
 
     pub fn deinit(self: *Tree) void {
-        self.arena_allocator.deinit();
+        // Here we copy the arena allocator into stack memory, because
+        // otherwise it would destroy itself while it was still working.
+        var arena_allocator = self.arena_allocator;
+        arena_allocator.deinit();
+        // self is destroyed
     }
 
     pub fn renderError(self: *Tree, parse_error: *Error, stream: var) !void {
diff --git a/std/zig/parse.zig b/std/zig/parse.zig
diff --git a/std/zig/parser_test.zig b/std/zig/parser_test.zig

Original file line number	Diff line number	Diff line change
`@@ -163,7 +163,6 @@ pub const Scope = struct {`
`163`	`163`	`pub fn destroy(self: AstTree, comp: Compilation) void {`
`164`	`164`	`comp.gpa().free(self.tree.source);`
`165`	`165`	`self.tree.deinit();`
`166`		`- comp.gpa().destroy(self.tree);`
`167`	`166`	`comp.gpa().destroy(self);`
`168`	`167`	`}`
`169`	`168`