Add event loop and handle JS async code

*Async JS*

For now only callback style is handled (Promises is planned later).

We use persistent handle on v8 JS callback call after retrieving the
event from the kernel, has the parent JS function is finished and
therefore local handles are already garbage collected by v8.

* Event Loop*

We do not use the event loop provided in Zig stdlib but instead Tigerbeetle
IO (https://github.com/tigerbeetledb/tigerbeetle/tree/main/src/io).
The main reason is to have a strictly single-threaded event loop, see
ziglang/zig#1908.
In addition the desing of Tigerbeetle IO based on io_uring (for Linux,
with wrapper around kqueue for MacOS), seems to be the right direction for IO.

Our loop provides callback style native APIs. Async/await style native
API are not planned until zig self-hosted compiler (stage2) support
concurrent features (see ziglang/zig#6025).

Signed-off-by: Francis Bouvier <[email protected]>

Author: francisbouvier

build.zig

@@ -10,6 +10,7 @@ pub fn build(b: *std.build.Builder) !void {
     exe.setBuildMode(mode);
     exe.single_threaded = true;
     try linkV8(exe, mode);
+    addDeps(exe);
     if (mode == .ReleaseSafe) {
         // remove debug info
         // TODO: check if mandatory in release-safe
@@ -33,11 +34,22 @@ pub fn build(b: *std.build.Builder) !void {
     test_exe.setBuildMode(mode);
     test_exe.single_threaded = true;
     try linkV8(test_exe, mode);
+    addDeps(test_exe);
 
     const test_step = b.step("test", "Run unit tests");
     test_step.dependOn(&test_exe.step);
 }
 
+fn addDeps(step: *std.build.LibExeObjStep) void {
+    // tigerbeetle IO loop
+    // incompatible stage1 and stage2 versions
+    if (step.builder.use_stage1 != null and step.builder.use_stage1.?) {
+        step.addPackagePath("tigerbeetle-io", "deps/tigerbeetle-io/io_stage1.zig");
+    } else {
+        step.addPackagePath("tigerbeetle-io", "deps/tigerbeetle-io/io.zig");
+    }
+}
+
 fn linkV8(step: *std.build.LibExeObjStep, mode: std.builtin.Mode) !void {
     const mode_str: []const u8 = if (mode == .Debug) "debug" else "release";
     // step.linkLibC(); // TODO: do we need to link libc?

src/callback.zig

@@ -0,0 +1,193 @@
+const std = @import("std");
+
+const v8 = @import("v8"); // TODO: remove
+
+const refl = @import("reflect.zig");
+const utils = @import("utils.zig");
+const Loop = @import("loop.zig").SingleThreaded;
+
+// TODO: Make this JS engine agnostic
+// by providing a common interface
+
+pub const Arg = struct {
+    // TODO: it's required to have a non-empty struct
+    // otherwise LLVM emits a warning
+    // "stack frame size (x) exceeds limit (y)"
+    foo: bool = false,
+};
+
+// TODO: set the correct "this" on Func object
+// see https://developer.mozilla.org/en-US/docs/Web/API/setTimeout#the_this_problem
+// should we use:
+// - the context globals?
+// - null?
+// - the calling function (info.getThis)?
+
+pub const FuncSync = struct {
+    js_func: v8.Function,
+    js_args: []v8.Value,
+    isolate: v8.Isolate,
+
+    pub fn init(
+        alloc: std.mem.Allocator,
+        comptime func: refl.Func,
+        info: v8.FunctionCallbackInfo,
+        isolate: v8.Isolate,
+    ) !FuncSync {
+
+        // retrieve callback arguments indexes
+        // TODO: Should we do that at reflection?
+        comptime var js_args_indexes: [func.args_callback_nb]usize = undefined;
+        comptime var x: usize = 0;
+        inline for (func.args) |arg, i| {
+            if (arg.T == Arg) {
+                js_args_indexes[x] = i;
+                x += 1;
+            }
+        }
+
+        // retrieve callback arguments
+        // var js_args: [func.args_callback_nb]v8.Value = undefined;
+        var js_args = try alloc.alloc(v8.Value, func.args_callback_nb);
+        for (js_args_indexes) |index, i| {
+            js_args[i] = info.getArg(@intCast(u32, index - func.index_offset));
+        }
+
+        // retrieve callback function
+        const js_func_index = func.callback_index.? - func.index_offset - 1; // -1 because of self
+        const js_func_val = info.getArg(js_func_index);
+        if (!js_func_val.isFunction()) {
+            return error.JSWrongType;
+        }
+        const js_func = js_func_val.castTo(v8.Function);
+
+        return FuncSync{
+            .js_func = js_func,
+            .js_args = js_args,
+            .isolate = isolate,
+        };
+    }
+
+    pub fn call(self: FuncSync, alloc: std.mem.Allocator) void {
+
+        // retrieve context
+        // NOTE: match the Func.call implementation
+        const ctx = self.isolate.getCurrentContext();
+
+        // retrieve JS this from persistent handle
+        // TODO: see correct "this" comment above
+        const this = ctx.getGlobal();
+
+        // execute function
+        _ = self.js_func.call(ctx, this, self.js_args);
+
+        // free heap
+        alloc.free(self.js_args);
+    }
+};
+
+const PersistentFunction = v8.Persistent(v8.Function);
+const PersistentValue = v8.Persistent(v8.Value);
+
+pub const Func = struct {
+
+    // NOTE: we use persistent handles here
+    // to ensure the references are not garbage collected
+    // at the end of the JS calling function execution.
+    js_func_pers: *PersistentFunction,
+    js_args_pers: []PersistentValue,
+
+    isolate: v8.Isolate,
+
+    pub fn init(
+        // NOTE: we need to store the JS callback arguments on the heap
+        // as the call method will be executed in another stack frame,
+        // once the asynchronous operation will be fetched back from the kernel.
+        alloc: std.mem.Allocator,
+        comptime func: refl.Func,
+        info: v8.FunctionCallbackInfo,
+        isolate: v8.Isolate,
+    ) !Func {
+
+        // retrieve callback arguments indexes
+        // TODO: Should we do that at reflection?
+        comptime var js_args_indexes: [func.args_callback_nb]usize = undefined;
+        comptime var x: usize = 0;
+        inline for (func.args) |arg, i| {
+            if (arg.T == Arg) {
+                js_args_indexes[x] = i;
+                x += 1;
+            }
+        }
+
+        // retrieve callback arguments
+        var js_args_pers = try alloc.alloc(PersistentValue, func.args_callback_nb);
+        for (js_args_indexes) |index, i| {
+            const js_arg = info.getArg(@intCast(u32, index - func.index_offset));
+            const js_arg_pers = PersistentValue.init(isolate, js_arg);
+            js_args_pers[i] = js_arg_pers;
+        }
+
+        // retrieve callback function
+        const js_func_index = func.callback_index.? - func.index_offset - 1; // -1 because of self
+        const js_func_val = info.getArg(js_func_index);
+        if (!js_func_val.isFunction()) {
+            return error.JSWrongType;
+        }
+        const js_func = js_func_val.castTo(v8.Function);
+
+        // const js_func_pers = PersistentFunction.init(isolate, js_func);
+        var js_func_pers = try alloc.create(PersistentFunction);
+        js_func_pers.* = PersistentFunction.init(isolate, js_func);
+
+        return Func{
+            .js_func_pers = js_func_pers,
+            .js_args_pers = js_args_pers,
+            .isolate = isolate,
+        };
+    }
+
+    fn deinit(self: Func, alloc: std.mem.Allocator) void {
+        // cleanup persistent references in v8
+        var js_func_pers = self.js_func_pers; // TODO: why do we need var here?
+        js_func_pers.deinit();
+
+        for (self.js_args_pers) |arg| {
+            var arg_pers = arg; // TODO: why do we need var here?
+            arg_pers.deinit();
+        }
+
+        // free heap
+        alloc.free(self.js_args_pers);
+        alloc.destroy(self.js_func_pers);
+    }
+
+    pub fn call(self: Func, alloc: std.mem.Allocator) !void {
+        defer self.deinit(alloc);
+
+        // retrieve context
+        // TODO: should we instead store the original context in the Func object?
+        // in this case we need to have a permanent handle (Global ?) on it.
+        const ctx = self.isolate.getCurrentContext();
+
+        // retrieve JS function from persistent handle
+        const js_func = self.js_func_pers.castToFunction();
+
+        // retrieve JS arguments from persistent handle
+        const js_args = try alloc.alloc(v8.Value, self.js_args_pers.len);
+        defer alloc.free(js_args);
+        for (self.js_args_pers) |arg, i| {
+            js_args[i] = arg.toValue();
+        }
+
+        // retrieve JS "this" from persistent handle
+        // TODO: see correct "this" comment above
+        const this = ctx.getGlobal();
+
+        // execute function
+        const result = js_func.call(ctx, this, js_args);
+        if (result == null) {
+            return error.JSCallback;
+        }
+    }
+};

src/cbk_test.zig

@@ -4,36 +4,55 @@ const v8 = @import("v8");
 const utils = @import("utils.zig");
 const gen = @import("generate.zig");
 const eng = @import("engine.zig");
+const Loop = @import("loop.zig").SingleThreaded;
+
+const u64Num = @import("types.zig").u64Num;
+const cbk = @import("callback.zig");
+
 const tests = @import("test_utils.zig");
 
+const Console = @import("console.zig").Console;
+
 const Window = struct {
-    // value: u32,
+    pub fn constructor() Window {
+        return Window{};
+    }
 
-    const Callback = @import("types.zig").Callback;
-    const CallbackArg = @import("types.zig").CallbackArg;
+    pub fn cbkSyncWithoutArg(_: Window, _: cbk.FuncSync) void {
+        tests.sleep(1 * std.time.ns_per_ms);
+    }
 
-    pub fn constructor(_: u32) Window {
-        return Window{};
+    pub fn cbkSyncWithArg(_: Window, _: cbk.FuncSync, _: cbk.Arg) void {
+        tests.sleep(1 * std.time.ns_per_ms);
     }
 
-    pub fn cbkSyncWithoutArg(_: Window, _: Callback) void {
-        // TODO: handle async
-        std.time.sleep(1 * std.time.ns_per_ms);
+    pub fn cbkAsync(_: Window, loop: *Loop, callback: cbk.Func, milliseconds: u32) void {
+        const n = @intCast(u63, milliseconds);
+        // TODO: check this value can be holded in u63
+        loop.timeout(n * std.time.ns_per_ms, callback);
     }
 
-    pub fn cbkSyncWithArg(_: Window, _: Callback, _: CallbackArg) void {
-        // TODO: handle async
-        std.time.sleep(1 * std.time.ns_per_ms);
+    pub fn cbkAsyncWithArg(
+        _: Window,
+        loop: *Loop,
+        callback: cbk.Func,
+        milliseconds: u32,
+        _: cbk.Arg,
+    ) void {
+        const n = @intCast(u63, milliseconds);
+        // TODO: check this value can be holded in u63
+        loop.timeout(n * std.time.ns_per_ms, callback);
     }
 };
 
 // generate API, comptime
 pub fn generate() []gen.API {
-    return gen.compile(.{Window});
+    return gen.compile(.{ Console, Window });
 }
 
 // exec tests
 pub fn exec(
+    loop: *Loop,
     isolate: v8.Isolate,
     globals: v8.ObjectTemplate,
     tpls: []gen.ProtoTpl,
@@ -45,11 +64,21 @@ pub fn exec(
     context.enter();
     defer context.exit();
 
+    // console
+    _ = try eng.createV8Object(
+        utils.allocator,
+        isolate,
+        context,
+        context.getGlobal(),
+        tpls[0].tpl,
+        apis[0].T_refl,
+    );
+
     // constructor
     const case_cstr = [_]tests.Case{
-        .{ .src = "let window = new Window(0);", .ex = "undefined" },
+        .{ .src = "let window = new Window();", .ex = "undefined" },
     };
-    try tests.checkCases(utils.allocator, isolate, context, case_cstr.len, case_cstr);
+    try tests.checkCases(loop, utils.allocator, isolate, context, case_cstr.len, case_cstr);
 
     // cbkSyncWithoutArg
     const cases_cbk_sync_without_arg = [_]tests.Case{
@@ -73,9 +102,9 @@ pub fn exec(
         },
         .{ .src = "m;", .ex = "2" },
     };
-    try tests.checkCases(utils.allocator, isolate, context, cases_cbk_sync_without_arg.len, cases_cbk_sync_without_arg);
+    try tests.checkCases(loop, utils.allocator, isolate, context, cases_cbk_sync_without_arg.len, cases_cbk_sync_without_arg);
 
-    // cbkSyncWithoutArg
+    // cbkSyncWithArg
     const cases_cbk_sync_with_arg = [_]tests.Case{
         // traditional anonymous function
         .{
@@ -97,7 +126,63 @@ pub fn exec(
         },
         .{ .src = "y;", .ex = "3" },
     };
-    try tests.checkCases(utils.allocator, isolate, context, cases_cbk_sync_with_arg.len, cases_cbk_sync_with_arg);
+    try tests.checkCases(loop, utils.allocator, isolate, context, cases_cbk_sync_with_arg.len, cases_cbk_sync_with_arg);
+
+    // cbkAsync
+    const cases_cbk_async = [_]tests.Case{
+        // traditional anonymous function
+        .{
+            .src = 
+            \\let o = 1;
+            \\function f() {
+            \\o++;
+            \\if (o != 2) {throw Error('cases_cbk_async error: o is not equal to 2');}
+            \\};
+            \\window.cbkAsync(f, 300); // 0.3 second
+            ,
+            .ex = "undefined",
+        },
+        // arrow functional
+        .{
+            .src = 
+            \\let p = 1;
+            \\window.cbkAsync(() => {
+            \\p++;
+            \\if (p != 2) {throw Error('cases_cbk_async error: p is not equal to 2');}
+            \\}, 300); // 0.3 second
+            ,
+            .ex = "undefined",
+        },
+    };
+    try tests.checkCases(loop, utils.allocator, isolate, context, cases_cbk_async.len, cases_cbk_async);
+
+    // cbkAsyncWithArg
+    const cases_cbk_async_with_arg = [_]tests.Case{
+        // traditional anonymous function
+        .{
+            .src = 
+            \\let i = 1;
+            \\function f(a) {
+            \\i = i + a;
+            \\if (i != 3) {throw Error('i is not equal to 3');}
+            \\};
+            \\window.cbkAsyncWithArg(f, 300, 2); // 0.3 second
+            ,
+            .ex = "undefined",
+        },
+        // arrow functional
+        .{
+            .src = 
+            \\let j = 1;
+            \\window.cbkAsyncWithArg((a) => {
+            \\j = j + a;
+            \\if (j != 3) {throw Error('j is not equal to 3');}
+            \\}, 300, 2); // 0.3 second
+            ,
+            .ex = "undefined",
+        },
+    };
+    try tests.checkCases(loop, utils.allocator, isolate, context, cases_cbk_async_with_arg.len, cases_cbk_async_with_arg);
 
     return eng.ExecOK;
 }