diff --git a/lib/compiler_rt/arm.zig b/lib/compiler_rt/arm.zig index e42694b2bfa9..b2388be8adc3 100644 --- a/lib/compiler_rt/arm.zig +++ b/lib/compiler_rt/arm.zig @@ -59,15 +59,15 @@ extern fn memmove(dest: ?[*]u8, src: ?[*]const u8, n: usize) ?[*]u8; pub fn __aeabi_memcpy(dest: [*]u8, src: [*]u8, n: usize) callconv(.AAPCS) void { @setRuntimeSafety(false); - _ = memcpy(dest, src, n); + _ = @call(.never_intrinsify, memcpy, .{ dest, src, n }); } pub fn __aeabi_memcpy4(dest: [*]u8, src: [*]u8, n: usize) callconv(.AAPCS) void { @setRuntimeSafety(false); - _ = memcpy(dest, src, n); + _ = @call(.never_intrinsify, memcpy, .{ dest, src, n }); } pub fn __aeabi_memcpy8(dest: [*]u8, src: [*]u8, n: usize) callconv(.AAPCS) void { @setRuntimeSafety(false); - _ = memcpy(dest, src, n); + _ = @call(.never_intrinsify, memcpy, .{ dest, src, n }); } pub fn __aeabi_memmove(dest: [*]u8, src: [*]u8, n: usize) callconv(.AAPCS) void { diff --git a/lib/std/builtin.zig b/lib/std/builtin.zig index d7f8876b24d0..a8213d70b33e 100644 --- a/lib/std/builtin.zig +++ b/lib/std/builtin.zig @@ -843,6 +843,10 @@ pub const CallModifier = enum { /// Evaluates the call at compile-time. If the call cannot be completed at /// compile-time, a compile error is emitted instead. compile_time, + + /// Prevents intrinsifying to corresponding implementation of builtin + /// function. + never_intrinsify, }; /// This data structure is used by the Zig language code generation and diff --git a/lib/std/zig/Zir.zig b/lib/std/zig/Zir.zig index 00a48e21f7c4..a8d3f5db7f99 100644 --- a/lib/std/zig/Zir.zig +++ b/lib/std/zig/Zir.zig @@ -2718,8 +2718,8 @@ pub const Inst = struct { pub const Flags = packed struct { /// std.builtin.CallModifier in packed form - pub const PackedModifier = u3; - pub const PackedArgsLen = u27; + pub const PackedModifier = u4; + pub const PackedArgsLen = u26; packed_modifier: PackedModifier, ensure_result_used: bool = false, diff --git a/src/Air.zig b/src/Air.zig index 3aa5f317c06d..054b25baa9d0 100644 --- a/src/Air.zig +++ b/src/Air.zig @@ -314,6 +314,8 @@ pub const Inst = struct { call_never_tail, /// Same as `call` except with the `never_inline` attribute. call_never_inline, + /// Same as `call` except with the `never_intrinsify` attribute. + call_never_intrinsify, /// Count leading zeroes of an integer according to its representation in twos complement. /// Result type will always be an unsigned integer big enough to fit the answer. /// Uses the `ty_op` field. @@ -1501,7 +1503,7 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool) .tag_name, .error_name => return Type.slice_const_u8_sentinel_0, - .call, .call_always_tail, .call_never_tail, .call_never_inline => { + .call, .call_always_tail, .call_never_tail, .call_never_inline, .call_never_intrinsify => { const callee_ty = air.typeOf(datas[@intFromEnum(inst)].pl_op.operand, ip); return Type.fromInterned(ip.funcTypeReturnType(callee_ty.toIntern())); }, @@ -1620,6 +1622,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool { .call_always_tail, .call_never_tail, .call_never_inline, + .call_never_intrinsify, .cond_br, .switch_br, .loop_switch_br, diff --git a/src/Air/types_resolved.zig b/src/Air/types_resolved.zig index 098cb29b2282..fbc19f1724d5 100644 --- a/src/Air/types_resolved.zig +++ b/src/Air/types_resolved.zig @@ -330,6 +330,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool { .call_always_tail, .call_never_tail, .call_never_inline, + .call_never_intrinsify, => { const extra = air.extraData(Air.Call, data.pl_op.payload); const args: []const Air.Inst.Ref = @ptrCast(air.extra[extra.end..][0..extra.data.args_len]); diff --git a/src/Liveness.zig b/src/Liveness.zig index b5bffc6a480f..869c2265e7f5 100644 --- a/src/Liveness.zig +++ b/src/Liveness.zig @@ -482,7 +482,7 @@ pub fn categorizeOperand( return .none; }, - .call, .call_always_tail, .call_never_tail, .call_never_inline => { + .call, .call_always_tail, .call_never_tail, .call_never_inline, .call_never_intrinsify => { const inst_data = air_datas[@intFromEnum(inst)].pl_op; const callee = inst_data.operand; const extra = air.extraData(Air.Call, inst_data.payload); @@ -1109,7 +1109,7 @@ fn analyzeInst( return analyzeOperands(a, pass, data, inst, .{ prefetch.ptr, .none, .none }); }, - .call, .call_always_tail, .call_never_tail, .call_never_inline => { + .call, .call_always_tail, .call_never_tail, .call_never_inline, .call_never_intrinsify => { const inst_data = inst_datas[@intFromEnum(inst)].pl_op; const callee = inst_data.operand; const extra = a.air.extraData(Air.Call, inst_data.payload); diff --git a/src/Liveness/Verify.zig b/src/Liveness/Verify.zig index aa2239793a1d..f56c601e1d68 100644 --- a/src/Liveness/Verify.zig +++ b/src/Liveness/Verify.zig @@ -340,7 +340,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void { } try self.verifyInst(inst); }, - .call, .call_always_tail, .call_never_tail, .call_never_inline => { + .call, .call_always_tail, .call_never_tail, .call_never_inline, .call_never_intrinsify => { const pl_op = data[@intFromEnum(inst)].pl_op; const extra = self.air.extraData(Air.Call, pl_op.payload); const args = @as( diff --git a/src/Sema.zig b/src/Sema.zig index 46fe7f0a3148..d17b80c5973b 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -7602,6 +7602,7 @@ fn analyzeCall( .never_tail => Air.Inst.Tag.call_never_tail, .never_inline => Air.Inst.Tag.call_never_inline, + .never_intrinsify => Air.Inst.Tag.call_never_intrinsify, .always_tail => Air.Inst.Tag.call_always_tail, .async_kw => return sema.failWithUseOfAsync(block, call_src), @@ -25719,7 +25720,7 @@ fn zirBuiltinCall(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError var modifier = zcu.toEnum(std.builtin.CallModifier, modifier_val); switch (modifier) { // These can be upgraded to comptime or nosuspend calls. - .auto, .never_tail, .no_async => { + .auto, .never_tail, .never_intrinsify, .no_async => { if (block.is_comptime) { if (modifier == .never_tail) { return sema.fail(block, modifier_src, "unable to perform 'never_tail' call at compile-time", .{}); diff --git a/src/arch/aarch64/CodeGen.zig b/src/arch/aarch64/CodeGen.zig index 6371cf92f37b..3706089b6585 100644 --- a/src/arch/aarch64/CodeGen.zig +++ b/src/arch/aarch64/CodeGen.zig @@ -806,10 +806,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .dbg_arg_inline, => try self.airDbgVar(inst), - .call => try self.airCall(inst, .auto), - .call_always_tail => try self.airCall(inst, .always_tail), - .call_never_tail => try self.airCall(inst, .never_tail), - .call_never_inline => try self.airCall(inst, .never_inline), + .call => try self.airCall(inst, .auto), + .call_always_tail => try self.airCall(inst, .always_tail), + .call_never_tail => try self.airCall(inst, .never_tail), + .call_never_inline => try self.airCall(inst, .never_inline), + .call_never_intrinsify => try self.airCall(inst, .never_intrinsify), .atomic_store_unordered => try self.airAtomicStore(inst, .unordered), .atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic), diff --git a/src/arch/arm/CodeGen.zig b/src/arch/arm/CodeGen.zig index 99666487598d..4d05e7ed7788 100644 --- a/src/arch/arm/CodeGen.zig +++ b/src/arch/arm/CodeGen.zig @@ -793,10 +793,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .dbg_arg_inline, => try self.airDbgVar(inst), - .call => try self.airCall(inst, .auto), - .call_always_tail => try self.airCall(inst, .always_tail), - .call_never_tail => try self.airCall(inst, .never_tail), - .call_never_inline => try self.airCall(inst, .never_inline), + .call => try self.airCall(inst, .auto), + .call_always_tail => try self.airCall(inst, .always_tail), + .call_never_tail => try self.airCall(inst, .never_tail), + .call_never_inline => try self.airCall(inst, .never_inline), + .call_never_intrinsify => try self.airCall(inst, .never_intrinsify), .atomic_store_unordered => try self.airAtomicStore(inst, .unordered), .atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic), diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig index 24497defa2e7..2c44b9ebf9c5 100644 --- a/src/arch/riscv64/CodeGen.zig +++ b/src/arch/riscv64/CodeGen.zig @@ -1658,10 +1658,11 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void { .dbg_inline_block => try func.airDbgInlineBlock(inst), - .call => try func.airCall(inst, .auto), - .call_always_tail => try func.airCall(inst, .always_tail), - .call_never_tail => try func.airCall(inst, .never_tail), - .call_never_inline => try func.airCall(inst, .never_inline), + .call => try func.airCall(inst, .auto), + .call_always_tail => try func.airCall(inst, .always_tail), + .call_never_tail => try func.airCall(inst, .never_tail), + .call_never_inline => try func.airCall(inst, .never_inline), + .call_never_intrinsify => try func.airCall(inst, .never_intrinsify), .atomic_store_unordered => try func.airAtomicStore(inst, .unordered), .atomic_store_monotonic => try func.airAtomicStore(inst, .monotonic), diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig index a1bef1f4cd3a..52c4eac1fd18 100644 --- a/src/arch/sparc64/CodeGen.zig +++ b/src/arch/sparc64/CodeGen.zig @@ -648,10 +648,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .dbg_arg_inline, => try self.airDbgVar(inst), - .call => try self.airCall(inst, .auto), - .call_always_tail => try self.airCall(inst, .always_tail), - .call_never_tail => try self.airCall(inst, .never_tail), - .call_never_inline => try self.airCall(inst, .never_inline), + .call => try self.airCall(inst, .auto), + .call_always_tail => try self.airCall(inst, .always_tail), + .call_never_tail => try self.airCall(inst, .never_tail), + .call_never_inline => try self.airCall(inst, .never_inline), + .call_never_intrinsify => try self.airCall(inst, .never_intrinsify), .atomic_store_unordered => @panic("TODO try self.airAtomicStore(inst, .unordered)"), .atomic_store_monotonic => @panic("TODO try self.airAtomicStore(inst, .monotonic)"), diff --git a/src/arch/wasm/CodeGen.zig b/src/arch/wasm/CodeGen.zig index 50a8869282bc..da42352f2266 100644 --- a/src/arch/wasm/CodeGen.zig +++ b/src/arch/wasm/CodeGen.zig @@ -1933,6 +1933,7 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { .call_always_tail => func.airCall(inst, .always_tail), .call_never_tail => func.airCall(inst, .never_tail), .call_never_inline => func.airCall(inst, .never_inline), + .call_never_intrinsify => func.airCall(inst, .never_intrinsify), .is_err => func.airIsErr(inst, .i32_ne), .is_non_err => func.airIsErr(inst, .i32_eq), diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 06ae399f25ac..51485cfb9723 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -2432,10 +2432,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .dbg_arg_inline, => try self.airDbgVar(inst), - .call => try self.airCall(inst, .auto), - .call_always_tail => try self.airCall(inst, .always_tail), - .call_never_tail => try self.airCall(inst, .never_tail), - .call_never_inline => try self.airCall(inst, .never_inline), + .call => try self.airCall(inst, .auto), + .call_always_tail => try self.airCall(inst, .always_tail), + .call_never_tail => try self.airCall(inst, .never_tail), + .call_never_inline => try self.airCall(inst, .never_inline), + .call_never_intrinsify => try self.airCall(inst, .never_intrinsify), .atomic_store_unordered => try self.airAtomicStore(inst, .unordered), .atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic), diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 4d55637f27e5..effe7e8a028b 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -3407,10 +3407,11 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail, // The instruction has type `noreturn`, but there are instructions (and maybe a safety // check) following nonetheless. The `unreachable` or safety check should be emitted by // backends instead. - .call => try airCall(f, inst, .auto), - .call_always_tail => .none, - .call_never_tail => try airCall(f, inst, .never_tail), - .call_never_inline => try airCall(f, inst, .never_inline), + .call => try airCall(f, inst, .auto), + .call_always_tail => .none, + .call_never_tail => try airCall(f, inst, .never_tail), + .call_never_inline => try airCall(f, inst, .never_inline), + .call_never_intrinsify => try airCall(f, inst, .never_intrinsify), // zig fmt: on }; @@ -4540,7 +4541,7 @@ fn airCall( else => break :known, }; switch (modifier) { - .auto, .always_tail => try f.object.dg.renderNavName(writer, fn_nav), + .auto, .always_tail, .never_intrinsify => try f.object.dg.renderNavName(writer, fn_nav), inline .never_tail, .never_inline => |m| try writer.writeAll(try f.getLazyFnName(@unionInit(LazyFnKey, @tagName(m), fn_nav))), else => unreachable, } @@ -4550,6 +4551,7 @@ fn airCall( .auto, .always_tail => {}, .never_tail => return f.fail("CBE: runtime callee with never_tail attribute unsupported", .{}), .never_inline => return f.fail("CBE: runtime callee with never_inline attribute unsupported", .{}), + .never_intrinsify => return f.fail("CBE: runtime callee with never_intrinsify attribute unsupported", .{}), else => unreachable, } // Fall back to function pointer call. diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index c5bfe6f0a995..af092969aab0 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -5435,13 +5435,14 @@ pub const FuncGen = struct { if (self.typeOfIndex(inst).isNoReturn(zcu)) return; break :res res; }, - .call, .call_always_tail, .call_never_tail, .call_never_inline => |tag| res: { + .call, .call_always_tail, .call_never_tail, .call_never_inline, .call_never_intrinsify => |tag| res: { const res = try self.airCall(inst, switch (tag) { - .call => .auto, - .call_always_tail => .always_tail, - .call_never_tail => .never_tail, - .call_never_inline => .never_inline, - else => unreachable, + .call => .auto, + .call_always_tail => .always_tail, + .call_never_tail => .never_tail, + .call_never_inline => .never_inline, + .call_never_intrinsify => .never_intrinsify, + else => unreachable, }); // TODO: the AIR we emit for calls is a bit weird - the instruction has // type `noreturn`, but there are instructions (and maybe a safety check) following @@ -5585,6 +5586,7 @@ pub const FuncGen = struct { switch (modifier) { .auto, .never_tail, .always_tail => {}, .never_inline => try attributes.addFnAttr(.@"noinline", &o.builder), + .never_intrinsify => try attributes.addFnAttr(.nobuiltin, &o.builder), .async_kw, .no_async, .always_inline, .compile_time => unreachable, } @@ -5794,7 +5796,7 @@ pub const FuncGen = struct { const call = try self.wip.call( switch (modifier) { - .auto, .never_inline => .normal, + .auto, .never_inline, .never_intrinsify => .normal, .never_tail => .notail, .always_tail => .musttail, .async_kw, .no_async, .always_inline, .compile_time => unreachable, diff --git a/src/codegen/spirv.zig b/src/codegen/spirv.zig index 34497d8de8f1..7a42d33f8d0c 100644 --- a/src/codegen/spirv.zig +++ b/src/codegen/spirv.zig @@ -3518,10 +3518,11 @@ const NavGen = struct { .assembly => try self.airAssembly(inst), - .call => try self.airCall(inst, .auto), - .call_always_tail => try self.airCall(inst, .always_tail), - .call_never_tail => try self.airCall(inst, .never_tail), - .call_never_inline => try self.airCall(inst, .never_inline), + .call => try self.airCall(inst, .auto), + .call_always_tail => try self.airCall(inst, .always_tail), + .call_never_tail => try self.airCall(inst, .never_tail), + .call_never_inline => try self.airCall(inst, .never_inline), + .call_never_intrinsify => try self.airCall(inst, .never_intrinsify), .work_item_id => try self.airWorkItemId(inst), .work_group_size => try self.airWorkGroupSize(inst), diff --git a/src/print_air.zig b/src/print_air.zig index 6a3f31a27c2c..1b959bd73992 100644 --- a/src/print_air.zig +++ b/src/print_air.zig @@ -279,6 +279,7 @@ const Writer = struct { .call_always_tail, .call_never_tail, .call_never_inline, + .call_never_intrinsify, => try w.writeCall(s, inst), .dbg_var_ptr,