Skip to content

Commit 1138513

Browse files
ParfenovIgorParfenovIgor
committed
Add CallModifier.never_intrinsify
1 parent ab89af3 commit 1138513

File tree

18 files changed

+68
-47
lines changed

18 files changed

+68
-47
lines changed

‎lib/compiler_rt/arm.zig

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,15 +59,15 @@ extern fn memmove(dest: ?[*]u8, src: ?[*]const u8, n: usize) ?[*]u8;
5959

6060
pub fn __aeabi_memcpy(dest: [*]u8, src: [*]u8, n: usize) callconv(.AAPCS) void {
6161
@setRuntimeSafety(false);
62-
_ = memcpy(dest, src, n);
62+
_ = @call(.never_intrinsify, memcpy, .{dest, src, n});
6363
}
6464
pub fn __aeabi_memcpy4(dest: [*]u8, src: [*]u8, n: usize) callconv(.AAPCS) void {
6565
@setRuntimeSafety(false);
66-
_ = memcpy(dest, src, n);
66+
_ = @call(.never_intrinsify, memcpy, .{dest, src, n});
6767
}
6868
pub fn __aeabi_memcpy8(dest: [*]u8, src: [*]u8, n: usize) callconv(.AAPCS) void {
6969
@setRuntimeSafety(false);
70-
_ = memcpy(dest, src, n);
70+
_ = @call(.never_intrinsify, memcpy, .{dest, src, n});
7171
}
7272

7373
pub fn __aeabi_memmove(dest: [*]u8, src: [*]u8, n: usize) callconv(.AAPCS) void {

‎lib/std/builtin.zig

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -843,6 +843,10 @@ pub const CallModifier = enum {
843843
/// Evaluates the call at compile-time. If the call cannot be completed at
844844
/// compile-time, a compile error is emitted instead.
845845
compile_time,
846+
847+
/// Prevents intrinsifying to corresponding implementation of builtin
848+
/// function.
849+
never_intrinsify,
846850
};
847851

848852
/// This data structure is used by the Zig language code generation and

‎lib/std/zig/Zir.zig

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2718,8 +2718,8 @@ pub const Inst = struct {
27182718

27192719
pub const Flags = packed struct {
27202720
/// std.builtin.CallModifier in packed form
2721-
pub const PackedModifier = u3;
2722-
pub const PackedArgsLen = u27;
2721+
pub const PackedModifier = u4;
2722+
pub const PackedArgsLen = u26;
27232723

27242724
packed_modifier: PackedModifier,
27252725
ensure_result_used: bool = false,

‎src/Air.zig

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,8 @@ pub const Inst = struct {
314314
call_never_tail,
315315
/// Same as `call` except with the `never_inline` attribute.
316316
call_never_inline,
317+
/// Same as `call` except with the `never_intrinsify` attribute.
318+
call_never_intrinsify,
317319
/// Count leading zeroes of an integer according to its representation in twos complement.
318320
/// Result type will always be an unsigned integer big enough to fit the answer.
319321
/// Uses the `ty_op` field.
@@ -1501,7 +1503,7 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool)
15011503

15021504
.tag_name, .error_name => return Type.slice_const_u8_sentinel_0,
15031505

1504-
.call, .call_always_tail, .call_never_tail, .call_never_inline => {
1506+
.call, .call_always_tail, .call_never_tail, .call_never_inline, .call_never_intrinsify => {
15051507
const callee_ty = air.typeOf(datas[@intFromEnum(inst)].pl_op.operand, ip);
15061508
return Type.fromInterned(ip.funcTypeReturnType(callee_ty.toIntern()));
15071509
},
@@ -1620,6 +1622,7 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool {
16201622
.call_always_tail,
16211623
.call_never_tail,
16221624
.call_never_inline,
1625+
.call_never_intrinsify,
16231626
.cond_br,
16241627
.switch_br,
16251628
.loop_switch_br,

‎src/Air/types_resolved.zig

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,7 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool {
330330
.call_always_tail,
331331
.call_never_tail,
332332
.call_never_inline,
333+
.call_never_intrinsify,
333334
=> {
334335
const extra = air.extraData(Air.Call, data.pl_op.payload);
335336
const args: []const Air.Inst.Ref = @ptrCast(air.extra[extra.end..][0..extra.data.args_len]);

‎src/Liveness.zig

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -482,7 +482,7 @@ pub fn categorizeOperand(
482482
return .none;
483483
},
484484

485-
.call, .call_always_tail, .call_never_tail, .call_never_inline => {
485+
.call, .call_always_tail, .call_never_tail, .call_never_inline, .call_never_intrinsify => {
486486
const inst_data = air_datas[@intFromEnum(inst)].pl_op;
487487
const callee = inst_data.operand;
488488
const extra = air.extraData(Air.Call, inst_data.payload);
@@ -1109,7 +1109,7 @@ fn analyzeInst(
11091109
return analyzeOperands(a, pass, data, inst, .{ prefetch.ptr, .none, .none });
11101110
},
11111111

1112-
.call, .call_always_tail, .call_never_tail, .call_never_inline => {
1112+
.call, .call_always_tail, .call_never_tail, .call_never_inline, .call_never_intrinsify => {
11131113
const inst_data = inst_datas[@intFromEnum(inst)].pl_op;
11141114
const callee = inst_data.operand;
11151115
const extra = a.air.extraData(Air.Call, inst_data.payload);

‎src/Liveness/Verify.zig

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -340,7 +340,7 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void {
340340
}
341341
try self.verifyInst(inst);
342342
},
343-
.call, .call_always_tail, .call_never_tail, .call_never_inline => {
343+
.call, .call_always_tail, .call_never_tail, .call_never_inline, .call_never_intrinsify => {
344344
const pl_op = data[@intFromEnum(inst)].pl_op;
345345
const extra = self.air.extraData(Air.Call, pl_op.payload);
346346
const args = @as(

‎src/Sema.zig

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7078,7 +7078,7 @@ fn zirCall(
70787078
};
70797079
const extra = sema.code.extraData(ExtraType, inst_data.payload_index);
70807080
const args_len = extra.data.flags.args_len;
7081-
7081+
70827082
const modifier: std.builtin.CallModifier = @enumFromInt(extra.data.flags.packed_modifier);
70837083
const ensure_result_used = extra.data.flags.ensure_result_used;
70847084
const pop_error_return_trace = extra.data.flags.pop_error_return_trace;
@@ -7602,6 +7602,7 @@ fn analyzeCall(
76027602

76037603
.never_tail => Air.Inst.Tag.call_never_tail,
76047604
.never_inline => Air.Inst.Tag.call_never_inline,
7605+
.never_intrinsify => Air.Inst.Tag.call_never_intrinsify,
76057606
.always_tail => Air.Inst.Tag.call_always_tail,
76067607

76077608
.async_kw => return sema.failWithUseOfAsync(block, call_src),
@@ -25719,7 +25720,7 @@ fn zirBuiltinCall(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError
2571925720
var modifier = zcu.toEnum(std.builtin.CallModifier, modifier_val);
2572025721
switch (modifier) {
2572125722
// These can be upgraded to comptime or nosuspend calls.
25722-
.auto, .never_tail, .no_async => {
25723+
.auto, .never_tail, .never_intrinsify, .no_async => {
2572325724
if (block.is_comptime) {
2572425725
if (modifier == .never_tail) {
2572525726
return sema.fail(block, modifier_src, "unable to perform 'never_tail' call at compile-time", .{});

‎src/arch/aarch64/CodeGen.zig

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -806,10 +806,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
806806
.dbg_arg_inline,
807807
=> try self.airDbgVar(inst),
808808

809-
.call => try self.airCall(inst, .auto),
810-
.call_always_tail => try self.airCall(inst, .always_tail),
811-
.call_never_tail => try self.airCall(inst, .never_tail),
812-
.call_never_inline => try self.airCall(inst, .never_inline),
809+
.call => try self.airCall(inst, .auto),
810+
.call_always_tail => try self.airCall(inst, .always_tail),
811+
.call_never_tail => try self.airCall(inst, .never_tail),
812+
.call_never_inline => try self.airCall(inst, .never_inline),
813+
.call_never_intrinsify => try self.airCall(inst, .never_intrinsify),
813814

814815
.atomic_store_unordered => try self.airAtomicStore(inst, .unordered),
815816
.atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic),

‎src/arch/arm/CodeGen.zig

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -793,10 +793,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
793793
.dbg_arg_inline,
794794
=> try self.airDbgVar(inst),
795795

796-
.call => try self.airCall(inst, .auto),
797-
.call_always_tail => try self.airCall(inst, .always_tail),
798-
.call_never_tail => try self.airCall(inst, .never_tail),
799-
.call_never_inline => try self.airCall(inst, .never_inline),
796+
.call => try self.airCall(inst, .auto),
797+
.call_always_tail => try self.airCall(inst, .always_tail),
798+
.call_never_tail => try self.airCall(inst, .never_tail),
799+
.call_never_inline => try self.airCall(inst, .never_inline),
800+
.call_never_intrinsify => try self.airCall(inst, .never_intrinsify),
800801

801802
.atomic_store_unordered => try self.airAtomicStore(inst, .unordered),
802803
.atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic),

‎src/arch/riscv64/CodeGen.zig

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1658,10 +1658,11 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void {
16581658

16591659
.dbg_inline_block => try func.airDbgInlineBlock(inst),
16601660

1661-
.call => try func.airCall(inst, .auto),
1662-
.call_always_tail => try func.airCall(inst, .always_tail),
1663-
.call_never_tail => try func.airCall(inst, .never_tail),
1664-
.call_never_inline => try func.airCall(inst, .never_inline),
1661+
.call => try func.airCall(inst, .auto),
1662+
.call_always_tail => try func.airCall(inst, .always_tail),
1663+
.call_never_tail => try func.airCall(inst, .never_tail),
1664+
.call_never_inline => try func.airCall(inst, .never_inline),
1665+
.call_never_intrinsify => try func.airCall(inst, .never_intrinsify),
16651666

16661667
.atomic_store_unordered => try func.airAtomicStore(inst, .unordered),
16671668
.atomic_store_monotonic => try func.airAtomicStore(inst, .monotonic),

‎src/arch/sparc64/CodeGen.zig

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -648,10 +648,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
648648
.dbg_arg_inline,
649649
=> try self.airDbgVar(inst),
650650

651-
.call => try self.airCall(inst, .auto),
652-
.call_always_tail => try self.airCall(inst, .always_tail),
653-
.call_never_tail => try self.airCall(inst, .never_tail),
654-
.call_never_inline => try self.airCall(inst, .never_inline),
651+
.call => try self.airCall(inst, .auto),
652+
.call_always_tail => try self.airCall(inst, .always_tail),
653+
.call_never_tail => try self.airCall(inst, .never_tail),
654+
.call_never_inline => try self.airCall(inst, .never_inline),
655+
.call_never_intrinsify => try self.airCall(inst, .never_intrinsify),
655656

656657
.atomic_store_unordered => @panic("TODO try self.airAtomicStore(inst, .unordered)"),
657658
.atomic_store_monotonic => @panic("TODO try self.airAtomicStore(inst, .monotonic)"),

‎src/arch/wasm/CodeGen.zig

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1933,6 +1933,7 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void {
19331933
.call_always_tail => func.airCall(inst, .always_tail),
19341934
.call_never_tail => func.airCall(inst, .never_tail),
19351935
.call_never_inline => func.airCall(inst, .never_inline),
1936+
.call_never_intrinsify => func.airCall(inst, .never_intrinsify),
19361937

19371938
.is_err => func.airIsErr(inst, .i32_ne),
19381939
.is_non_err => func.airIsErr(inst, .i32_eq),

‎src/arch/x86_64/CodeGen.zig

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2432,10 +2432,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void {
24322432
.dbg_arg_inline,
24332433
=> try self.airDbgVar(inst),
24342434

2435-
.call => try self.airCall(inst, .auto),
2436-
.call_always_tail => try self.airCall(inst, .always_tail),
2437-
.call_never_tail => try self.airCall(inst, .never_tail),
2438-
.call_never_inline => try self.airCall(inst, .never_inline),
2435+
.call => try self.airCall(inst, .auto),
2436+
.call_always_tail => try self.airCall(inst, .always_tail),
2437+
.call_never_tail => try self.airCall(inst, .never_tail),
2438+
.call_never_inline => try self.airCall(inst, .never_inline),
2439+
.call_never_intrinsify => try self.airCall(inst, .never_intrinsify),
24392440

24402441
.atomic_store_unordered => try self.airAtomicStore(inst, .unordered),
24412442
.atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic),

‎src/codegen/c.zig

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3407,10 +3407,11 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) error{ AnalysisFail,
34073407
// The instruction has type `noreturn`, but there are instructions (and maybe a safety
34083408
// check) following nonetheless. The `unreachable` or safety check should be emitted by
34093409
// backends instead.
3410-
.call => try airCall(f, inst, .auto),
3411-
.call_always_tail => .none,
3412-
.call_never_tail => try airCall(f, inst, .never_tail),
3413-
.call_never_inline => try airCall(f, inst, .never_inline),
3410+
.call => try airCall(f, inst, .auto),
3411+
.call_always_tail => .none,
3412+
.call_never_tail => try airCall(f, inst, .never_tail),
3413+
.call_never_inline => try airCall(f, inst, .never_inline),
3414+
.call_never_intrinsify => try airCall(f, inst, .never_intrinsify),
34143415

34153416
// zig fmt: on
34163417
};
@@ -4540,7 +4541,7 @@ fn airCall(
45404541
else => break :known,
45414542
};
45424543
switch (modifier) {
4543-
.auto, .always_tail => try f.object.dg.renderNavName(writer, fn_nav),
4544+
.auto, .always_tail, .never_intrinsify => try f.object.dg.renderNavName(writer, fn_nav),
45444545
inline .never_tail, .never_inline => |m| try writer.writeAll(try f.getLazyFnName(@unionInit(LazyFnKey, @tagName(m), fn_nav))),
45454546
else => unreachable,
45464547
}
@@ -4550,6 +4551,7 @@ fn airCall(
45504551
.auto, .always_tail => {},
45514552
.never_tail => return f.fail("CBE: runtime callee with never_tail attribute unsupported", .{}),
45524553
.never_inline => return f.fail("CBE: runtime callee with never_inline attribute unsupported", .{}),
4554+
.never_intrinsify => return f.fail("CBE: runtime callee with never_intrinsify attribute unsupported", .{}),
45534555
else => unreachable,
45544556
}
45554557
// Fall back to function pointer call.

‎src/codegen/llvm.zig

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5435,13 +5435,14 @@ pub const FuncGen = struct {
54355435
if (self.typeOfIndex(inst).isNoReturn(zcu)) return;
54365436
break :res res;
54375437
},
5438-
.call, .call_always_tail, .call_never_tail, .call_never_inline => |tag| res: {
5438+
.call, .call_always_tail, .call_never_tail, .call_never_inline, .call_never_intrinsify => |tag| res: {
54395439
const res = try self.airCall(inst, switch (tag) {
5440-
.call => .auto,
5441-
.call_always_tail => .always_tail,
5442-
.call_never_tail => .never_tail,
5443-
.call_never_inline => .never_inline,
5444-
else => unreachable,
5440+
.call => .auto,
5441+
.call_always_tail => .always_tail,
5442+
.call_never_tail => .never_tail,
5443+
.call_never_inline => .never_inline,
5444+
.call_never_intrinsify => .never_intrinsify,
5445+
else => unreachable,
54455446
});
54465447
// TODO: the AIR we emit for calls is a bit weird - the instruction has
54475448
// type `noreturn`, but there are instructions (and maybe a safety check) following
@@ -5585,6 +5586,7 @@ pub const FuncGen = struct {
55855586
switch (modifier) {
55865587
.auto, .never_tail, .always_tail => {},
55875588
.never_inline => try attributes.addFnAttr(.@"noinline", &o.builder),
5589+
.never_intrinsify => try attributes.addFnAttr(.nobuiltin, &o.builder),
55885590
.async_kw, .no_async, .always_inline, .compile_time => unreachable,
55895591
}
55905592

@@ -5794,7 +5796,7 @@ pub const FuncGen = struct {
57945796

57955797
const call = try self.wip.call(
57965798
switch (modifier) {
5797-
.auto, .never_inline => .normal,
5799+
.auto, .never_inline, .never_intrinsify => .normal,
57985800
.never_tail => .notail,
57995801
.always_tail => .musttail,
58005802
.async_kw, .no_async, .always_inline, .compile_time => unreachable,

‎src/codegen/spirv.zig

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3518,10 +3518,11 @@ const NavGen = struct {
35183518

35193519
.assembly => try self.airAssembly(inst),
35203520

3521-
.call => try self.airCall(inst, .auto),
3522-
.call_always_tail => try self.airCall(inst, .always_tail),
3523-
.call_never_tail => try self.airCall(inst, .never_tail),
3524-
.call_never_inline => try self.airCall(inst, .never_inline),
3521+
.call => try self.airCall(inst, .auto),
3522+
.call_always_tail => try self.airCall(inst, .always_tail),
3523+
.call_never_tail => try self.airCall(inst, .never_tail),
3524+
.call_never_inline => try self.airCall(inst, .never_inline),
3525+
.call_never_intrinsify => try self.airCall(inst, .never_intrinsify),
35253526

35263527
.work_item_id => try self.airWorkItemId(inst),
35273528
.work_group_size => try self.airWorkGroupSize(inst),

‎src/print_air.zig

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,7 @@ const Writer = struct {
279279
.call_always_tail,
280280
.call_never_tail,
281281
.call_never_inline,
282+
.call_never_intrinsify,
282283
=> try w.writeCall(s, inst),
283284

284285
.dbg_var_ptr,

0 commit comments

Comments
 (0)