Merge pull request #22154 from alexrp/disable-intrinsics

compiler: Implement `@disableIntrinsics()` builtin function.

Author: alexrp

lib/std/zig/AstGen.zig

@@ -2956,6 +2956,7 @@ fn addEnsureResult(gz: *GenZir, maybe_unused_result: Zir.Inst.Ref, statement: As
             .extended => switch (gz.astgen.instructions.items(.data)[@intFromEnum(inst)].extended.opcode) {
                 .breakpoint,
                 .disable_instrumentation,
+                .disable_intrinsics,
                 .set_float_mode,
                 .branch_hint,
                 => break :b true,
@@ -9578,6 +9579,7 @@ fn builtinCall(
         .frame_address           => return rvalue(gz, ri, try gz.addNodeExtended(.frame_address,           node), node),
         .breakpoint              => return rvalue(gz, ri, try gz.addNodeExtended(.breakpoint,              node), node),
         .disable_instrumentation => return rvalue(gz, ri, try gz.addNodeExtended(.disable_instrumentation, node), node),
+        .disable_intrinsics      => return rvalue(gz, ri, try gz.addNodeExtended(.disable_intrinsics,      node), node),
 
         .type_info   => return simpleUnOpType(gz, scope, ri, node, params[0], .type_info),
         .size_of     => return simpleUnOpType(gz, scope, ri, node, params[0], .size_of),

lib/std/zig/AstRlAnnotate.zig

@@ -882,6 +882,7 @@ fn builtinCall(astrl: *AstRlAnnotate, block: ?*Block, ri: ResultInfo, node: Ast.
         .frame,
         .breakpoint,
         .disable_instrumentation,
+        .disable_intrinsics,
         .in_comptime,
         .panic,
         .trap,

lib/std/zig/BuiltinFn.zig

@@ -15,6 +15,7 @@ pub const Tag = enum {
     branch_hint,
     breakpoint,
     disable_instrumentation,
+    disable_intrinsics,
     mul_add,
     byte_swap,
     bit_reverse,
@@ -262,6 +263,14 @@ pub const list = list: {
                 .illegal_outside_function = true,
             },
         },
+        .{
+            "@disableIntrinsics",
+            .{
+                .tag = .disable_intrinsics,
+                .param_count = 0,
+                .illegal_outside_function = true,
+            },
+        },
         .{
             "@mulAdd",
             .{

lib/std/zig/Zir.zig

@@ -1587,7 +1587,11 @@ pub const Inst = struct {
                 => false,
 
                 .extended => switch (data.extended.opcode) {
-                    .branch_hint, .breakpoint, .disable_instrumentation => true,
+                    .branch_hint,
+                    .breakpoint,
+                    .disable_instrumentation,
+                    .disable_intrinsics,
+                    => true,
                     else => false,
                 },
             };
@@ -2004,6 +2008,8 @@ pub const Inst = struct {
         breakpoint,
         /// Implement builtin `@disableInstrumentation`. `operand` is `src_node: i32`.
         disable_instrumentation,
+        /// Implement builtin `@disableIntrinsics`. `operand` is `src_node: i32`.
+        disable_intrinsics,
         /// Implements the `@select` builtin.
         /// `operand` is payload index to `Select`.
         select,
@@ -4332,6 +4338,7 @@ fn findTrackableInner(
                 .await_nosuspend,
                 .breakpoint,
                 .disable_instrumentation,
+                .disable_intrinsics,
                 .select,
                 .int_from_error,
                 .error_from_int,

lib/zig.h

@@ -223,6 +223,12 @@
 #define zig_restrict
 #endif
 
+#if zig_has_attribute(no_builtin)
+#define zig_no_builtin __attribute__((no_builtin))
+#else
+#define zig_no_builtin
+#endif
+
 #if zig_has_attribute(aligned) || defined(zig_tinyc)
 #define zig_under_align(alignment) __attribute__((aligned(alignment)))
 #elif defined(zig_msvc)

src/InternPool.zig

@@ -6045,8 +6045,9 @@ pub const FuncAnalysis = packed struct(u32) {
     /// True if this function has an inferred error set.
     inferred_error_set: bool,
     disable_instrumentation: bool,
+    disable_intrinsics: bool,
 
-    _: u24 = 0,
+    _: u23 = 0,
 };
 
 pub const Bytes = struct {
@@ -9077,6 +9078,7 @@ pub fn getFuncDecl(
             .has_error_trace = false,
             .inferred_error_set = false,
             .disable_instrumentation = false,
+            .disable_intrinsics = false,
         },
         .owner_nav = key.owner_nav,
         .ty = key.ty,
@@ -9186,6 +9188,7 @@ pub fn getFuncDeclIes(
             .has_error_trace = false,
             .inferred_error_set = true,
             .disable_instrumentation = false,
+            .disable_intrinsics = false,
         },
         .owner_nav = key.owner_nav,
         .ty = func_ty,
@@ -9382,6 +9385,7 @@ pub fn getFuncInstance(
             .has_error_trace = false,
             .inferred_error_set = false,
             .disable_instrumentation = false,
+            .disable_intrinsics = false,
         },
         // This is populated after we create the Nav below. It is not read
         // by equality or hashing functions.
@@ -9480,6 +9484,7 @@ pub fn getFuncInstanceIes(
             .has_error_trace = false,
             .inferred_error_set = true,
             .disable_instrumentation = false,
+            .disable_intrinsics = false,
         },
         // This is populated after we create the Nav below. It is not read
         // by equality or hashing functions.
@@ -12313,6 +12318,18 @@ pub fn funcSetDisableInstrumentation(ip: *InternPool, func: Index) void {
     @atomicStore(FuncAnalysis, analysis_ptr, analysis, .release);
 }
 
+pub fn funcSetDisableIntrinsics(ip: *InternPool, func: Index) void {
+    const unwrapped_func = func.unwrap(ip);
+    const extra_mutex = &ip.getLocal(unwrapped_func.tid).mutate.extra.mutex;
+    extra_mutex.lock();
+    defer extra_mutex.unlock();
+
+    const analysis_ptr = ip.funcAnalysisPtr(func);
+    var analysis = analysis_ptr.*;
+    analysis.disable_intrinsics = true;
+    @atomicStore(FuncAnalysis, analysis_ptr, analysis, .release);
+}
+
 pub fn funcZirBodyInst(ip: *const InternPool, func: Index) TrackedInst.Index {
     const unwrapped_func = func.unwrap(ip);
     const item = unwrapped_func.getItem(ip);

src/Sema.zig

@@ -1409,6 +1409,11 @@ fn analyzeBodyInner(
                         i += 1;
                         continue;
                     },
+                    .disable_intrinsics => {
+                        try sema.zirDisableIntrinsics();
+                        i += 1;
+                        continue;
+                    },
                     .restore_err_ret_index => {
                         try sema.zirRestoreErrRetIndex(block, extended);
                         i += 1;
@@ -6642,6 +6647,23 @@ fn zirDisableInstrumentation(sema: *Sema) CompileError!void {
     sema.allow_memoize = false;
 }
 
+fn zirDisableIntrinsics(sema: *Sema) CompileError!void {
+    const pt = sema.pt;
+    const zcu = pt.zcu;
+    const ip = &zcu.intern_pool;
+    const func = switch (sema.owner.unwrap()) {
+        .func => |func| func,
+        .@"comptime",
+        .nav_val,
+        .nav_ty,
+        .type,
+        .memoized_state,
+        => return, // does nothing outside a function
+    };
+    ip.funcSetDisableIntrinsics(func);
+    sema.allow_memoize = false;
+}
+
 fn zirSetFloatMode(sema: *Sema, block: *Block, extended: Zir.Inst.Extended.InstData) CompileError!void {
     const extra = sema.code.extraData(Zir.Inst.UnNode, extended.operand).data;
     const src = block.builtinCallArgSrc(extra.node, 0);

src/codegen/c.zig

@@ -1859,8 +1859,17 @@ pub const DeclGen = struct {
                 else => unreachable,
             }
         }
-        if (fn_val.getFunction(zcu)) |func| if (func.analysisUnordered(ip).branch_hint == .cold)
-            try w.writeAll("zig_cold ");
+
+        if (fn_val.getFunction(zcu)) |func| {
+            const func_analysis = func.analysisUnordered(ip);
+
+            if (func_analysis.branch_hint == .cold)
+                try w.writeAll("zig_cold ");
+
+            if (kind == .complete and func_analysis.disable_intrinsics or dg.mod.no_builtin)
+                try w.writeAll("zig_no_builtin ");
+        }
+
         if (fn_info.return_type == .noreturn_type) try w.writeAll("zig_noreturn ");
 
         var trailing = try renderTypePrefix(dg.pass, &dg.ctype_pool, zcu, w, fn_ctype, .suffix, .{});

src/codegen/llvm.zig

@@ -1447,6 +1447,19 @@ pub const Object = struct {
             try attributes.addFnAttr(.nosanitize_coverage, &o.builder);
         }
 
+        const disable_intrinsics = func_analysis.disable_intrinsics or owner_mod.no_builtin;
+        if (disable_intrinsics) {
+            // The intent here is for compiler-rt and libc functions to not generate
+            // infinite recursion. For example, if we are compiling the memcpy function,
+            // and llvm detects that the body is equivalent to memcpy, it may replace the
+            // body of memcpy with a call to memcpy, which would then cause a stack
+            // overflow instead of performing memcpy.
+            try attributes.addFnAttr(.{ .string = .{
+                .kind = try o.builder.string("no-builtins"),
+                .value = .empty,
+            } }, &o.builder);
+        }
+
         // TODO: disable this if safety is off for the function scope
         const ssp_buf_size = owner_mod.stack_protector;
         if (ssp_buf_size != 0) {
@@ -1750,6 +1763,7 @@ pub const Object = struct {
             .prev_dbg_line = 0,
             .prev_dbg_column = 0,
             .err_ret_trace = err_ret_trace,
+            .disable_intrinsics = disable_intrinsics,
         };
         defer fg.deinit();
         deinit_wip = false;
@@ -3129,17 +3143,6 @@ pub const Object = struct {
                 &o.builder,
             );
         }
-        if (owner_mod.no_builtin) {
-            // The intent here is for compiler-rt and libc functions to not generate
-            // infinite recursion. For example, if we are compiling the memcpy function,
-            // and llvm detects that the body is equivalent to memcpy, it may replace the
-            // body of memcpy with a call to memcpy, which would then cause a stack
-            // overflow instead of performing memcpy.
-            try attributes.addFnAttr(.{ .string = .{
-                .kind = try o.builder.string("no-builtins"),
-                .value = .empty,
-            } }, &o.builder);
-        }
         if (owner_mod.optimize_mode == .ReleaseSmall) {
             try attributes.addFnAttr(.minsize, &o.builder);
             try attributes.addFnAttr(.optsize, &o.builder);
@@ -4918,6 +4921,8 @@ pub const FuncGen = struct {
 
     sync_scope: Builder.SyncScope,
 
+    disable_intrinsics: bool,
+
     const Fuzz = struct {
         counters_variable: Builder.Variable.Index,
         pcs: std.ArrayListUnmanaged(Builder.Constant),
@@ -5443,7 +5448,7 @@ pub const FuncGen = struct {
         var attributes: Builder.FunctionAttributes.Wip = .{};
         defer attributes.deinit(&o.builder);
 
-        if (self.ng.ownerModule().no_builtin) {
+        if (self.disable_intrinsics) {
             try attributes.addFnAttr(.nobuiltin, &o.builder);
         }
 
@@ -5770,7 +5775,7 @@ pub const FuncGen = struct {
                     try o.builder.intValue(.i8, 0xaa),
                     len,
                     if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
-                    self.ng.ownerModule().no_builtin,
+                    self.disable_intrinsics,
                 );
                 const owner_mod = self.ng.ownerModule();
                 if (owner_mod.valgrind) {
@@ -5821,7 +5826,7 @@ pub const FuncGen = struct {
                 try o.builder.intValue(.i8, 0xaa),
                 len,
                 .normal,
-                self.ng.ownerModule().no_builtin,
+                self.disable_intrinsics,
             );
             const owner_mod = self.ng.ownerModule();
             if (owner_mod.valgrind) {
@@ -9735,7 +9740,7 @@ pub const FuncGen = struct {
                 if (safety) try o.builder.intValue(.i8, 0xaa) else try o.builder.undefValue(.i8),
                 len,
                 if (ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal,
-                self.ng.ownerModule().no_builtin,
+                self.disable_intrinsics,
             );
             if (safety and owner_mod.valgrind) {
                 try self.valgrindMarkUndef(dest_ptr, len);
@@ -10057,7 +10062,7 @@ pub const FuncGen = struct {
                         fill_byte,
                         len,
                         access_kind,
-                        self.ng.ownerModule().no_builtin,
+                        self.disable_intrinsics,
                     );
                 }
                 const owner_mod = self.ng.ownerModule();
@@ -10089,7 +10094,7 @@ pub const FuncGen = struct {
                         fill_byte,
                         len,
                         access_kind,
-                        self.ng.ownerModule().no_builtin,
+                        self.disable_intrinsics,
                     );
                 }
                 return .none;
@@ -10119,7 +10124,7 @@ pub const FuncGen = struct {
                     fill_byte,
                     len,
                     access_kind,
-                    self.ng.ownerModule().no_builtin,
+                    self.disable_intrinsics,
                 );
             }
             return .none;
@@ -10172,7 +10177,7 @@ pub const FuncGen = struct {
                 elem_abi_align.toLlvm(),
                 try o.builder.intValue(llvm_usize_ty, elem_abi_size),
                 access_kind,
-                self.ng.ownerModule().no_builtin,
+                self.disable_intrinsics,
             );
         } else _ = try self.wip.store(access_kind, value, it_ptr.toValue(), it_ptr_align);
         const next_ptr = try self.wip.gep(.inbounds, elem_llvm_ty, it_ptr.toValue(), &.{
@@ -10206,7 +10211,7 @@ pub const FuncGen = struct {
             fill_byte,
             len,
             access_kind,
-            self.ng.ownerModule().no_builtin,
+            self.disable_intrinsics,
         );
         _ = try self.wip.br(end_block);
         self.wip.cursor = .{ .block = end_block };
@@ -10249,7 +10254,7 @@ pub const FuncGen = struct {
                 src_ptr_ty.ptrAlignment(zcu).toLlvm(),
                 len,
                 access_kind,
-                self.ng.ownerModule().no_builtin,
+                self.disable_intrinsics,
             );
             _ = try self.wip.br(end_block);
             self.wip.cursor = .{ .block = end_block };
@@ -10263,7 +10268,7 @@ pub const FuncGen = struct {
             src_ptr_ty.ptrAlignment(zcu).toLlvm(),
             len,
             access_kind,
-            self.ng.ownerModule().no_builtin,
+            self.disable_intrinsics,
         );
         return .none;
     }
@@ -11397,7 +11402,7 @@ pub const FuncGen = struct {
             ptr_alignment,
             try o.builder.intValue(try o.lowerType(Type.usize), size_bytes),
             access_kind,
-            fg.ng.ownerModule().no_builtin,
+            fg.disable_intrinsics,
         );
         return result_ptr;
     }
@@ -11565,7 +11570,7 @@ pub const FuncGen = struct {
             elem_ty.abiAlignment(zcu).toLlvm(),
             try o.builder.intValue(try o.lowerType(Type.usize), elem_ty.abiSize(zcu)),
             access_kind,
-            self.ng.ownerModule().no_builtin,
+            self.disable_intrinsics,
         );
     }
 

src/print_zir.zig

@@ -531,6 +531,7 @@ const Writer = struct {
             .frame_address,
             .breakpoint,
             .disable_instrumentation,
+            .disable_intrinsics,
             .c_va_start,
             .in_comptime,
             .value_placeholder,