Merge pull request #18409 from JuliaLang/tb/always_inline

llvmcall: use AlwaysInliner + test inlining behavior

Author: maleadt

src/ccall.cpp

@@ -1000,6 +1000,7 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c
         assert(isPtr);
         // Create Function skeleton
         f = (llvm::Function*)jl_unbox_voidpointer(ir);
+        assert(!f->isDeclaration());
         assert(f->getReturnType() == rettype);
         int i = 0;
         for (std::vector<Type *>::iterator it = argtypes.begin();
@@ -1025,15 +1026,6 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c
         }
     }
 
-    /*
-     * It might be tempting to just try to set the Always inline attribute on the function
-     * and hope for the best. However, this doesn't work since that would require an inlining
-     * pass (which is a Call Graph pass and cannot be managed by a FunctionPassManager). Instead
-     * We are sneaky and call the inliner directly. This however doesn't work until we've actually
-     * generated the entire function, so we need to store it in the context until the end of the
-     * function. This also has the benefit of looking exactly like we cut/pasted it in in `code_llvm`.
-     */
-
     // Since we dumped all of f's dependencies into the active module,
     // we cannot reasonably inline it, so leave it there and just emit
     // a regular call
@@ -1064,7 +1056,7 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c
     mark_gc_uses(gc_uses);
     CallInst *inst = builder.CreateCall(f, ArrayRef<Value*>(&argvals[0], nargt));
     if (isString)
-        ctx->to_inline.push_back(inst);
+        f->addFnAttr(Attribute::AlwaysInline);
     mark_gc_uses(gc_uses);
 
     JL_GC_POP();

src/codegen.cpp

@@ -548,7 +548,6 @@ typedef struct {
 
     bool debug_enabled;
     bool is_inbounds{false};
-    std::vector<CallInst*> to_inline;
 } jl_codectx_t;
 
 static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx);
@@ -5025,22 +5024,7 @@ static std::unique_ptr<Module> emit_function(jl_method_instance_t *lam, jl_code_
     builder.SetCurrentDebugLocation(noDbg);
     builder.ClearInsertionPoint();
 
-    // step 13, Apply LLVM level inlining
-    for(std::vector<CallInst*>::iterator it = ctx.to_inline.begin(); it != ctx.to_inline.end(); ++it) {
-        Function *inlinef = (*it)->getCalledFunction();
-        assert(inlinef->getParent());
-        InlineFunctionInfo info;
-        if (!InlineFunction(*it,info))
-            jl_error("Inlining Pass failed");
-        if (inlinef->getParent())
-            inlinef->eraseFromParent();
-        else {
-            inlinef->dropAllReferences();
-            delete inlinef;
-        }
-    }
-
-    // step 14. Perform any delayed instantiations
+    // step 13. Perform any delayed instantiations
     if (ctx.debug_enabled) {
         dbuilder.finalize();
     }

src/jitlayers.cpp

@@ -27,6 +27,7 @@
 #include <polly/CodeGen/CodegenCleanup.h>
 #endif
 
+#include <llvm/Transforms/IPO.h>
 #include <llvm/Transforms/Scalar.h>
 #include <llvm/Transforms/Utils/BasicBlockUtils.h>
 #include <llvm/Transforms/Instrumentation.h>
@@ -141,6 +142,7 @@ void addOptimizationPasses(PassManager *PM)
     // list of passes from vmkit
     PM->add(createCFGSimplificationPass()); // Clean up disgusting code
     PM->add(createPromoteMemoryToRegisterPass());// Kill useless allocas
+    PM->add(createAlwaysInlinerPass()); // Respect always_inline
 
 #ifndef INSTCOMBINE_BUG
     PM->add(createInstructionCombiningPass()); // Cleanup for scalarrepl.

test/llvmcall.jl

@@ -85,6 +85,8 @@ function declared_floor(x::Float64)
     Float64, Tuple{Float64}, x)
 end
 @test declared_floor(4.2) ≈ 4.
+ir = sprint(io->code_llvm(io, declared_floor, Tuple{Float64}))
+@test contains(ir, "call double @llvm.floor.f64") # should be inlined
 
 function doubly_declared_floor(x::Float64)
     llvmcall(