From e9313098df85360f222c9f39117d1c5a9eb41037 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 23 Mar 2025 14:16:27 +0000 Subject: [PATCH 1/5] [VPlan] Add exit phi operands during initial construction (NFC). Add incoming exit phi operands during the initial VPlan construction. This ensures all users are added to the initial VPlan and is also needed in preparation to retaining exiting edges during initial construction. --- .../Transforms/Vectorize/LoopVectorize.cpp | 6 +--- .../Vectorize/VPlanConstruction.cpp | 19 +++++++++++++ .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +- .../Transforms/Vectorize/VPlanTransforms.cpp | 28 +++++++++++-------- 4 files changed, 38 insertions(+), 17 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 7a5f618d09e95..f9a7769b76b94 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -9392,11 +9392,7 @@ collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder, continue; } - PHINode &ExitPhi = ExitIRI->getIRPhi(); - BasicBlock *ExitingBB = OrigLoop->getLoopLatch(); - Value *IncomingValue = ExitPhi.getIncomingValueForBlock(ExitingBB); - VPValue *V = Builder.getVPValueOrAddLiveIn(IncomingValue); - ExitIRI->addOperand(V); + VPValue *V = ExitIRI->getOperand(0); if (V->isLiveIn()) continue; assert(V->getDefiningRecipe()->getParent()->getEnclosingLoopRegion() && diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index 9fcccfcf8117f..95f0a91113fb9 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -352,6 +352,19 @@ std::unique_ptr PlainCFGBuilder::buildPlainCFG( Plan->getEntry()->setOneSuccessor(getOrCreateVPBB(TheLoop->getHeader())); Plan->getEntry()->setPlan(&*Plan); + // Add incoming operands for the VPIRInstructions wrapping the exit phis. + for (auto *EB : Plan->getExitBlocks()) { + for (VPRecipeBase &R : *EB) { + auto *PhiR = dyn_cast(&R); + if (!PhiR) + break; + PHINode &Phi = PhiR->getIRPhi(); + for (BasicBlock *Pred : predecessors(EB->getIRBasicBlock())) + PhiR->addOperand( + getOrCreateVPOperand(Phi.getIncomingValueForBlock(Pred))); + } + } + for (const auto &[IRBB, VPB] : BB2VPBB) VPB2IRBB[VPB] = IRBB; @@ -464,6 +477,12 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan, Type *InductionTy, VPBlockUtils::connectBlocks(ScalarPH, Plan.getScalarHeader()); if (!RequiresScalarEpilogueCheck) { VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH); + // The exit blocks are dead, remove any recipes to make sure no users remain + // that may pessimize transforms. + for (auto *EB : Plan.getExitBlocks()) { + for (VPRecipeBase &R : make_early_inc_range(*EB)) + R.eraseFromParent(); + } return; } diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 2cc558f49ccce..9e68ab55783ce 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1139,7 +1139,7 @@ InstructionCost VPIRInstruction::computeCost(ElementCount VF, void VPIRInstruction::extractLastLaneOfOperand(VPBuilder &Builder) { assert(isa(getInstruction()) && "can only add exiting operands to phi nodes"); - assert(getNumOperands() == 1 && "must have a single operand"); + assert(getNumOperands() > 0 && "must have at least one operand"); VPValue *Exiting = getOperand(0); if (!Exiting->isLiveIn()) { LLVMContext &Ctx = getInstruction().getContext(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index b80fe18d1bd66..9f5d76c2c856d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2501,20 +2501,24 @@ void VPlanTransforms::handleUncountableEarlyExit( if (!ExitIRI) break; - PHINode &ExitPhi = ExitIRI->getIRPhi(); - VPValue *IncomingFromEarlyExit = RecipeBuilder.getVPValueOrAddLiveIn( - ExitPhi.getIncomingValueForBlock(UncountableExitingBlock)); - + unsigned EarlyExitIdx = 0; if (OrigLoop->getUniqueExitBlock()) { + // After the transform, the first incoming value is coming from the + // orignial loop latch, while the second operand is from the early exit. + // Sawp the phi operands, if the first predecessor in the original IR is + // not the loop latch. + if (*pred_begin(VPEarlyExitBlock->getIRBasicBlock()) != + OrigLoop->getLoopLatch()) + ExitIRI->swapOperands(); + + EarlyExitIdx = 1; // If there's a unique exit block, VPEarlyExitBlock has 2 predecessors // (MiddleVPBB and NewMiddle). Add the incoming value from MiddleVPBB // which is coming from the original latch. - VPValue *IncomingFromLatch = RecipeBuilder.getVPValueOrAddLiveIn( - ExitPhi.getIncomingValueForBlock(OrigLoop->getLoopLatch())); - ExitIRI->addOperand(IncomingFromLatch); ExitIRI->extractLastLaneOfOperand(MiddleBuilder); } + VPValue *IncomingFromEarlyExit = ExitIRI->getOperand(EarlyExitIdx); auto IsVector = [](ElementCount VF) { return VF.isVector(); }; // When the VFs are vectors, need to add `extract` to get the incoming value // from early exit. When the range contains scalar VF, limit the range to @@ -2522,14 +2526,16 @@ void VPlanTransforms::handleUncountableEarlyExit( // and vector VFs. if (!IncomingFromEarlyExit->isLiveIn() && LoopVectorizationPlanner::getDecisionAndClampRange(IsVector, Range)) { + // Add the incoming value from the early exit. VPValue *FirstActiveLane = EarlyExitB.createNaryOp( VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr, "first.active.lane"); - IncomingFromEarlyExit = EarlyExitB.createNaryOp( - Instruction::ExtractElement, {IncomingFromEarlyExit, FirstActiveLane}, - nullptr, "early.exit.value"); + ExitIRI->setOperand( + EarlyExitIdx, + EarlyExitB.createNaryOp(Instruction::ExtractElement, + {IncomingFromEarlyExit, FirstActiveLane}, + nullptr, "early.exit.value")); } - ExitIRI->addOperand(IncomingFromEarlyExit); } MiddleBuilder.createNaryOp(VPInstruction::BranchOnCond, {IsEarlyExitTaken}); From 70523376ae33c458adf311d2f4fee6a163f42e78 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 21 Apr 2025 20:45:59 +0100 Subject: [PATCH 2/5] !fixup address comments, thanks --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index aa8ec1783a8ba..afb18881c7b02 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -9370,8 +9370,7 @@ static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan, } } -// Collect VPIRInstructions for phis in the exit blocks that are modeled -// in VPlan and add the exiting VPValue as operand. +// Collect VPIRInstructions for phis in the exit block from the latch only. static SetVector collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder, VPlan &Plan) { @@ -9392,6 +9391,7 @@ collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder, continue; } + assert(ExitIRI->getNumOperands() == 1 && "must have a single operand"); VPValue *V = ExitIRI->getOperand(0); if (V->isLiveIn()) continue; From d0081f77b585b3b176f67e72d80d06a92b56d0e4 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 21 Apr 2025 21:29:22 +0100 Subject: [PATCH 3/5] !fixup address latest comments, thanks --- .../Transforms/Vectorize/LoopVectorize.cpp | 2 +- llvm/lib/Transforms/Vectorize/VPlan.h | 8 ++--- .../Vectorize/VPlanConstruction.cpp | 15 +++++----- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 4 +-- .../Transforms/Vectorize/VPlanTransforms.cpp | 30 ++++++++++--------- 5 files changed, 31 insertions(+), 28 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 48db0294fcfe7..6527aec6dc5db 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -9417,7 +9417,7 @@ addUsersInExitBlocks(VPlan &Plan, ExitIRI->getParent()->getSinglePredecessor() == MiddleVPBB && "exit values from early exits must be fixed when branch to " "early-exit is added"); - ExitIRI->extractLastLaneOfOperand(B); + ExitIRI->extractLastLaneOfFirstOperand(B); } } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index fdcee2bd28997..15f22e0e6cddc 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1161,10 +1161,10 @@ class VPIRInstruction : public VPRecipeBase { return true; } - /// Update the recipes single operand to the last lane of the operand using \p - /// Builder. Must only be used for single operand VPIRInstructions wrapping a - /// PHINode. - void extractLastLaneOfOperand(VPBuilder &Builder); + /// Update the recipes first operand to the last lane of the operand using \p + /// Builder. Must only be used for VPIRInstructions with at least one operand + /// wrapping a PHINode. + void extractLastLaneOfFirstOperand(VPBuilder &Builder); }; /// An overlay for VPIRInstructions wrapping PHI nodes enabling convenient use diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index 95f0a91113fb9..6b6108c30e272 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -352,13 +352,14 @@ std::unique_ptr PlainCFGBuilder::buildPlainCFG( Plan->getEntry()->setOneSuccessor(getOrCreateVPBB(TheLoop->getHeader())); Plan->getEntry()->setPlan(&*Plan); - // Add incoming operands for the VPIRInstructions wrapping the exit phis. + // Fix VPlan loop-closed-ssa exit phi's by add incoming operands to the + // VPIRInstructions wrapping them. for (auto *EB : Plan->getExitBlocks()) { - for (VPRecipeBase &R : *EB) { - auto *PhiR = dyn_cast(&R); - if (!PhiR) - break; + for (VPRecipeBase &R : EB->phis()) { + auto *PhiR = cast(&R); PHINode &Phi = PhiR->getIRPhi(); + assert(PhiR->getNumOperands() == 0 && + "no phi operands should be added yet"); for (BasicBlock *Pred : predecessors(EB->getIRBasicBlock())) PhiR->addOperand( getOrCreateVPOperand(Phi.getIncomingValueForBlock(Pred))); @@ -477,8 +478,8 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan, Type *InductionTy, VPBlockUtils::connectBlocks(ScalarPH, Plan.getScalarHeader()); if (!RequiresScalarEpilogueCheck) { VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH); - // The exit blocks are dead, remove any recipes to make sure no users remain - // that may pessimize transforms. + // The exit blocks are unreachable, remove their recipes to make sure no + // users remain that may pessimize transforms. for (auto *EB : Plan.getExitBlocks()) { for (VPRecipeBase &R : make_early_inc_range(*EB)) R.eraseFromParent(); diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index b1fc38d338d74..f6481aaf55ecc 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1137,9 +1137,9 @@ InstructionCost VPIRInstruction::computeCost(ElementCount VF, return 0; } -void VPIRInstruction::extractLastLaneOfOperand(VPBuilder &Builder) { +void VPIRInstruction::extractLastLaneOfFirstOperand(VPBuilder &Builder) { assert(isa(getInstruction()) && - "can only add exiting operands to phi nodes"); + "can only update exiting operands to phi nodes"); assert(getNumOperands() > 0 && "must have at least one operand"); VPValue *Exiting = getOperand(0); if (!Exiting->isLiveIn()) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index e539c68799228..6d01ea7cce708 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2503,23 +2503,26 @@ void VPlanTransforms::handleUncountableEarlyExit( // Update the exit phis in the early exit block. VPBuilder MiddleBuilder(NewMiddle); VPBuilder EarlyExitB(VectorEarlyExitVPBB); - for (VPRecipeBase &R : *VPEarlyExitBlock->phis()) { + for (VPRecipeBase &R : VPEarlyExitBlock->phis()) { auto *ExitIRI = cast(&R); + // By default, assume early exit operand is first, e.g., when the two exit + // blocks are distinct - VPEarlyExitBlock has a single predecessor. unsigned EarlyExitIdx = 0; if (OrigLoop->getUniqueExitBlock()) { - // After the transform, the first incoming value is coming from the - // orignial loop latch, while the second operand is from the early exit. - // Sawp the phi operands, if the first predecessor in the original IR is - // not the loop latch. + // The incoming values currently correspond to the original IR + // predecessors. After the transform, the first incoming value is coming + // from the original loop latch, while the second operand is from the + // early exit. Swap the phi operands, if the first predecessor in the + // original IR is not the loop latch. if (*pred_begin(VPEarlyExitBlock->getIRBasicBlock()) != OrigLoop->getLoopLatch()) ExitIRI->swapOperands(); EarlyExitIdx = 1; // If there's a unique exit block, VPEarlyExitBlock has 2 predecessors - // (MiddleVPBB and NewMiddle). Add the incoming value from MiddleVPBB - // which is coming from the original latch. - ExitIRI->extractLastLaneOfOperand(MiddleBuilder); + // (MiddleVPBB and NewMiddle). Extract the last lane of the incoming value + // from MiddleVPBB which is coming from the original latch. + ExitIRI->extractLastLaneOfFirstOperand(MiddleBuilder); } VPValue *IncomingFromEarlyExit = ExitIRI->getOperand(EarlyExitIdx); @@ -2530,15 +2533,14 @@ void VPlanTransforms::handleUncountableEarlyExit( // and vector VFs. if (!IncomingFromEarlyExit->isLiveIn() && LoopVectorizationPlanner::getDecisionAndClampRange(IsVector, Range)) { - // Add the incoming value from the early exit. + // Update the incoming value from the early exit. VPValue *FirstActiveLane = EarlyExitB.createNaryOp( VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr, "first.active.lane"); - ExitIRI->setOperand( - EarlyExitIdx, - EarlyExitB.createNaryOp(Instruction::ExtractElement, - {IncomingFromEarlyExit, FirstActiveLane}, - nullptr, "early.exit.value")); + IncomingFromEarlyExit = EarlyExitB.createNaryOp( + Instruction::ExtractElement, {IncomingFromEarlyExit, FirstActiveLane}, + nullptr, "early.exit.value"); + ExitIRI->setOperand(EarlyExitIdx, IncomingFromEarlyExit); } } MiddleBuilder.createNaryOp(VPInstruction::BranchOnCond, {IsEarlyExitTaken}); From a900c6333043d278cb361933c344f20f34f6b458 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 22 Apr 2025 22:37:41 +0100 Subject: [PATCH 4/5] !fixup address latest comments, thanks --- .../Transforms/Vectorize/LoopVectorize.cpp | 6 ++-- .../Vectorize/VPlanConstruction.cpp | 23 ++++++++------ .../Transforms/Vectorize/VPlanTransforms.cpp | 30 ++++++++++--------- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 6527aec6dc5db..c7fcc3a5831f8 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -9370,8 +9370,8 @@ static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan, // Collect VPIRInstructions for phis in the exit block from the latch only. static SetVector -collectUsersInExitBlocks(Loop *OrigLoop, VPRecipeBuilder &Builder, - VPlan &Plan) { +collectUsersInLatchExitBlock(Loop *OrigLoop, VPRecipeBuilder &Builder, + VPlan &Plan) { SetVector ExitUsersToFix; for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) { // Nothing to do for unreachable exit blocks. @@ -9758,7 +9758,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { DenseMap IVEndValues; addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues); SetVector ExitUsersToFix = - collectUsersInExitBlocks(OrigLoop, RecipeBuilder, *Plan); + collectUsersInLatchExitBlock(OrigLoop, RecipeBuilder, *Plan); addExitUsersForFirstOrderRecurrences(*Plan, ExitUsersToFix); addUsersInExitBlocks(*Plan, ExitUsersToFix); diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index 6b6108c30e272..e59d2d44862bb 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -352,8 +352,10 @@ std::unique_ptr PlainCFGBuilder::buildPlainCFG( Plan->getEntry()->setOneSuccessor(getOrCreateVPBB(TheLoop->getHeader())); Plan->getEntry()->setPlan(&*Plan); - // Fix VPlan loop-closed-ssa exit phi's by add incoming operands to the + // Fix VPlan loop-closed-ssa exit phi's by adding incoming operands to the // VPIRInstructions wrapping them. + // Note that the operand order may need adjusting when predecessors are added, + // if an exit block has multiple predecessor. for (auto *EB : Plan->getExitBlocks()) { for (VPRecipeBase &R : EB->phis()) { auto *PhiR = cast(&R); @@ -476,6 +478,17 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan, Type *InductionTy, VPBasicBlock *ScalarPH = Plan.createVPBasicBlock("scalar.ph"); VPBlockUtils::connectBlocks(ScalarPH, Plan.getScalarHeader()); + + // If needed, add a check in the middle block to see if we have completed + // all of the iterations in the first vector loop. Three cases: + // 1) If we require a scalar epilogue, there is no conditional branch as + // we unconditionally branch to the scalar preheader. Remove the recipes + // from the exit blocks. + // 2) If (N - N%VF) == N, then we *don't* need to run the remainder. + // Thus if tail is to be folded, we know we don't need to run the + // remainder and we can set the condition to true. + // 3) Otherwise, construct a runtime check. + if (!RequiresScalarEpilogueCheck) { VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH); // The exit blocks are unreachable, remove their recipes to make sure no @@ -487,14 +500,6 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan, Type *InductionTy, return; } - // If needed, add a check in the middle block to see if we have completed - // all of the iterations in the first vector loop. Three cases: - // 1) If (N - N%VF) == N, then we *don't* need to run the remainder. - // Thus if tail is to be folded, we know we don't need to run the - // remainder and we can set the condition to true. - // 2) If we require a scalar epilogue, there is no conditional branch as - // we unconditionally branch to the scalar preheader. Do nothing. - // 3) Otherwise, construct a runtime check. BasicBlock *IRExitBlock = TheLoop->getUniqueLatchExitBlock(); auto *VPExitBlock = Plan.getExitBlock(IRExitBlock); // The connection order corresponds to the operands of the conditional branch. diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 6d01ea7cce708..ee33a58404818 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2505,23 +2505,25 @@ void VPlanTransforms::handleUncountableEarlyExit( VPBuilder EarlyExitB(VectorEarlyExitVPBB); for (VPRecipeBase &R : VPEarlyExitBlock->phis()) { auto *ExitIRI = cast(&R); - // By default, assume early exit operand is first, e.g., when the two exit - // blocks are distinct - VPEarlyExitBlock has a single predecessor. - unsigned EarlyExitIdx = 0; + // Early exit operand should always be last, i.e., 0 if VPEarlyExitBlock has + // a single predecessor and 1 if it has two. + unsigned EarlyExitIdx = ExitIRI->getNumOperands() - 1; if (OrigLoop->getUniqueExitBlock()) { - // The incoming values currently correspond to the original IR - // predecessors. After the transform, the first incoming value is coming - // from the original loop latch, while the second operand is from the - // early exit. Swap the phi operands, if the first predecessor in the - // original IR is not the loop latch. - if (*pred_begin(VPEarlyExitBlock->getIRBasicBlock()) != - OrigLoop->getLoopLatch()) + // If VPEarlyExitBlock has two predecessors, they are already ordered such + // that early exit is second (and latch exit is first), by construction. + // But its underlying IRBB (EarlyExitIRBB) may have its predecessors + // ordered the other way around, and it is the order of the latter which + // corresponds to the order of operands of VPEarlyExitBlock's phi recipes. + // Therefore, if early exit (UncountableExitingBlock) is the first + // predecessor of EarlyExitIRBB, we swap the operands of phi recipes, + // thereby bringing them to match VPEarlyExitBlock's predecessor order, + // with early exit being last (second). Otherwise they already match. + if (*pred_begin(VPEarlyExitBlock->getIRBasicBlock()) == + UncountableExitingBlock) ExitIRI->swapOperands(); - EarlyExitIdx = 1; - // If there's a unique exit block, VPEarlyExitBlock has 2 predecessors - // (MiddleVPBB and NewMiddle). Extract the last lane of the incoming value - // from MiddleVPBB which is coming from the original latch. + // The first of two operands corresponds to the latch exit, via MiddleVPBB + // predecessor. Extract its last lane. ExitIRI->extractLastLaneOfFirstOperand(MiddleBuilder); } From 412b278868100fbf8510f294182459f6acb5216a Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 23 Apr 2025 19:23:56 +0100 Subject: [PATCH 5/5] !fixup address latest comments, thanks! --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 6 ++---- llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 95c00fa3d0152..54f6d1e42f504 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -9374,9 +9374,7 @@ static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan, } // Collect VPIRInstructions for phis in the exit block from the latch only. -static SetVector -collectUsersInLatchExitBlock(Loop *OrigLoop, VPRecipeBuilder &Builder, - VPlan &Plan) { +static SetVector collectUsersInLatchExitBlock(VPlan &Plan) { SetVector ExitUsersToFix; for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) { // Nothing to do for unreachable exit blocks. @@ -9763,7 +9761,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { DenseMap IVEndValues; addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues); SetVector ExitUsersToFix = - collectUsersInLatchExitBlock(OrigLoop, RecipeBuilder, *Plan); + collectUsersInLatchExitBlock(*Plan); addExitUsersForFirstOrderRecurrences(*Plan, ExitUsersToFix); addUsersInExitBlocks(*Plan, ExitUsersToFix); diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index e59d2d44862bb..b187fae4abcc6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -354,8 +354,9 @@ std::unique_ptr PlainCFGBuilder::buildPlainCFG( // Fix VPlan loop-closed-ssa exit phi's by adding incoming operands to the // VPIRInstructions wrapping them. - // Note that the operand order may need adjusting when predecessors are added, - // if an exit block has multiple predecessor. + // // Note that the operand order corresponds to IR predecessor order, and may + // need adjusting when VPlan predecessors are added, if an exit block has + // multiple predecessor. for (auto *EB : Plan->getExitBlocks()) { for (VPRecipeBase &R : EB->phis()) { auto *PhiR = cast(&R);