diff options
Diffstat (limited to 'lib/Transforms')
23 files changed, 622 insertions, 262 deletions
diff --git a/lib/Transforms/IPO/GlobalOpt.cpp b/lib/Transforms/IPO/GlobalOpt.cpp index cdf7b76..4ac721d 100644 --- a/lib/Transforms/IPO/GlobalOpt.cpp +++ b/lib/Transforms/IPO/GlobalOpt.cpp @@ -1999,9 +1999,13 @@ static std::vector<Function*> ParseGlobalCtors(GlobalVariable *GV) { static GlobalVariable *InstallGlobalCtors(GlobalVariable *GCL, const std::vector<Function*> &Ctors) { // If we made a change, reassemble the initializer list. - std::vector<Constant*> CSVals; - CSVals.push_back(ConstantInt::get(Type::getInt32Ty(GCL->getContext()),65535)); - CSVals.push_back(0); + Constant *CSVals[2]; + CSVals[0] = ConstantInt::get(Type::getInt32Ty(GCL->getContext()), 65535); + CSVals[1] = 0; + + const StructType *StructTy = + cast <StructType>( + cast<ArrayType>(GCL->getType()->getElementType())->getElementType()); // Create the new init list. std::vector<Constant*> CAList; @@ -2016,12 +2020,10 @@ static GlobalVariable *InstallGlobalCtors(GlobalVariable *GCL, CSVals[0] = ConstantInt::get(Type::getInt32Ty(GCL->getContext()), 0x7fffffff); } - CAList.push_back(ConstantStruct::get(GCL->getContext(), CSVals, false)); + CAList.push_back(ConstantStruct::get(StructTy, CSVals)); } // Create the array initializer. - const Type *StructTy = - cast<ArrayType>(GCL->getType()->getElementType())->getElementType(); Constant *CA = ConstantArray::get(ArrayType::get(StructTy, CAList.size()), CAList); @@ -2218,42 +2220,40 @@ static Constant *EvaluateStoreInto(Constant *Init, Constant *Val, Elts[Idx] = EvaluateStoreInto(Elts[Idx], Val, Addr, OpNo+1); // Return the modified struct. - return ConstantStruct::get(Init->getContext(), &Elts[0], Elts.size(), - STy->isPacked()); - } else { - ConstantInt *CI = cast<ConstantInt>(Addr->getOperand(OpNo)); - const SequentialType *InitTy = cast<SequentialType>(Init->getType()); - - uint64_t NumElts; - if (const ArrayType *ATy = dyn_cast<ArrayType>(InitTy)) - NumElts = ATy->getNumElements(); - else - NumElts = cast<VectorType>(InitTy)->getNumElements(); - + return ConstantStruct::get(STy, Elts); + } + + ConstantInt *CI = cast<ConstantInt>(Addr->getOperand(OpNo)); + const SequentialType *InitTy = cast<SequentialType>(Init->getType()); - // Break up the array into elements. - if (ConstantArray *CA = dyn_cast<ConstantArray>(Init)) { - for (User::op_iterator i = CA->op_begin(), e = CA->op_end(); i != e; ++i) - Elts.push_back(cast<Constant>(*i)); - } else if (ConstantVector *CV = dyn_cast<ConstantVector>(Init)) { - for (User::op_iterator i = CV->op_begin(), e = CV->op_end(); i != e; ++i) - Elts.push_back(cast<Constant>(*i)); - } else if (isa<ConstantAggregateZero>(Init)) { - Elts.assign(NumElts, Constant::getNullValue(InitTy->getElementType())); - } else { - assert(isa<UndefValue>(Init) && "This code is out of sync with " - " ConstantFoldLoadThroughGEPConstantExpr"); - Elts.assign(NumElts, UndefValue::get(InitTy->getElementType())); - } + uint64_t NumElts; + if (const ArrayType *ATy = dyn_cast<ArrayType>(InitTy)) + NumElts = ATy->getNumElements(); + else + NumElts = cast<VectorType>(InitTy)->getNumElements(); + + // Break up the array into elements. + if (ConstantArray *CA = dyn_cast<ConstantArray>(Init)) { + for (User::op_iterator i = CA->op_begin(), e = CA->op_end(); i != e; ++i) + Elts.push_back(cast<Constant>(*i)); + } else if (ConstantVector *CV = dyn_cast<ConstantVector>(Init)) { + for (User::op_iterator i = CV->op_begin(), e = CV->op_end(); i != e; ++i) + Elts.push_back(cast<Constant>(*i)); + } else if (isa<ConstantAggregateZero>(Init)) { + Elts.assign(NumElts, Constant::getNullValue(InitTy->getElementType())); + } else { + assert(isa<UndefValue>(Init) && "This code is out of sync with " + " ConstantFoldLoadThroughGEPConstantExpr"); + Elts.assign(NumElts, UndefValue::get(InitTy->getElementType())); + } - assert(CI->getZExtValue() < NumElts); - Elts[CI->getZExtValue()] = - EvaluateStoreInto(Elts[CI->getZExtValue()], Val, Addr, OpNo+1); + assert(CI->getZExtValue() < NumElts); + Elts[CI->getZExtValue()] = + EvaluateStoreInto(Elts[CI->getZExtValue()], Val, Addr, OpNo+1); - if (Init->getType()->isArrayTy()) - return ConstantArray::get(cast<ArrayType>(InitTy), Elts); - return ConstantVector::get(Elts); - } + if (Init->getType()->isArrayTy()) + return ConstantArray::get(cast<ArrayType>(InitTy), Elts); + return ConstantVector::get(Elts); } /// CommitValueTo - We have decided that Addr (which satisfies the predicate diff --git a/lib/Transforms/InstCombine/InstCombineCalls.cpp b/lib/Transforms/InstCombine/InstCombineCalls.cpp index ef67701..455cd0a 100644 --- a/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -412,7 +412,8 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { UndefValue::get(LHS->getType()), ConstantInt::getTrue(II->getContext()) }; - Constant *Struct = ConstantStruct::get(II->getContext(), V, 2, false); + const StructType *ST = cast<StructType>(II->getType()); + Constant *Struct = ConstantStruct::get(ST, V); return InsertValueInst::Create(Struct, Add, 0); } @@ -425,7 +426,8 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { UndefValue::get(LHS->getType()), ConstantInt::getFalse(II->getContext()) }; - Constant *Struct = ConstantStruct::get(II->getContext(), V, 2, false); + const StructType *ST = cast<StructType>(II->getType()); + Constant *Struct = ConstantStruct::get(ST, V); return InsertValueInst::Create(Struct, Add, 0); } } @@ -452,7 +454,8 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { UndefValue::get(II->getArgOperand(0)->getType()), ConstantInt::getFalse(II->getContext()) }; - Constant *Struct = ConstantStruct::get(II->getContext(), V, 2, false); + Constant *Struct = + ConstantStruct::get(cast<StructType>(II->getType()), V); return InsertValueInst::Create(Struct, II->getArgOperand(0), 0); } } @@ -472,7 +475,8 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { UndefValue::get(II->getArgOperand(0)->getType()), ConstantInt::getFalse(II->getContext()) }; - Constant *Struct = ConstantStruct::get(II->getContext(), V, 2, false); + Constant *Struct = + ConstantStruct::get(cast<StructType>(II->getType()), V); return InsertValueInst::Create(Struct, II->getArgOperand(0), 0); } } @@ -503,7 +507,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { UndefValue::get(LHS->getType()), Builder->getFalse() }; - Constant *Struct = ConstantStruct::get(II->getContext(), V, 2, false); + Constant *Struct = ConstantStruct::get(cast<StructType>(II->getType()),V); return InsertValueInst::Create(Struct, Mul, 0); } } // FALL THROUGH @@ -532,7 +536,8 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) { UndefValue::get(II->getArgOperand(0)->getType()), ConstantInt::getFalse(II->getContext()) }; - Constant *Struct = ConstantStruct::get(II->getContext(), V, 2, false); + Constant *Struct = + ConstantStruct::get(cast<StructType>(II->getType()), V); return InsertValueInst::Create(Struct, II->getArgOperand(0), 0); } } diff --git a/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/lib/Transforms/Instrumentation/GCOVProfiling.cpp index b902213..07d69e8 100644 --- a/lib/Transforms/Instrumentation/GCOVProfiling.cpp +++ b/lib/Transforms/Instrumentation/GCOVProfiling.cpp @@ -561,11 +561,11 @@ GlobalVariable *GCOVProfiler::buildEdgeLookupTable( Edge += Successors; } + ArrayRef<Constant*> V(&EdgeTable[0], Succs.size() * Preds.size()); GlobalVariable *EdgeTableGV = new GlobalVariable( *M, EdgeTableTy, true, GlobalValue::InternalLinkage, - ConstantArray::get(EdgeTableTy, - &EdgeTable[0], Succs.size() * Preds.size()), + ConstantArray::get(EdgeTableTy, V), "__llvm_gcda_edge_table"); EdgeTableGV->setUnnamedAddr(true); return EdgeTableGV; diff --git a/lib/Transforms/Instrumentation/PathProfiling.cpp b/lib/Transforms/Instrumentation/PathProfiling.cpp index 182a43d..1e5e3f6 100644 --- a/lib/Transforms/Instrumentation/PathProfiling.cpp +++ b/lib/Transforms/Instrumentation/PathProfiling.cpp @@ -376,7 +376,7 @@ namespace llvm { public: static const StructType *get(LLVMContext& C) { return( StructType::get( - C, TypeBuilder<types::i<32>, xcompile>::get(C), // type + TypeBuilder<types::i<32>, xcompile>::get(C), // type TypeBuilder<types::i<32>, xcompile>::get(C), // array size TypeBuilder<types::i<8>*, xcompile>::get(C), // array/hash ptr NULL)); diff --git a/lib/Transforms/Instrumentation/ProfilingUtils.cpp b/lib/Transforms/Instrumentation/ProfilingUtils.cpp index 7435bc3..4224ee3 100644 --- a/lib/Transforms/Instrumentation/ProfilingUtils.cpp +++ b/lib/Transforms/Instrumentation/ProfilingUtils.cpp @@ -164,7 +164,8 @@ void llvm::InsertProfilingShutdownCall(Function *Callee, Module *Mod) { GlobalVariable *GlobalDtors = new GlobalVariable( *Mod, ArrayType::get(GlobalDtorElemTy, 1), false, GlobalValue::AppendingLinkage, NULL, "llvm.global_dtors"); - dtors.push_back(ConstantStruct::get(Mod->getContext(), Elem, 2, false)); + + dtors.push_back(ConstantStruct::get(GlobalDtorElemTy, Elem)); GlobalDtors->setInitializer(ConstantArray::get( cast<ArrayType>(GlobalDtors->getType()->getElementType()), dtors)); } diff --git a/lib/Transforms/Scalar/GVN.cpp b/lib/Transforms/Scalar/GVN.cpp index 0e1e6f3..759e681 100644 --- a/lib/Transforms/Scalar/GVN.cpp +++ b/lib/Transforms/Scalar/GVN.cpp @@ -1190,8 +1190,10 @@ static Value *ConstructSSAForLoadSet(LoadInst *LI, // escaping uses to any values that are operands to these PHIs. for (unsigned i = 0, e = NewPHIs.size(); i != e; ++i) { PHINode *P = NewPHIs[i]; - for (unsigned ii = 0, ee = P->getNumIncomingValues(); ii != ee; ++ii) - AA->addEscapingUse(P->getOperandUse(2*ii)); + for (unsigned ii = 0, ee = P->getNumIncomingValues(); ii != ee; ++ii) { + unsigned jj = PHINode::getOperandNumForIncomingValue(ii); + AA->addEscapingUse(P->getOperandUse(jj)); + } } } @@ -2149,8 +2151,11 @@ bool GVN::performPRE(Function &F) { // Because we have added a PHI-use of the pointer value, it has now // "escaped" from alias analysis' perspective. We need to inform // AA of this. - for (unsigned ii = 0, ee = Phi->getNumIncomingValues(); ii != ee; ++ii) - VN.getAliasAnalysis()->addEscapingUse(Phi->getOperandUse(2*ii)); + for (unsigned ii = 0, ee = Phi->getNumIncomingValues(); ii != ee; + ++ii) { + unsigned jj = PHINode::getOperandNumForIncomingValue(ii); + VN.getAliasAnalysis()->addEscapingUse(Phi->getOperandUse(jj)); + } if (MD) MD->invalidateCachedPointerInfo(Phi); diff --git a/lib/Transforms/Scalar/IndVarSimplify.cpp b/lib/Transforms/Scalar/IndVarSimplify.cpp index 04ee7c8..1d79339 100644 --- a/lib/Transforms/Scalar/IndVarSimplify.cpp +++ b/lib/Transforms/Scalar/IndVarSimplify.cpp @@ -62,14 +62,15 @@ #include "llvm/ADT/STLExtras.h" using namespace llvm; -STATISTIC(NumRemoved , "Number of aux indvars removed"); -STATISTIC(NumWidened , "Number of indvars widened"); -STATISTIC(NumInserted, "Number of canonical indvars added"); -STATISTIC(NumReplaced, "Number of exit values replaced"); -STATISTIC(NumLFTR , "Number of loop exit tests replaced"); -STATISTIC(NumElimExt , "Number of IV sign/zero extends eliminated"); -STATISTIC(NumElimRem , "Number of IV remainder operations eliminated"); -STATISTIC(NumElimCmp , "Number of IV comparisons eliminated"); +STATISTIC(NumRemoved , "Number of aux indvars removed"); +STATISTIC(NumWidened , "Number of indvars widened"); +STATISTIC(NumInserted , "Number of canonical indvars added"); +STATISTIC(NumReplaced , "Number of exit values replaced"); +STATISTIC(NumLFTR , "Number of loop exit tests replaced"); +STATISTIC(NumElimIdentity, "Number of IV identities eliminated"); +STATISTIC(NumElimExt , "Number of IV sign/zero extends eliminated"); +STATISTIC(NumElimRem , "Number of IV remainder operations eliminated"); +STATISTIC(NumElimCmp , "Number of IV comparisons eliminated"); // DisableIVRewrite mode currently affects IVUsers, so is defined in libAnalysis // and referenced here. @@ -84,12 +85,22 @@ namespace { ScalarEvolution *SE; DominatorTree *DT; TargetData *TD; + + PHINode *CurrIV; // Current IV being simplified. + + // Instructions processed by SimplifyIVUsers for CurrIV. + SmallPtrSet<Instruction*,16> Simplified; + + // Use-def pairs if IVUsers waiting to be processed for CurrIV. + SmallVector<std::pair<Instruction*, Instruction*>, 8> SimpleIVUsers; + SmallVector<WeakVH, 16> DeadInsts; bool Changed; public: static char ID; // Pass identification, replacement for typeid - IndVarSimplify() : LoopPass(ID), IU(0), LI(0), SE(0), DT(0), TD(0) { + IndVarSimplify() : LoopPass(ID), IU(0), LI(0), SE(0), DT(0), TD(0), + CurrIV(0), Changed(false) { initializeIndVarSimplifyPass(*PassRegistry::getPassRegistry()); } @@ -105,7 +116,8 @@ namespace { AU.addPreserved<ScalarEvolution>(); AU.addPreservedID(LoopSimplifyID); AU.addPreservedID(LCSSAID); - AU.addPreserved<IVUsers>(); + if (!DisableIVRewrite) + AU.addPreserved<IVUsers>(); AU.setPreservesCFG(); } @@ -113,11 +125,15 @@ namespace { bool isValidRewrite(Value *FromVal, Value *ToVal); void SimplifyIVUsers(SCEVExpander &Rewriter); + void SimplifyIVUsersNoRewrite(Loop *L, SCEVExpander &Rewriter); + + bool EliminateIVUser(Instruction *UseInst, Instruction *IVOperand); void EliminateIVComparison(ICmpInst *ICmp, Value *IVOperand); void EliminateIVRemainder(BinaryOperator *Rem, Value *IVOperand, - bool IsSigned, - PHINode *IVPhi); + bool IsSigned); + void pushIVUsers(Instruction *Def); + bool isSimpleIVUser(Instruction *I, const Loop *L); void RewriteNonIntegerIVs(Loop *L); ICmpInst *LinearFunctionTestReplace(Loop *L, const SCEV *BackedgeTakenCount, @@ -483,6 +499,36 @@ void IndVarSimplify::RewriteNonIntegerIVs(Loop *L) { SE->forgetLoop(L); } +/// SimplifyIVUsers - Iteratively perform simplification on IVUsers within this +/// loop. IVUsers is treated as a worklist. Each successive simplification may +/// push more users which may themselves be candidates for simplification. +/// +/// This is the old approach to IV simplification to be replaced by +/// SimplifyIVUsersNoRewrite. +/// +void IndVarSimplify::SimplifyIVUsers(SCEVExpander &Rewriter) { + // Each round of simplification involves a round of eliminating operations + // followed by a round of widening IVs. A single IVUsers worklist is used + // across all rounds. The inner loop advances the user. If widening exposes + // more uses, then another pass through the outer loop is triggered. + for (IVUsers::iterator I = IU->begin(); I != IU->end(); ++I) { + Instruction *UseInst = I->getUser(); + Value *IVOperand = I->getOperandValToReplace(); + + if (ICmpInst *ICmp = dyn_cast<ICmpInst>(UseInst)) { + EliminateIVComparison(ICmp, IVOperand); + continue; + } + if (BinaryOperator *Rem = dyn_cast<BinaryOperator>(UseInst)) { + bool IsSigned = Rem->getOpcode() == Instruction::SRem; + if (IsSigned || Rem->getOpcode() == Instruction::URem) { + EliminateIVRemainder(Rem, IVOperand, IsSigned); + continue; + } + } + } +} + namespace { // Collect information about induction variables that are used by sign/zero // extend operations. This information is recorded by CollectExtend and @@ -493,33 +539,30 @@ namespace { WideIVInfo() : WidestNativeType(0), IsSigned(false) {} }; - typedef std::map<PHINode *, WideIVInfo> WideIVMap; } /// CollectExtend - Update information about the induction variable that is /// extended by this sign or zero extend operation. This is used to determine /// the final width of the IV before actually widening it. -static void CollectExtend(CastInst *Cast, PHINode *Phi, bool IsSigned, - WideIVMap &IVMap, ScalarEvolution *SE, - const TargetData *TD) { +static void CollectExtend(CastInst *Cast, bool IsSigned, WideIVInfo &WI, + ScalarEvolution *SE, const TargetData *TD) { const Type *Ty = Cast->getType(); uint64_t Width = SE->getTypeSizeInBits(Ty); if (TD && !TD->isLegalInteger(Width)) return; - WideIVInfo &IVInfo = IVMap[Phi]; - if (!IVInfo.WidestNativeType) { - IVInfo.WidestNativeType = SE->getEffectiveSCEVType(Ty); - IVInfo.IsSigned = IsSigned; + if (!WI.WidestNativeType) { + WI.WidestNativeType = SE->getEffectiveSCEVType(Ty); + WI.IsSigned = IsSigned; return; } // We extend the IV to satisfy the sign of its first user, arbitrarily. - if (IVInfo.IsSigned != IsSigned) + if (WI.IsSigned != IsSigned) return; - if (Width > SE->getTypeSizeInBits(IVInfo.WidestNativeType)) - IVInfo.WidestNativeType = SE->getEffectiveSCEVType(Ty); + if (Width > SE->getTypeSizeInBits(WI.WidestNativeType)) + WI.WidestNativeType = SE->getEffectiveSCEVType(Ty); } namespace { @@ -529,43 +572,44 @@ namespace { /// inserting truncs whenever we stop propagating the type. /// class WidenIV { + // Parameters PHINode *OrigPhi; const Type *WideType; bool IsSigned; - IVUsers *IU; - LoopInfo *LI; - Loop *L; + // Context + LoopInfo *LI; + Loop *L; ScalarEvolution *SE; - DominatorTree *DT; - SmallVectorImpl<WeakVH> &DeadInsts; + DominatorTree *DT; + // Result PHINode *WidePhi; Instruction *WideInc; const SCEV *WideIncExpr; + SmallVectorImpl<WeakVH> &DeadInsts; - SmallPtrSet<Instruction*,16> Processed; + SmallPtrSet<Instruction*,16> Widened; public: - WidenIV(PHINode *PN, const WideIVInfo &IVInfo, IVUsers *IUsers, - LoopInfo *LInfo, ScalarEvolution *SEv, DominatorTree *DTree, + WidenIV(PHINode *PN, const WideIVInfo &WI, LoopInfo *LInfo, + ScalarEvolution *SEv, DominatorTree *DTree, SmallVectorImpl<WeakVH> &DI) : OrigPhi(PN), - WideType(IVInfo.WidestNativeType), - IsSigned(IVInfo.IsSigned), - IU(IUsers), + WideType(WI.WidestNativeType), + IsSigned(WI.IsSigned), LI(LInfo), L(LI->getLoopFor(OrigPhi->getParent())), SE(SEv), DT(DTree), - DeadInsts(DI), WidePhi(0), WideInc(0), - WideIncExpr(0) { + WideIncExpr(0), + DeadInsts(DI) { assert(L->getHeader() == OrigPhi->getParent() && "Phi must be an IV"); } - bool CreateWideIV(SCEVExpander &Rewriter); + PHINode *CreateWideIV(SCEVExpander &Rewriter); protected: Instruction *CloneIVUser(Instruction *NarrowUse, @@ -580,52 +624,6 @@ protected: }; } // anonymous namespace -/// SimplifyIVUsers - Iteratively perform simplification on IVUsers within this -/// loop. IVUsers is treated as a worklist. Each successive simplification may -/// push more users which may themselves be candidates for simplification. -/// -void IndVarSimplify::SimplifyIVUsers(SCEVExpander &Rewriter) { - WideIVMap IVMap; - - // Each round of simplification involves a round of eliminating operations - // followed by a round of widening IVs. A single IVUsers worklist is used - // across all rounds. The inner loop advances the user. If widening exposes - // more uses, then another pass through the outer loop is triggered. - for (IVUsers::iterator I = IU->begin(), E = IU->end(); I != E;) { - for(; I != E; ++I) { - Instruction *UseInst = I->getUser(); - Value *IVOperand = I->getOperandValToReplace(); - - if (DisableIVRewrite) { - if (CastInst *Cast = dyn_cast<CastInst>(UseInst)) { - bool IsSigned = Cast->getOpcode() == Instruction::SExt; - if (IsSigned || Cast->getOpcode() == Instruction::ZExt) { - CollectExtend(Cast, I->getPhi(), IsSigned, IVMap, SE, TD); - continue; - } - } - } - if (ICmpInst *ICmp = dyn_cast<ICmpInst>(UseInst)) { - EliminateIVComparison(ICmp, IVOperand); - continue; - } - if (BinaryOperator *Rem = dyn_cast<BinaryOperator>(UseInst)) { - bool IsSigned = Rem->getOpcode() == Instruction::SRem; - if (IsSigned || Rem->getOpcode() == Instruction::URem) { - EliminateIVRemainder(Rem, IVOperand, IsSigned, I->getPhi()); - continue; - } - } - } - for (WideIVMap::const_iterator I = IVMap.begin(), E = IVMap.end(); - I != E; ++I) { - WidenIV Widener(I->first, I->second, IU, LI, SE, DT, DeadInsts); - if (Widener.CreateWideIV(Rewriter)) - Changed = true; - } - } -} - static Value *getExtend( Value *NarrowOper, const Type *WideType, bool IsSigned, IRBuilder<> &Builder) { return IsSigned ? Builder.CreateSExt(NarrowOper, WideType) : @@ -744,7 +742,7 @@ Instruction *WidenIV::WidenIVUse(Instruction *NarrowUse, return 0; // Handle data flow merges and bizarre phi cycles. - if (!Processed.insert(NarrowUse)) + if (!Widened.insert(NarrowUse)) return 0; // Our raison d'etre! Eliminate sign and zero extension. @@ -775,9 +773,11 @@ Instruction *WidenIV::WidenIVUse(Instruction *NarrowUse, NarrowUse->replaceAllUsesWith(NewDef); DeadInsts.push_back(NarrowUse); } - // Now that the extend is gone, expose it's uses to IVUsers for potential - // further simplification within SimplifyIVUsers. - IU->AddUsersIfInteresting(WideDef, WidePhi); + // Now that the extend is gone, we want to expose it's uses for potential + // further simplification. We don't need to directly inform SimplifyIVUsers + // of the new users, because their parent IV will be processed later as a + // new loop phi. If we preserved IVUsers analysis, we would also want to + // push the uses of WideDef here. // No further widening is needed. The deceased [sz]ext had done it for us. return 0; @@ -807,7 +807,7 @@ Instruction *WidenIV::WidenIVUse(Instruction *NarrowUse, // outside the loop without overflow. This suggests that the wide use // evaluates to the same expression as the extended narrow use, but doesn't // absolutely guarantee it. Hence the following failsafe check. In rare cases - // where it fails, we simple throw away the newly created wide use. + // where it fails, we simply throw away the newly created wide use. if (WideAddRec != SE->getSCEV(WideUse)) { DEBUG(dbgs() << "Wide use expression mismatch: " << *WideUse << ": " << *SE->getSCEV(WideUse) << " != " << *WideAddRec << "\n"); @@ -822,18 +822,18 @@ Instruction *WidenIV::WidenIVUse(Instruction *NarrowUse, /// CreateWideIV - Process a single induction variable. First use the /// SCEVExpander to create a wide induction variable that evaluates to the same /// recurrence as the original narrow IV. Then use a worklist to forward -/// traverse the narrow IV's def-use chain. After WidenIVUse as processed all +/// traverse the narrow IV's def-use chain. After WidenIVUse has processed all /// interesting IV users, the narrow IV will be isolated for removal by /// DeleteDeadPHIs. /// /// It would be simpler to delete uses as they are processed, but we must avoid /// invalidating SCEV expressions. /// -bool WidenIV::CreateWideIV(SCEVExpander &Rewriter) { +PHINode *WidenIV::CreateWideIV(SCEVExpander &Rewriter) { // Is this phi an induction variable? const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(SE->getSCEV(OrigPhi)); if (!AddRec) - return false; + return NULL; // Widen the induction variable expression. const SCEV *WideIVExpr = IsSigned ? @@ -846,9 +846,9 @@ bool WidenIV::CreateWideIV(SCEVExpander &Rewriter) { // Can the IV be extended outside the loop without overflow? AddRec = dyn_cast<SCEVAddRecExpr>(WideIVExpr); if (!AddRec || AddRec->getLoop() != L) - return false; + return NULL; - // An AddRec must have loop-invariant operands. Since this AddRec it + // An AddRec must have loop-invariant operands. Since this AddRec is // materialized by a loop header phi, the expression cannot have any post-loop // operands, so they must dominate the loop header. assert(SE->properlyDominates(AddRec->getStart(), L->getHeader()) && @@ -876,7 +876,7 @@ bool WidenIV::CreateWideIV(SCEVExpander &Rewriter) { ++NumWidened; // Traverse the def-use chain using a worklist starting at the original IV. - assert(Processed.empty() && "expect initial state" ); + assert(Widened.empty() && "expect initial state" ); // Each worklist entry has a Narrow def-use link and Wide def. SmallVector<std::pair<Use *, Instruction *>, 8> NarrowIVUsers; @@ -906,7 +906,7 @@ bool WidenIV::CreateWideIV(SCEVExpander &Rewriter) { if (NarrowDef->use_empty()) DeadInsts.push_back(NarrowDef); } - return true; + return WidePhi; } void IndVarSimplify::EliminateIVComparison(ICmpInst *ICmp, Value *IVOperand) { @@ -945,8 +945,7 @@ void IndVarSimplify::EliminateIVComparison(ICmpInst *ICmp, Value *IVOperand) { void IndVarSimplify::EliminateIVRemainder(BinaryOperator *Rem, Value *IVOperand, - bool IsSigned, - PHINode *IVPhi) { + bool IsSigned) { // We're only interested in the case where we know something about // the numerator. if (IVOperand != Rem->getOperand(0)) @@ -989,15 +988,144 @@ void IndVarSimplify::EliminateIVRemainder(BinaryOperator *Rem, } // Inform IVUsers about the new users. - if (Instruction *I = dyn_cast<Instruction>(Rem->getOperand(0))) - IU->AddUsersIfInteresting(I, IVPhi); - + if (IU) { + if (Instruction *I = dyn_cast<Instruction>(Rem->getOperand(0))) + IU->AddUsersIfInteresting(I); + } DEBUG(dbgs() << "INDVARS: Simplified rem: " << *Rem << '\n'); ++NumElimRem; Changed = true; DeadInsts.push_back(Rem); } +/// EliminateIVUser - Eliminate an operation that consumes a simple IV and has +/// no observable side-effect given the range of IV values. +bool IndVarSimplify::EliminateIVUser(Instruction *UseInst, + Instruction *IVOperand) { + if (ICmpInst *ICmp = dyn_cast<ICmpInst>(UseInst)) { + EliminateIVComparison(ICmp, IVOperand); + return true; + } + if (BinaryOperator *Rem = dyn_cast<BinaryOperator>(UseInst)) { + bool IsSigned = Rem->getOpcode() == Instruction::SRem; + if (IsSigned || Rem->getOpcode() == Instruction::URem) { + EliminateIVRemainder(Rem, IVOperand, IsSigned); + return true; + } + } + + // Eliminate any operation that SCEV can prove is an identity function. + if (!SE->isSCEVable(UseInst->getType()) || + (SE->getSCEV(UseInst) != SE->getSCEV(IVOperand))) + return false; + + UseInst->replaceAllUsesWith(IVOperand); + + DEBUG(dbgs() << "INDVARS: Eliminated identity: " << *UseInst << '\n'); + ++NumElimIdentity; + Changed = true; + DeadInsts.push_back(UseInst); + return true; +} + +/// pushIVUsers - Add all uses of Def to the current IV's worklist. +/// +void IndVarSimplify::pushIVUsers(Instruction *Def) { + + for (Value::use_iterator UI = Def->use_begin(), E = Def->use_end(); + UI != E; ++UI) { + Instruction *User = cast<Instruction>(*UI); + + // Avoid infinite or exponential worklist processing. + // Also ensure unique worklist users. + if (Simplified.insert(User)) + SimpleIVUsers.push_back(std::make_pair(User, Def)); + } +} + +/// isSimpleIVUser - Return true if this instruction generates a simple SCEV +/// expression in terms of that IV. +/// +/// This is similar to IVUsers' isInsteresting() but processes each instruction +/// non-recursively when the operand is already known to be a simpleIVUser. +/// +bool IndVarSimplify::isSimpleIVUser(Instruction *I, const Loop *L) { + if (!SE->isSCEVable(I->getType())) + return false; + + // Get the symbolic expression for this instruction. + const SCEV *S = SE->getSCEV(I); + + // Only consider affine recurrences. + const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(S); + if (AR && AR->getLoop() == L) + return true; + + return false; +} + +/// SimplifyIVUsersNoRewrite - Iteratively perform simplification on a worklist +/// of IV users. Each successive simplification may push more users which may +/// themselves be candidates for simplification. +/// +/// The "NoRewrite" algorithm does not require IVUsers analysis. Instead, it +/// simplifies instructions in-place during analysis. Rather than rewriting +/// induction variables bottom-up from their users, it transforms a chain of +/// IVUsers top-down, updating the IR only when it encouters a clear +/// optimization opportunitiy. A SCEVExpander "Rewriter" instance is still +/// needed, but only used to generate a new IV (phi) of wider type for sign/zero +/// extend elimination. +/// +/// Once DisableIVRewrite is default, LSR will be the only client of IVUsers. +/// +void IndVarSimplify::SimplifyIVUsersNoRewrite(Loop *L, SCEVExpander &Rewriter) { + // Simplification is performed independently for each IV, as represented by a + // loop header phi. Each round of simplification first iterates through the + // SimplifyIVUsers worklist, then determines whether the current IV should be + // widened. Widening adds a new phi to LoopPhis, inducing another round of + // simplification on the wide IV. + SmallVector<PHINode*, 8> LoopPhis; + for (BasicBlock::iterator I = L->getHeader()->begin(); isa<PHINode>(I); ++I) { + LoopPhis.push_back(cast<PHINode>(I)); + } + while (!LoopPhis.empty()) { + CurrIV = LoopPhis.pop_back_val(); + Simplified.clear(); + assert(SimpleIVUsers.empty() && "expect empty IV users list"); + + WideIVInfo WI; + + pushIVUsers(CurrIV); + + while (!SimpleIVUsers.empty()) { + Instruction *UseInst, *Operand; + tie(UseInst, Operand) = SimpleIVUsers.pop_back_val(); + + if (EliminateIVUser(UseInst, Operand)) { + pushIVUsers(Operand); + continue; + } + if (CastInst *Cast = dyn_cast<CastInst>(UseInst)) { + bool IsSigned = Cast->getOpcode() == Instruction::SExt; + if (IsSigned || Cast->getOpcode() == Instruction::ZExt) { + CollectExtend(Cast, IsSigned, WI, SE, TD); + } + continue; + } + if (isSimpleIVUser(UseInst, L)) { + pushIVUsers(UseInst); + } + } + if (WI.WidestNativeType) { + WidenIV Widener(CurrIV, WI, LI, SE, DT, DeadInsts); + if (PHINode *WidePhi = Widener.CreateWideIV(Rewriter)) { + Changed = true; + LoopPhis.push_back(WidePhi); + } + } + } +} + bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) { // If LoopSimplify form is not available, stay out of trouble. Some notes: // - LSR currently only supports LoopSimplify-form loops. Indvars' @@ -1010,12 +1138,15 @@ bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) { if (!L->isLoopSimplifyForm()) return false; - IU = &getAnalysis<IVUsers>(); + if (!DisableIVRewrite) + IU = &getAnalysis<IVUsers>(); LI = &getAnalysis<LoopInfo>(); SE = &getAnalysis<ScalarEvolution>(); DT = &getAnalysis<DominatorTree>(); TD = getAnalysisIfAvailable<TargetData>(); + CurrIV = NULL; + Simplified.clear(); DeadInsts.clear(); Changed = false; @@ -1040,7 +1171,10 @@ bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) { RewriteLoopExitValues(L, Rewriter); // Eliminate redundant IV users. - SimplifyIVUsers(Rewriter); + if (DisableIVRewrite) + SimplifyIVUsersNoRewrite(L, Rewriter); + else + SimplifyIVUsers(Rewriter); // Compute the type of the largest recurrence expression, and decide whether // a canonical induction variable should be inserted. @@ -1146,9 +1280,8 @@ bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) { // For completeness, inform IVUsers of the IV use in the newly-created // loop exit test instruction. - if (NewICmp) - IU->AddUsersIfInteresting(cast<Instruction>(NewICmp->getOperand(0)), - IndVar); + if (NewICmp && IU) + IU->AddUsersIfInteresting(cast<Instruction>(NewICmp->getOperand(0))); // Clean up dead instructions. Changed |= DeleteDeadPHIs(L->getHeader()); @@ -1267,6 +1400,8 @@ void IndVarSimplify::RewriteIVExpressions(Loop *L, SCEVExpander &Rewriter) { // Patch the new value into place. if (Op->hasName()) NewVal->takeName(Op); + if (Instruction *NewValI = dyn_cast<Instruction>(NewVal)) + NewValI->setDebugLoc(User->getDebugLoc()); User->replaceUsesOfWith(Op, NewVal); UI->setOperandValToReplace(NewVal); @@ -1579,5 +1714,6 @@ void IndVarSimplify::HandleFloatingPointIV(Loop *L, PHINode *PN) { } // Add a new IVUsers entry for the newly-created integer PHI. - IU->AddUsersIfInteresting(NewPHI, NewPHI); + if (IU) + IU->AddUsersIfInteresting(NewPHI); } diff --git a/lib/Transforms/Scalar/LoopDeletion.cpp b/lib/Transforms/Scalar/LoopDeletion.cpp index 753a558..f7f3298 100644 --- a/lib/Transforms/Scalar/LoopDeletion.cpp +++ b/lib/Transforms/Scalar/LoopDeletion.cpp @@ -190,7 +190,9 @@ bool LoopDeletion::runOnLoop(Loop* L, LPPassManager& LPM) { BasicBlock* exitingBlock = exitingBlocks[0]; BasicBlock::iterator BI = exitBlock->begin(); while (PHINode* P = dyn_cast<PHINode>(BI)) { - P->replaceUsesOfWith(exitingBlock, preheader); + int j = P->getBasicBlockIndex(exitingBlock); + assert(j >= 0 && "Can't find exiting block in exit block's phi node!"); + P->setIncomingBlock(j, preheader); for (unsigned i = 1; i < exitingBlocks.size(); ++i) P->removeIncomingValue(exitingBlocks[i]); ++BI; diff --git a/lib/Transforms/Scalar/LoopRotation.cpp b/lib/Transforms/Scalar/LoopRotation.cpp index 47dced3..9fd0958 100644 --- a/lib/Transforms/Scalar/LoopRotation.cpp +++ b/lib/Transforms/Scalar/LoopRotation.cpp @@ -220,7 +220,7 @@ bool LoopRotate::rotateLoop(Loop *L) { // For PHI nodes, the value available in OldPreHeader is just the // incoming value from OldPreHeader. for (; PHINode *PN = dyn_cast<PHINode>(I); ++I) - ValueMap[PN] = PN->getIncomingValue(PN->getBasicBlockIndex(OrigPreheader)); + ValueMap[PN] = PN->getIncomingValueForBlock(OrigPreheader); // For the rest of the instructions, either hoist to the OrigPreheader if // possible or create a clone in the OldPreHeader if not. diff --git a/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/lib/Transforms/Scalar/LoopStrengthReduce.cpp index 73ebd61..afa0bf8 100644 --- a/lib/Transforms/Scalar/LoopStrengthReduce.cpp +++ b/lib/Transforms/Scalar/LoopStrengthReduce.cpp @@ -1804,8 +1804,7 @@ LSRInstance::OptimizeLoopTermCond() { ExitingBlock->getInstList().insert(TermBr, Cond); // Clone the IVUse, as the old use still exists! - CondUse = &IU.AddUser(Cond, CondUse->getOperandValToReplace(), - CondUse->getPhi()); + CondUse = &IU.AddUser(Cond, CondUse->getOperandValToReplace()); TermBr->replaceUsesOfWith(OldCond, Cond); } } diff --git a/lib/Transforms/Scalar/LoopUnswitch.cpp b/lib/Transforms/Scalar/LoopUnswitch.cpp index e05f29c..840c4b6 100644 --- a/lib/Transforms/Scalar/LoopUnswitch.cpp +++ b/lib/Transforms/Scalar/LoopUnswitch.cpp @@ -1021,6 +1021,10 @@ void LoopUnswitch::SimplifyCode(std::vector<Instruction*> &Worklist, Loop *L) { while (PHINode *PN = dyn_cast<PHINode>(Succ->begin())) ReplaceUsesOfWith(PN, PN->getIncomingValue(0), Worklist, L, LPM); + // If Succ has any successors with PHI nodes, update them to have + // entries coming from Pred instead of Succ. + Succ->replaceAllUsesWith(Pred); + // Move all of the successor contents from Succ to Pred. Pred->getInstList().splice(BI, Succ->getInstList(), Succ->begin(), Succ->end()); @@ -1028,10 +1032,6 @@ void LoopUnswitch::SimplifyCode(std::vector<Instruction*> &Worklist, Loop *L) { BI->eraseFromParent(); RemoveFromWorklist(BI, Worklist); - // If Succ has any successors with PHI nodes, update them to have - // entries coming from Pred instead of Succ. - Succ->replaceAllUsesWith(Pred); - // Remove Succ from the loop tree. LI->removeBlock(Succ); LPM->deleteSimpleAnalysisValue(Succ, L); diff --git a/lib/Transforms/Scalar/ObjCARC.cpp b/lib/Transforms/Scalar/ObjCARC.cpp index 6cd35e5..89a451e 100644 --- a/lib/Transforms/Scalar/ObjCARC.cpp +++ b/lib/Transforms/Scalar/ObjCARC.cpp @@ -33,6 +33,7 @@ #include "llvm/Intrinsics.h" #include "llvm/GlobalVariable.h" #include "llvm/DerivedTypes.h" +#include "llvm/Module.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Support/CallSite.h" @@ -564,6 +565,29 @@ static const Value *FindSingleUseIdentifiedObject(const Value *Arg) { return 0; } +/// ModuleHasARC - Test if the given module looks interesting to run ARC +/// optimization on. +static bool ModuleHasARC(const Module &M) { + return + M.getNamedValue("objc_retain") || + M.getNamedValue("objc_release") || + M.getNamedValue("objc_autorelease") || + M.getNamedValue("objc_retainAutoreleasedReturnValue") || + M.getNamedValue("objc_retainBlock") || + M.getNamedValue("objc_autoreleaseReturnValue") || + M.getNamedValue("objc_autoreleasePoolPush") || + M.getNamedValue("objc_loadWeakRetained") || + M.getNamedValue("objc_loadWeak") || + M.getNamedValue("objc_destroyWeak") || + M.getNamedValue("objc_storeWeak") || + M.getNamedValue("objc_initWeak") || + M.getNamedValue("objc_moveWeak") || + M.getNamedValue("objc_copyWeak") || + M.getNamedValue("objc_retainedObject") || + M.getNamedValue("objc_unretainedObject") || + M.getNamedValue("objc_unretainedPointer"); +} + //===----------------------------------------------------------------------===// // ARC AliasAnalysis. //===----------------------------------------------------------------------===// @@ -749,8 +773,12 @@ namespace { /// ObjCARCExpand - Early ARC transformations. class ObjCARCExpand : public FunctionPass { virtual void getAnalysisUsage(AnalysisUsage &AU) const; + virtual bool doInitialization(Module &M); virtual bool runOnFunction(Function &F); + /// Run - A flag indicating whether this optimization pass should run. + bool Run; + public: static char ID; ObjCARCExpand() : FunctionPass(ID) { @@ -771,10 +799,19 @@ void ObjCARCExpand::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesCFG(); } +bool ObjCARCExpand::doInitialization(Module &M) { + Run = ModuleHasARC(M); + return false; +} + bool ObjCARCExpand::runOnFunction(Function &F) { if (!EnableARCOpts) return false; + // If nothing in the Module uses ARC, don't do anything. + if (!Run) + return false; + bool Changed = false; for (inst_iterator I = inst_begin(&F), E = inst_end(&F); I != E; ++I) { @@ -840,8 +877,9 @@ bool ObjCARCExpand::runOnFunction(Function &F) { // usually can't sink them past other calls, which would be the main // case where it would be useful. +/// TODO: The pointer returned from objc_loadWeakRetained is retained. + #include "llvm/GlobalAlias.h" -#include "llvm/Module.h" #include "llvm/Constants.h" #include "llvm/LLVMContext.h" #include "llvm/Support/ErrorHandling.h" @@ -1365,10 +1403,12 @@ namespace { bool Changed; ProvenanceAnalysis PA; + /// Run - A flag indicating whether this optimization pass should run. + bool Run; + /// RetainFunc, RelaseFunc - Declarations for objc_retain, /// objc_retainBlock, and objc_release. - Function *RetainFunc, *RetainBlockFunc, *RetainRVFunc, *ReleaseFunc, - *AutoreleaseFunc; + Function *RetainFunc, *RetainBlockFunc, *RetainRVFunc, *ReleaseFunc; /// RetainRVCallee, etc. - Declarations for ObjC runtime /// functions, for use in creating calls to them. These are initialized @@ -3069,6 +3109,10 @@ bool ObjCARCOpt::doInitialization(Module &M) { if (!EnableARCOpts) return false; + Run = ModuleHasARC(M); + if (!Run) + return false; + // Identify the imprecise release metadata kind. ImpreciseReleaseMDKind = M.getContext().getMDKindID("clang.imprecise_release"); @@ -3078,7 +3122,6 @@ bool ObjCARCOpt::doInitialization(Module &M) { RetainBlockFunc = M.getFunction("objc_retainBlock"); RetainRVFunc = M.getFunction("objc_retainAutoreleasedReturnValue"); ReleaseFunc = M.getFunction("objc_release"); - AutoreleaseFunc = M.getFunction("objc_autorelease"); // Intuitively, objc_retain and others are nocapture, however in practice // they are not, because they return their argument value. And objc_release @@ -3098,6 +3141,10 @@ bool ObjCARCOpt::runOnFunction(Function &F) { if (!EnableARCOpts) return false; + // If nothing in the Module uses ARC, don't do anything. + if (!Run) + return false; + Changed = false; PA.setAA(&getAnalysis<AliasAnalysis>()); @@ -3162,6 +3209,9 @@ namespace { DominatorTree *DT; ProvenanceAnalysis PA; + /// Run - A flag indicating whether this optimization pass should run. + bool Run; + /// StoreStrongCallee, etc. - Declarations for ObjC runtime /// functions, for use in creating calls to them. These are initialized /// lazily to avoid cluttering up the Module with unused declarations. @@ -3384,6 +3434,10 @@ void ObjCARCContract::ContractRelease(Instruction *Release, } bool ObjCARCContract::doInitialization(Module &M) { + Run = ModuleHasARC(M); + if (!Run) + return false; + // These are initialized lazily. StoreStrongCallee = 0; RetainAutoreleaseCallee = 0; @@ -3407,6 +3461,10 @@ bool ObjCARCContract::runOnFunction(Function &F) { if (!EnableARCOpts) return false; + // If nothing in the Module uses ARC, don't do anything. + if (!Run) + return false; + Changed = false; AA = &getAnalysis<AliasAnalysis>(); DT = &getAnalysis<DominatorTree>(); diff --git a/lib/Transforms/Scalar/ScalarReplAggregates.cpp b/lib/Transforms/Scalar/ScalarReplAggregates.cpp index beef127..46ac948 100644 --- a/lib/Transforms/Scalar/ScalarReplAggregates.cpp +++ b/lib/Transforms/Scalar/ScalarReplAggregates.cpp @@ -293,6 +293,11 @@ AllocaInst *ConvertToScalarInfo::TryConvert(AllocaInst *AI) { if (ScalarKind == Unknown) ScalarKind = Integer; + // FIXME: It should be possible to promote the vector type up to the alloca's + // size. + if (ScalarKind == Vector && VectorTy->getBitWidth() != AllocaSize * 8) + ScalarKind = Integer; + // If we were able to find a vector type that can handle this with // insert/extract elements, and if there was at least one use that had // a vector type, promote this to a vector. We don't want to promote @@ -384,7 +389,6 @@ void ConvertToScalarInfo::MergeInTypeForLoadOrStore(const Type *In, // Otherwise, we have a case that we can't handle with an optimized vector // form. We can still turn this into a large integer. ScalarKind = Integer; - VectorTy = 0; } /// MergeInVectorType - Handles the vector case of MergeInTypeForLoadOrStore, @@ -522,10 +526,22 @@ bool ConvertToScalarInfo::CanConvertToScalar(Value *V, uint64_t Offset) { // If this is a constant sized memset of a constant value (e.g. 0) we can // handle it. if (MemSetInst *MSI = dyn_cast<MemSetInst>(User)) { - // Store of constant value and constant size. - if (!isa<ConstantInt>(MSI->getValue()) || - !isa<ConstantInt>(MSI->getLength())) + // Store of constant value. + if (!isa<ConstantInt>(MSI->getValue())) + return false; + + // Store of constant size. + ConstantInt *Len = dyn_cast<ConstantInt>(MSI->getLength()); + if (!Len) return false; + + // If the size differs from the alloca, we can only convert the alloca to + // an integer bag-of-bits. + // FIXME: This should handle all of the cases that are currently accepted + // as vector element insertions. + if (Len->getZExtValue() != AllocaSize || Offset != 0) + ScalarKind = Integer; + IsNotTrivial = true; // Can't be mem2reg'd. HadNonMemTransferAccess = true; continue; diff --git a/lib/Transforms/Utils/BasicBlockUtils.cpp b/lib/Transforms/Utils/BasicBlockUtils.cpp index 92464e8..b4f74f9 100644 --- a/lib/Transforms/Utils/BasicBlockUtils.cpp +++ b/lib/Transforms/Utils/BasicBlockUtils.cpp @@ -153,13 +153,13 @@ bool llvm::MergeBlockIntoPredecessor(BasicBlock *BB, Pass *P) { // Delete the unconditional branch from the predecessor... PredBB->getInstList().pop_back(); - // Move all definitions in the successor to the predecessor... - PredBB->getInstList().splice(PredBB->end(), BB->getInstList()); - // Make all PHI nodes that referred to BB now refer to Pred as their // source... BB->replaceAllUsesWith(PredBB); + // Move all definitions in the successor to the predecessor... + PredBB->getInstList().splice(PredBB->end(), BB->getInstList()); + // Inherit predecessors name if it exists. if (!PredBB->hasName()) PredBB->takeName(BB); diff --git a/lib/Transforms/Utils/BreakCriticalEdges.cpp b/lib/Transforms/Utils/BreakCriticalEdges.cpp index d6206a3..92ce500 100644 --- a/lib/Transforms/Utils/BreakCriticalEdges.cpp +++ b/lib/Transforms/Utils/BreakCriticalEdges.cpp @@ -193,44 +193,22 @@ BasicBlock *llvm::SplitCriticalEdge(TerminatorInst *TI, unsigned SuccNum, // If there are any PHI nodes in DestBB, we need to update them so that they // merge incoming values from NewBB instead of from TIBB. - if (PHINode *APHI = dyn_cast<PHINode>(DestBB->begin())) { - // This conceptually does: - // foreach (PHINode *PN in DestBB) - // PN->setIncomingBlock(PN->getIncomingBlock(TIBB), NewBB); - // but is optimized for two cases. - - if (APHI->getNumIncomingValues() <= 8) { // Small # preds case. - unsigned BBIdx = 0; - for (BasicBlock::iterator I = DestBB->begin(); isa<PHINode>(I); ++I) { - // We no longer enter through TIBB, now we come in through NewBB. - // Revector exactly one entry in the PHI node that used to come from - // TIBB to come from NewBB. - PHINode *PN = cast<PHINode>(I); - - // Reuse the previous value of BBIdx if it lines up. In cases where we - // have multiple phi nodes with *lots* of predecessors, this is a speed - // win because we don't have to scan the PHI looking for TIBB. This - // happens because the BB list of PHI nodes are usually in the same - // order. - if (PN->getIncomingBlock(BBIdx) != TIBB) - BBIdx = PN->getBasicBlockIndex(TIBB); - PN->setIncomingBlock(BBIdx, NewBB); - } - } else { - // However, the foreach loop is slow for blocks with lots of predecessors - // because PHINode::getIncomingBlock is O(n) in # preds. Instead, walk - // the user list of TIBB to find the PHI nodes. - SmallPtrSet<PHINode*, 16> UpdatedPHIs; - - for (Value::use_iterator UI = TIBB->use_begin(), E = TIBB->use_end(); - UI != E; ) { - Value::use_iterator Use = UI++; - if (PHINode *PN = dyn_cast<PHINode>(*Use)) { - // Remove one entry from each PHI. - if (PN->getParent() == DestBB && UpdatedPHIs.insert(PN)) - PN->setOperand(Use.getOperandNo(), NewBB); - } - } + { + unsigned BBIdx = 0; + for (BasicBlock::iterator I = DestBB->begin(); isa<PHINode>(I); ++I) { + // We no longer enter through TIBB, now we come in through NewBB. + // Revector exactly one entry in the PHI node that used to come from + // TIBB to come from NewBB. + PHINode *PN = cast<PHINode>(I); + + // Reuse the previous value of BBIdx if it lines up. In cases where we + // have multiple phi nodes with *lots* of predecessors, this is a speed + // win because we don't have to scan the PHI looking for TIBB. This + // happens because the BB list of PHI nodes are usually in the same + // order. + if (PN->getIncomingBlock(BBIdx) != TIBB) + BBIdx = PN->getBasicBlockIndex(TIBB); + PN->setIncomingBlock(BBIdx, NewBB); } } diff --git a/lib/Transforms/Utils/CloneFunction.cpp b/lib/Transforms/Utils/CloneFunction.cpp index d967ceb..561b69d 100644 --- a/lib/Transforms/Utils/CloneFunction.cpp +++ b/lib/Transforms/Utils/CloneFunction.cpp @@ -572,12 +572,12 @@ void llvm::CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc, // removed, so we just need to splice the blocks. BI->eraseFromParent(); - // Move all the instructions in the succ to the pred. - I->getInstList().splice(I->end(), Dest->getInstList()); - // Make all PHI nodes that referred to Dest now refer to I as their source. Dest->replaceAllUsesWith(I); + // Move all the instructions in the succ to the pred. + I->getInstList().splice(I->end(), Dest->getInstList()); + // Remove the dest block. Dest->eraseFromParent(); diff --git a/lib/Transforms/Utils/InlineFunction.cpp b/lib/Transforms/Utils/InlineFunction.cpp index 946e62f..18ecd61 100644 --- a/lib/Transforms/Utils/InlineFunction.cpp +++ b/lib/Transforms/Utils/InlineFunction.cpp @@ -1097,15 +1097,15 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI) { TheCall->replaceAllUsesWith(Returns[0]->getReturnValue()); } + // Update PHI nodes that use the ReturnBB to use the AfterCallBB. + BasicBlock *ReturnBB = Returns[0]->getParent(); + ReturnBB->replaceAllUsesWith(AfterCallBB); + // Splice the code from the return block into the block that it will return // to, which contains the code that was after the call. - BasicBlock *ReturnBB = Returns[0]->getParent(); AfterCallBB->getInstList().splice(AfterCallBB->begin(), ReturnBB->getInstList()); - // Update PHI nodes that use the ReturnBB to use the AfterCallBB. - ReturnBB->replaceAllUsesWith(AfterCallBB); - // Delete the return instruction now and empty ReturnBB now. Returns[0]->eraseFromParent(); ReturnBB->eraseFromParent(); @@ -1125,8 +1125,8 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI) { // Splice the code entry block into calling block, right before the // unconditional branch. - OrigBB->getInstList().splice(Br, CalleeEntry->getInstList()); CalleeEntry->replaceAllUsesWith(OrigBB); // Update PHI nodes + OrigBB->getInstList().splice(Br, CalleeEntry->getInstList()); // Remove the unconditional branch. OrigBB->getInstList().erase(Br); diff --git a/lib/Transforms/Utils/Local.cpp b/lib/Transforms/Utils/Local.cpp index 19c3c72..506e5e8 100644 --- a/lib/Transforms/Utils/Local.cpp +++ b/lib/Transforms/Utils/Local.cpp @@ -427,10 +427,6 @@ void llvm::MergeBasicBlockIntoOnlyPred(BasicBlock *DestBB, Pass *P) { BasicBlock *PredBB = DestBB->getSinglePredecessor(); assert(PredBB && "Block doesn't have a single predecessor!"); - // Splice all the instructions from PredBB to DestBB. - PredBB->getTerminator()->eraseFromParent(); - DestBB->getInstList().splice(DestBB->begin(), PredBB->getInstList()); - // Zap anything that took the address of DestBB. Not doing this will give the // address an invalid value. if (DestBB->hasAddressTaken()) { @@ -445,6 +441,10 @@ void llvm::MergeBasicBlockIntoOnlyPred(BasicBlock *DestBB, Pass *P) { // Anything that branched to PredBB now branches to DestBB. PredBB->replaceAllUsesWith(DestBB); + // Splice all the instructions from PredBB to DestBB. + PredBB->getTerminator()->eraseFromParent(); + DestBB->getInstList().splice(DestBB->begin(), PredBB->getInstList()); + if (P) { DominatorTree *DT = P->getAnalysisIfAvailable<DominatorTree>(); if (DT) { @@ -660,12 +660,17 @@ bool llvm::EliminateDuplicatePHINodes(BasicBlock *BB) { // them, which helps expose duplicates, but we have to check all the // operands to be safe in case instcombine hasn't run. uintptr_t Hash = 0; + // This hash algorithm is quite weak as hash functions go, but it seems + // to do a good enough job for this particular purpose, and is very quick. for (User::op_iterator I = PN->op_begin(), E = PN->op_end(); I != E; ++I) { - // This hash algorithm is quite weak as hash functions go, but it seems - // to do a good enough job for this particular purpose, and is very quick. Hash ^= reinterpret_cast<uintptr_t>(static_cast<Value *>(*I)); Hash = (Hash << 7) | (Hash >> (sizeof(uintptr_t) * CHAR_BIT - 7)); } + for (PHINode::block_iterator I = PN->block_begin(), E = PN->block_end(); + I != E; ++I) { + Hash ^= reinterpret_cast<uintptr_t>(static_cast<BasicBlock *>(*I)); + Hash = (Hash << 7) | (Hash >> (sizeof(uintptr_t) * CHAR_BIT - 7)); + } // Avoid colliding with the DenseMap sentinels ~0 and ~0-1. Hash >>= 1; // If we've never seen this hash value before, it's a unique PHI. diff --git a/lib/Transforms/Utils/LoopSimplify.cpp b/lib/Transforms/Utils/LoopSimplify.cpp index f02ffd2..e79fb5a 100644 --- a/lib/Transforms/Utils/LoopSimplify.cpp +++ b/lib/Transforms/Utils/LoopSimplify.cpp @@ -375,6 +375,7 @@ BasicBlock *LoopSimplify::InsertPreheaderForLoop(Loop *L) { SplitBlockPredecessors(Header, &OutsideBlocks[0], OutsideBlocks.size(), ".preheader", this); + NewBB->getTerminator()->setDebugLoc(Header->getFirstNonPHI()->getDebugLoc()); DEBUG(dbgs() << "LoopSimplify: Creating pre-header " << NewBB->getName() << "\n"); diff --git a/lib/Transforms/Utils/LoopUnroll.cpp b/lib/Transforms/Utils/LoopUnroll.cpp index 7da7271..6772511 100644 --- a/lib/Transforms/Utils/LoopUnroll.cpp +++ b/lib/Transforms/Utils/LoopUnroll.cpp @@ -47,6 +47,14 @@ static inline void RemapInstruction(Instruction *I, if (It != VMap.end()) I->setOperand(op, It->second); } + + if (PHINode *PN = dyn_cast<PHINode>(I)) { + for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { + ValueToValueMapTy::iterator It = VMap.find(PN->getIncomingBlock(i)); + if (It != VMap.end()) + PN->setIncomingBlock(i, cast<BasicBlock>(It->second)); + } + } } /// FoldBlockIntoPredecessor - Folds a basic block into its predecessor if it @@ -75,13 +83,13 @@ static BasicBlock *FoldBlockIntoPredecessor(BasicBlock *BB, LoopInfo* LI) { // Delete the unconditional branch from the predecessor... OnlyPred->getInstList().pop_back(); - // Move all definitions in the successor to the predecessor... - OnlyPred->getInstList().splice(OnlyPred->end(), BB->getInstList()); - // Make all PHI nodes that referred to BB now refer to Pred as their // source... BB->replaceAllUsesWith(OnlyPred); + // Move all definitions in the successor to the predecessor... + OnlyPred->getInstList().splice(OnlyPred->end(), BB->getInstList()); + std::string OldName = BB->getName(); // Erase basic block from the function... @@ -247,16 +255,14 @@ bool llvm::UnrollLoop(Loop *L, unsigned Count, // the successor of the latch block. The successor of the exit block will // be updated specially after unrolling all the way. if (*BB != LatchBlock) - for (Value::use_iterator UI = (*BB)->use_begin(), UE = (*BB)->use_end(); - UI != UE;) { - Instruction *UseInst = cast<Instruction>(*UI); - ++UI; - if (isa<PHINode>(UseInst) && !L->contains(UseInst)) { - PHINode *phi = cast<PHINode>(UseInst); - Value *Incoming = phi->getIncomingValueForBlock(*BB); - phi->addIncoming(Incoming, New); - } - } + for (succ_iterator SI = succ_begin(*BB), SE = succ_end(*BB); SI != SE; + ++SI) + if (!L->contains(*SI)) + for (BasicBlock::iterator BBI = (*SI)->begin(); + PHINode *phi = dyn_cast<PHINode>(BBI); ++BBI) { + Value *Incoming = phi->getIncomingValueForBlock(*BB); + phi->addIncoming(Incoming, New); + } // Keep track of new headers and latches as we create them, so that // we can insert the proper branches later. @@ -288,24 +294,20 @@ bool llvm::UnrollLoop(Loop *L, unsigned Count, // successor blocks, update them to use the appropriate values computed as the // last iteration of the loop. if (Count != 1) { - SmallPtrSet<PHINode*, 8> Users; - for (Value::use_iterator UI = LatchBlock->use_begin(), - UE = LatchBlock->use_end(); UI != UE; ++UI) - if (PHINode *phi = dyn_cast<PHINode>(*UI)) - Users.insert(phi); - BasicBlock *LastIterationBB = cast<BasicBlock>(LastValueMap[LatchBlock]); - for (SmallPtrSet<PHINode*,8>::iterator SI = Users.begin(), SE = Users.end(); + for (succ_iterator SI = succ_begin(LatchBlock), SE = succ_end(LatchBlock); SI != SE; ++SI) { - PHINode *PN = *SI; - Value *InVal = PN->removeIncomingValue(LatchBlock, false); - // If this value was defined in the loop, take the value defined by the - // last iteration of the loop. - if (Instruction *InValI = dyn_cast<Instruction>(InVal)) { - if (L->contains(InValI)) - InVal = LastValueMap[InVal]; + for (BasicBlock::iterator BBI = (*SI)->begin(); + PHINode *PN = dyn_cast<PHINode>(BBI); ++BBI) { + Value *InVal = PN->removeIncomingValue(LatchBlock, false); + // If this value was defined in the loop, take the value defined by the + // last iteration of the loop. + if (Instruction *InValI = dyn_cast<Instruction>(InVal)) { + if (L->contains(InValI)) + InVal = LastValueMap[InVal]; + } + PN->addIncoming(InVal, LastIterationBB); } - PN->addIncoming(InVal, LastIterationBB); } } @@ -352,11 +354,16 @@ bool llvm::UnrollLoop(Loop *L, unsigned Count, // Replace the conditional branch with an unconditional one. BranchInst::Create(Dest, Term); Term->eraseFromParent(); - // Merge adjacent basic blocks, if possible. - if (BasicBlock *Fold = FoldBlockIntoPredecessor(Dest, LI)) { + } + } + + // Merge adjacent basic blocks, if possible. + for (unsigned i = 0, e = Latches.size(); i != e; ++i) { + BranchInst *Term = cast<BranchInst>(Latches[i]->getTerminator()); + if (Term->isUnconditional()) { + BasicBlock *Dest = Term->getSuccessor(0); + if (BasicBlock *Fold = FoldBlockIntoPredecessor(Dest, LI)) std::replace(Latches.begin(), Latches.end(), Dest, Fold); - std::replace(Headers.begin(), Headers.end(), Dest, Fold); - } } } diff --git a/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/lib/Transforms/Utils/PromoteMemoryToRegister.cpp index a1736b9..32d1dcc 100644 --- a/lib/Transforms/Utils/PromoteMemoryToRegister.cpp +++ b/lib/Transforms/Utils/PromoteMemoryToRegister.cpp @@ -73,6 +73,22 @@ struct DenseMapInfo<std::pair<BasicBlock*, unsigned> > { }; } +/// onlyUsedByLifetimeMarkers - Return true if the only users of this pointer +/// are lifetime markers. +/// +static bool onlyUsedByLifetimeMarkers(const Value *V) { + for (Value::const_use_iterator UI = V->use_begin(), UE = V->use_end(); + UI != UE; ++UI) { + const IntrinsicInst *II = dyn_cast<IntrinsicInst>(*UI); + if (!II) return false; + + if (II->getIntrinsicID() != Intrinsic::lifetime_start && + II->getIntrinsicID() != Intrinsic::lifetime_end) + return false; + } + return true; +} + /// isAllocaPromotable - Return true if this alloca is legal for promotion. /// This is true if there are only loads and stores to the alloca. /// @@ -92,6 +108,22 @@ bool llvm::isAllocaPromotable(const AllocaInst *AI) { return false; // Don't allow a store OF the AI, only INTO the AI. if (SI->isVolatile()) return false; + } else if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(U)) { + if (II->getIntrinsicID() != Intrinsic::lifetime_start && + II->getIntrinsicID() != Intrinsic::lifetime_end) + return false; + } else if (const BitCastInst *BCI = dyn_cast<BitCastInst>(U)) { + if (BCI->getType() != Type::getInt8PtrTy(U->getContext())) + return false; + if (!onlyUsedByLifetimeMarkers(BCI)) + return false; + } else if (const GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(U)) { + if (GEPI->getType() != Type::getInt8PtrTy(U->getContext())) + return false; + if (!GEPI->hasAllZeroIndices()) + return false; + if (!onlyUsedByLifetimeMarkers(GEPI)) + return false; } else { return false; } @@ -335,6 +367,31 @@ namespace { }; } // end of anonymous namespace +static void removeLifetimeIntrinsicUsers(AllocaInst *AI) { + // Knowing that this alloca is promotable, we know that it's safe to kill all + // instructions except for load and store. + + for (Value::use_iterator UI = AI->use_begin(), UE = AI->use_end(); + UI != UE;) { + Instruction *I = cast<Instruction>(*UI); + ++UI; + if (isa<LoadInst>(I) || isa<StoreInst>(I)) + continue; + + if (!I->getType()->isVoidTy()) { + // The only users of this bitcast/GEP instruction are lifetime intrinsics. + // Follow the use/def chain to erase them now instead of leaving it for + // dead code elimination later. + for (Value::use_iterator UI = I->use_begin(), UE = I->use_end(); + UI != UE;) { + Instruction *Inst = cast<Instruction>(*UI); + ++UI; + Inst->eraseFromParent(); + } + } + I->eraseFromParent(); + } +} void PromoteMem2Reg::run() { Function &F = *DT.getRoot()->getParent(); @@ -353,6 +410,8 @@ void PromoteMem2Reg::run() { assert(AI->getParent()->getParent() == &F && "All allocas should be in the same function, which is same as DF!"); + removeLifetimeIntrinsicUsers(AI); + if (AI->use_empty()) { // If there are no uses of the alloca, just delete it now. if (AST) AST->deleteValue(AI); diff --git a/lib/Transforms/Utils/SimplifyCFG.cpp b/lib/Transforms/Utils/SimplifyCFG.cpp index 6df846c..7b93b4a 100644 --- a/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/lib/Transforms/Utils/SimplifyCFG.cpp @@ -2450,6 +2450,77 @@ static bool EliminateDeadSwitchCases(SwitchInst *SI) { return !DeadCases.empty(); } +/// FindPHIForConditionForwarding - If BB would be eligible for simplification +/// by TryToSimplifyUncondBranchFromEmptyBlock (i.e. it is empty and terminated +/// by an unconditional branch), look at the phi node for BB in the successor +/// block and see if the incoming value is equal to CaseValue. If so, return +/// the phi node, and set PhiIndex to BB's index in the phi node. +static PHINode *FindPHIForConditionForwarding(ConstantInt *CaseValue, + BasicBlock *BB, + int *PhiIndex) { + if (BB->getFirstNonPHIOrDbg() != BB->getTerminator()) + return NULL; // BB must be empty to be a candidate for simplification. + if (!BB->getSinglePredecessor()) + return NULL; // BB must be dominated by the switch. + + BranchInst *Branch = dyn_cast<BranchInst>(BB->getTerminator()); + if (!Branch || !Branch->isUnconditional()) + return NULL; // Terminator must be unconditional branch. + + BasicBlock *Succ = Branch->getSuccessor(0); + + BasicBlock::iterator I = Succ->begin(); + while (PHINode *PHI = dyn_cast<PHINode>(I++)) { + int Idx = PHI->getBasicBlockIndex(BB); + assert(Idx >= 0 && "PHI has no entry for predecessor?"); + + Value *InValue = PHI->getIncomingValue(Idx); + if (InValue != CaseValue) continue; + + *PhiIndex = Idx; + return PHI; + } + + return NULL; +} + +/// ForwardSwitchConditionToPHI - Try to forward the condition of a switch +/// instruction to a phi node dominated by the switch, if that would mean that +/// some of the destination blocks of the switch can be folded away. +/// Returns true if a change is made. +static bool ForwardSwitchConditionToPHI(SwitchInst *SI) { + typedef DenseMap<PHINode*, SmallVector<int,4> > ForwardingNodesMap; + ForwardingNodesMap ForwardingNodes; + + for (unsigned I = 1; I < SI->getNumCases(); ++I) { // 0 is the default case. + ConstantInt *CaseValue = SI->getCaseValue(I); + BasicBlock *CaseDest = SI->getSuccessor(I); + + int PhiIndex; + PHINode *PHI = FindPHIForConditionForwarding(CaseValue, CaseDest, + &PhiIndex); + if (!PHI) continue; + + ForwardingNodes[PHI].push_back(PhiIndex); + } + + bool Changed = false; + + for (ForwardingNodesMap::iterator I = ForwardingNodes.begin(), + E = ForwardingNodes.end(); I != E; ++I) { + PHINode *Phi = I->first; + SmallVector<int,4> &Indexes = I->second; + + if (Indexes.size() < 2) continue; + + for (size_t I = 0, E = Indexes.size(); I != E; ++I) + Phi->setIncomingValue(Indexes[I], SI->getCondition()); + Changed = true; + } + + return Changed; +} + bool SimplifyCFGOpt::SimplifySwitch(SwitchInst *SI, IRBuilder<> &Builder) { // If this switch is too complex to want to look at, ignore it. if (!isValueEqualityComparison(SI)) @@ -2486,6 +2557,9 @@ bool SimplifyCFGOpt::SimplifySwitch(SwitchInst *SI, IRBuilder<> &Builder) { if (EliminateDeadSwitchCases(SI)) return SimplifyCFG(BB) | true; + if (ForwardSwitchConditionToPHI(SI)) + return SimplifyCFG(BB) | true; + return false; } diff --git a/lib/Transforms/Utils/ValueMapper.cpp b/lib/Transforms/Utils/ValueMapper.cpp index a73bf04..de6cbdc 100644 --- a/lib/Transforms/Utils/ValueMapper.cpp +++ b/lib/Transforms/Utils/ValueMapper.cpp @@ -16,6 +16,7 @@ #include "llvm/Type.h" #include "llvm/Constants.h" #include "llvm/Function.h" +#include "llvm/Instructions.h" #include "llvm/Metadata.h" #include "llvm/ADT/SmallVector.h" using namespace llvm; @@ -128,6 +129,19 @@ void llvm::RemapInstruction(Instruction *I, ValueToValueMapTy &VMap, "Referenced value not in value map!"); } + // Remap phi nodes' incoming blocks. + if (PHINode *PN = dyn_cast<PHINode>(I)) { + for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { + Value *V = MapValue(PN->getIncomingBlock(i), VMap, Flags); + // If we aren't ignoring missing entries, assert that something happened. + if (V != 0) + PN->setIncomingBlock(i, cast<BasicBlock>(V)); + else + assert((Flags & RF_IgnoreMissingEntries) && + "Referenced block not in value map!"); + } + } + // Remap attached metadata. SmallVector<std::pair<unsigned, MDNode *>, 4> MDs; I->getAllMetadata(MDs); |