From 6173ed95daf2f209fe3883faee45967e4800ae75 Mon Sep 17 00:00:00 2001 From: Hal Finkel Date: Sat, 31 Mar 2012 03:38:40 +0000 Subject: Correctly vectorize powi. The powi intrinsic requires special handling because it always takes a single integer power regardless of the result type. As a result, we can vectorize only if the powers are equal. Fixes PR12364. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@153797 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Transforms/Vectorize/BBVectorize.cpp | 44 ++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 11 deletions(-) (limited to 'lib/Transforms/Vectorize') diff --git a/lib/Transforms/Vectorize/BBVectorize.cpp b/lib/Transforms/Vectorize/BBVectorize.cpp index 32eec79..28eb634 100644 --- a/lib/Transforms/Vectorize/BBVectorize.cpp +++ b/lib/Transforms/Vectorize/BBVectorize.cpp @@ -647,6 +647,20 @@ namespace { // FIXME: We may want to vectorize non-constant shuffles also. } + // The powi intrinsic is special because only the first argument is + // vectorized, the second arguments must be equal. + CallInst *CI = dyn_cast(I); + Function *FI; + if (CI && (FI = CI->getCalledFunction()) && + FI->getIntrinsicID() == Intrinsic::powi) { + + Value *A1I = CI->getArgOperand(1), + *A1J = cast(J)->getArgOperand(1); + const SCEV *A1ISCEV = SE->getSCEV(A1I), + *A1JSCEV = SE->getSCEV(A1J); + return (A1ISCEV == A1JSCEV); + } + return true; } @@ -1518,19 +1532,27 @@ namespace { ReplacedOperands[o] = getReplacementPointerInput(Context, I, J, o, FlipMemInputs); continue; - } else if (isa(I) && o == NumOperands-1) { + } else if (isa(I)) { Function *F = cast(I)->getCalledFunction(); unsigned IID = F->getIntrinsicID(); - BasicBlock &BB = *I->getParent(); - - Module *M = BB.getParent()->getParent(); - Type *ArgType = I->getType(); - Type *VArgType = getVecTypeForPair(ArgType); - - // FIXME: is it safe to do this here? - ReplacedOperands[o] = Intrinsic::getDeclaration(M, - (Intrinsic::ID) IID, VArgType); - continue; + if (o == NumOperands-1) { + BasicBlock &BB = *I->getParent(); + + Module *M = BB.getParent()->getParent(); + Type *ArgType = I->getType(); + Type *VArgType = getVecTypeForPair(ArgType); + + // FIXME: is it safe to do this here? + ReplacedOperands[o] = Intrinsic::getDeclaration(M, + (Intrinsic::ID) IID, VArgType); + continue; + } else if (IID == Intrinsic::powi && o == 1) { + // The second argument of powi is a single integer and we've already + // checked that both arguments are equal. As a result, we just keep + // I's second argument. + ReplacedOperands[o] = I->getOperand(o); + continue; + } } else if (isa(I) && o == NumOperands-1) { ReplacedOperands[o] = getReplacementShuffleMask(Context, I, J); continue; -- cgit v1.1 From 87825e7970a361ce5a8bab19bc880ff7f6242ca2 Mon Sep 17 00:00:00 2001 From: Hongbin Zheng Date: Thu, 5 Apr 2012 08:05:16 +0000 Subject: Add the function "vectorizeBasicBlock" which allow users vectorize a BasicBlock in other passes, e.g. we can call vectorizeBasicBlock in the loop unroll pass right after the loop is unrolled. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@154089 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Transforms/Vectorize/BBVectorize.cpp | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) (limited to 'lib/Transforms/Vectorize') diff --git a/lib/Transforms/Vectorize/BBVectorize.cpp b/lib/Transforms/Vectorize/BBVectorize.cpp index 28eb634..5abb242 100644 --- a/lib/Transforms/Vectorize/BBVectorize.cpp +++ b/lib/Transforms/Vectorize/BBVectorize.cpp @@ -144,6 +144,12 @@ namespace { initializeBBVectorizePass(*PassRegistry::getPassRegistry()); } + BBVectorize(Pass *P) : BasicBlockPass(ID) { + AA = &P->getAnalysis(); + SE = &P->getAnalysis(); + TD = P->getAnalysisIfAvailable(); + } + typedef std::pair ValuePair; typedef std::pair ValuePairWithDepth; typedef std::pair VPPair; // A ValuePair pair @@ -280,11 +286,7 @@ namespace { Instruction *&InsertionPt, Instruction *I, Instruction *J); - virtual bool runOnBasicBlock(BasicBlock &BB) { - AA = &getAnalysis(); - SE = &getAnalysis(); - TD = getAnalysisIfAvailable(); - + bool vectorizeBB(BasicBlock &BB) { bool changed = false; // Iterate a sufficient number of times to merge types of size 1 bit, // then 2 bits, then 4, etc. up to half of the target vector width of the @@ -304,6 +306,14 @@ namespace { return changed; } + virtual bool runOnBasicBlock(BasicBlock &BB) { + AA = &getAnalysis(); + SE = &getAnalysis(); + TD = getAnalysisIfAvailable(); + + return vectorizeBB(BB); + } + virtual void getAnalysisUsage(AnalysisUsage &AU) const { BasicBlockPass::getAnalysisUsage(AU); AU.addRequired(); @@ -1861,3 +1871,7 @@ BasicBlockPass *llvm::createBBVectorizePass() { return new BBVectorize(); } +bool llvm::vectorizeBasicBlock(Pass *P, BasicBlock &BB) { + BBVectorize BBVectorizer(P); + return BBVectorizer.vectorizeBB(BB); +} -- cgit v1.1 From bef377b7d7ce31edb40c87f8786d1b7bb6cdd6b1 Mon Sep 17 00:00:00 2001 From: Hongbin Zheng Date: Thu, 5 Apr 2012 15:46:55 +0000 Subject: Introduce the VectorizeConfig class, with which we can control the behavior of the BBVectorizePass without using command line option. As pointed out by Hal, we can ask the TargetLoweringInfo for the architecture specific VectorizeConfig to perform vectorizing with architecture specific information. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@154096 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Transforms/Vectorize/BBVectorize.cpp | 92 +++++++++++++++++++++----------- 1 file changed, 60 insertions(+), 32 deletions(-) (limited to 'lib/Transforms/Vectorize') diff --git a/lib/Transforms/Vectorize/BBVectorize.cpp b/lib/Transforms/Vectorize/BBVectorize.cpp index 5abb242..7d5bb31 100644 --- a/lib/Transforms/Vectorize/BBVectorize.cpp +++ b/lib/Transforms/Vectorize/BBVectorize.cpp @@ -140,11 +140,16 @@ STATISTIC(NumFusedOps, "Number of operations fused by bb-vectorize"); namespace { struct BBVectorize : public BasicBlockPass { static char ID; // Pass identification, replacement for typeid - BBVectorize() : BasicBlockPass(ID) { + + VectorizeConfig Config; + + BBVectorize(const VectorizeConfig &C = VectorizeConfig()) + : BasicBlockPass(ID), Config(C) { initializeBBVectorizePass(*PassRegistry::getPassRegistry()); } - BBVectorize(Pass *P) : BasicBlockPass(ID) { + BBVectorize(Pass *P, const VectorizeConfig &C) + : BasicBlockPass(ID), Config(C) { AA = &P->getAnalysis(); SE = &P->getAnalysis(); TD = P->getAnalysisIfAvailable(); @@ -291,9 +296,10 @@ namespace { // Iterate a sufficient number of times to merge types of size 1 bit, // then 2 bits, then 4, etc. up to half of the target vector width of the // target vector register. - for (unsigned v = 2, n = 1; v <= VectorBits && (!MaxIter || n <= MaxIter); + for (unsigned v = 2, n = 1; + v <= Config.VectorBits && (!Config.MaxIter || n <= Config.MaxIter); v *= 2, ++n) { - DEBUG(dbgs() << "BBV: fusing loop #" << n << + DEBUG(dbgs() << "BBV: fusing loop #" << n << " for " << BB.getName() << " in " << BB.getParent()->getName() << "...\n"); if (vectorizePairs(BB)) @@ -343,7 +349,7 @@ namespace { // candidate chains where longer chains are considered to be better. // Note: when this function returns 0, the resulting instructions are // not actually fused. - static inline size_t getDepthFactor(Value *V) { + inline size_t getDepthFactor(Value *V) { // InsertElement and ExtractElement have a depth factor of zero. This is // for two reasons: First, they cannot be usefully fused. Second, because // the pass generates a lot of these, they can confuse the simple metric @@ -357,8 +363,8 @@ namespace { // Give a load or store half of the required depth so that load/store // pairs will vectorize. - if (!NoMemOpBoost && (isa(V) || isa(V))) - return ReqChainDepth/2; + if (!Config.NoMemOpBoost && (isa(V) || isa(V))) + return Config.ReqChainDepth/2; return 1; } @@ -431,9 +437,9 @@ namespace { case Intrinsic::exp: case Intrinsic::exp2: case Intrinsic::pow: - return !NoMath; + return !Config.NoMath; case Intrinsic::fma: - return !NoFMA; + return !Config.NoFMA; } } @@ -527,16 +533,16 @@ namespace { } else if (LoadInst *L = dyn_cast(I)) { // Vectorize simple loads if possbile: IsSimpleLoadStore = L->isSimple(); - if (!IsSimpleLoadStore || NoMemOps) + if (!IsSimpleLoadStore || Config.NoMemOps) return false; } else if (StoreInst *S = dyn_cast(I)) { // Vectorize simple stores if possbile: IsSimpleLoadStore = S->isSimple(); - if (!IsSimpleLoadStore || NoMemOps) + if (!IsSimpleLoadStore || Config.NoMemOps) return false; } else if (CastInst *C = dyn_cast(I)) { // We can vectorize casts, but not casts of pointer types, etc. - if (NoCasts) + if (Config.NoCasts) return false; Type *SrcTy = C->getSrcTy(); @@ -576,14 +582,14 @@ namespace { !(VectorType::isValidElementType(T2) || T2->isVectorTy())) return false; - if (NoInts && (T1->isIntOrIntVectorTy() || T2->isIntOrIntVectorTy())) + if (Config.NoInts && (T1->isIntOrIntVectorTy() || T2->isIntOrIntVectorTy())) return false; - if (NoFloats && (T1->isFPOrFPVectorTy() || T2->isFPOrFPVectorTy())) + if (Config.NoFloats && (T1->isFPOrFPVectorTy() || T2->isFPOrFPVectorTy())) return false; - if (T1->getPrimitiveSizeInBits() > VectorBits/2 || - T2->getPrimitiveSizeInBits() > VectorBits/2) + if (T1->getPrimitiveSizeInBits() > Config.VectorBits/2 || + T2->getPrimitiveSizeInBits() > Config.VectorBits/2) return false; return true; @@ -611,7 +617,7 @@ namespace { LI->isVolatile() != LJ->isVolatile() || LI->getOrdering() != LJ->getOrdering() || LI->getSynchScope() != LJ->getSynchScope()) - return false; + return false; } else if ((SI = dyn_cast(I)) && (SJ = dyn_cast(J))) { if (SI->getValueOperand()->getType() != SJ->getValueOperand()->getType() || @@ -632,7 +638,7 @@ namespace { int64_t OffsetInElmts = 0; if (getPairPtrInfo(I, J, IPtr, JPtr, IAlignment, JAlignment, OffsetInElmts) && abs64(OffsetInElmts) == 1) { - if (AlignedOnly) { + if (Config.AlignedOnly) { Type *aType = isa(I) ? cast(I)->getValueOperand()->getType() : I->getType(); // An aligned load or store is possible only if the instruction @@ -753,12 +759,12 @@ namespace { AliasSetTracker WriteSet(*AA); bool JAfterStart = IAfterStart; BasicBlock::iterator J = llvm::next(I); - for (unsigned ss = 0; J != E && ss <= SearchLimit; ++J, ++ss) { + for (unsigned ss = 0; J != E && ss <= Config.SearchLimit; ++J, ++ss) { if (J == Start) JAfterStart = true; // Determine if J uses I, if so, exit the loop. - bool UsesI = trackUsesOfI(Users, WriteSet, I, J, !FastDep); - if (FastDep) { + bool UsesI = trackUsesOfI(Users, WriteSet, I, J, !Config.FastDep); + if (Config.FastDep) { // Note: For this heuristic to be effective, independent operations // must tend to be intermixed. This is likely to be true from some // kinds of grouped loop unrolling (but not the generic LLVM pass), @@ -796,7 +802,7 @@ namespace { // If we have already found too many pairs, break here and this function // will be called again starting after the last instruction selected // during this invocation. - if (PairableInsts.size() >= MaxInsts) { + if (PairableInsts.size() >= Config.MaxInsts) { ShouldContinue = true; break; } @@ -841,7 +847,7 @@ namespace { ConnectedPairs.insert(VPPair(P, ValuePair(*J, *I))); } - if (SplatBreaksChain) continue; + if (Config.SplatBreaksChain) continue; // Look for cases where just the first value in the pair is used by // both members of another pair (splatting). for (Value::use_iterator J = P.first->use_begin(); J != E; ++J) { @@ -850,7 +856,7 @@ namespace { } } - if (SplatBreaksChain) return; + if (Config.SplatBreaksChain) return; // Look for cases where just the second value in the pair is used by // both members of another pair (splatting). for (Value::use_iterator I = P.second->use_begin(), @@ -1280,7 +1286,7 @@ namespace { << *J->first << " <-> " << *J->second << "} of depth " << MaxDepth << " and size " << PrunedTree.size() << " (effective size: " << EffSize << ")\n"); - if (MaxDepth >= ReqChainDepth && EffSize > BestEffSize) { + if (MaxDepth >= Config.ReqChainDepth && EffSize > BestEffSize) { BestMaxDepth = MaxDepth; BestEffSize = EffSize; BestTree = PrunedTree; @@ -1296,7 +1302,8 @@ namespace { std::multimap &ConnectedPairs, DenseSet &PairableInstUsers, DenseMap& ChosenPairs) { - bool UseCycleCheck = CandidatePairs.size() <= MaxCandPairsForCycleCheck; + bool UseCycleCheck = + CandidatePairs.size() <= Config.MaxCandPairsForCycleCheck; std::multimap PairableInstUserMap; for (std::vector::iterator I = PairableInsts.begin(), E = PairableInsts.end(); I != E; ++I) { @@ -1547,11 +1554,11 @@ namespace { unsigned IID = F->getIntrinsicID(); if (o == NumOperands-1) { BasicBlock &BB = *I->getParent(); - + Module *M = BB.getParent()->getParent(); Type *ArgType = I->getType(); Type *VArgType = getVecTypeForPair(ArgType); - + // FIXME: is it safe to do this here? ReplacedOperands[o] = Intrinsic::getDeclaration(M, (Intrinsic::ID) IID, VArgType); @@ -1867,11 +1874,32 @@ INITIALIZE_AG_DEPENDENCY(AliasAnalysis) INITIALIZE_PASS_DEPENDENCY(ScalarEvolution) INITIALIZE_PASS_END(BBVectorize, BBV_NAME, bb_vectorize_name, false, false) -BasicBlockPass *llvm::createBBVectorizePass() { - return new BBVectorize(); +BasicBlockPass *llvm::createBBVectorizePass(const VectorizeConfig &C) { + return new BBVectorize(C); } -bool llvm::vectorizeBasicBlock(Pass *P, BasicBlock &BB) { - BBVectorize BBVectorizer(P); +bool +llvm::vectorizeBasicBlock(Pass *P, BasicBlock &BB, const VectorizeConfig &C) { + BBVectorize BBVectorizer(P, C); return BBVectorizer.vectorizeBB(BB); } + +//===----------------------------------------------------------------------===// +VectorizeConfig::VectorizeConfig() { + VectorBits = ::VectorBits; + NoInts = ::NoInts; + NoFloats = ::NoFloats; + NoCasts = ::NoCasts; + NoMath = ::NoMath; + NoFMA = ::NoFMA; + NoMemOps = ::NoMemOps; + AlignedOnly = ::AlignedOnly; + ReqChainDepth= ::ReqChainDepth; + SearchLimit = ::SearchLimit; + MaxCandPairsForCycleCheck = ::MaxCandPairsForCycleCheck; + SplatBreaksChain = ::SplatBreaksChain; + MaxInsts = ::MaxInsts; + MaxIter = ::MaxIter; + NoMemOpBoost = ::NoMemOpBoost; + FastDep = ::FastDep; +} -- cgit v1.1 From 940371bc65570ec0add1ede4f4d9f0a41ba25e09 Mon Sep 17 00:00:00 2001 From: Hongbin Zheng Date: Thu, 5 Apr 2012 16:07:49 +0000 Subject: BBVectorize: Add the const modifier to the VectorizeConfig because we won't modify it. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@154098 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Transforms/Vectorize/BBVectorize.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/Transforms/Vectorize') diff --git a/lib/Transforms/Vectorize/BBVectorize.cpp b/lib/Transforms/Vectorize/BBVectorize.cpp index 7d5bb31..82be697 100644 --- a/lib/Transforms/Vectorize/BBVectorize.cpp +++ b/lib/Transforms/Vectorize/BBVectorize.cpp @@ -141,7 +141,7 @@ namespace { struct BBVectorize : public BasicBlockPass { static char ID; // Pass identification, replacement for typeid - VectorizeConfig Config; + const VectorizeConfig Config; BBVectorize(const VectorizeConfig &C = VectorizeConfig()) : BasicBlockPass(ID), Config(C) { -- cgit v1.1 From 86312cc15f29ce2bbd9647b94862e068045280c3 Mon Sep 17 00:00:00 2001 From: Hongbin Zheng Date: Sat, 7 Apr 2012 03:56:23 +0000 Subject: Refactor: Use positive field names in VectorizeConfig. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@154249 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Transforms/Vectorize/BBVectorize.cpp | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) (limited to 'lib/Transforms/Vectorize') diff --git a/lib/Transforms/Vectorize/BBVectorize.cpp b/lib/Transforms/Vectorize/BBVectorize.cpp index 82be697..286b54f 100644 --- a/lib/Transforms/Vectorize/BBVectorize.cpp +++ b/lib/Transforms/Vectorize/BBVectorize.cpp @@ -437,9 +437,9 @@ namespace { case Intrinsic::exp: case Intrinsic::exp2: case Intrinsic::pow: - return !Config.NoMath; + return Config.VectorizeMath; case Intrinsic::fma: - return !Config.NoFMA; + return Config.VectorizeFMA; } } @@ -533,16 +533,16 @@ namespace { } else if (LoadInst *L = dyn_cast(I)) { // Vectorize simple loads if possbile: IsSimpleLoadStore = L->isSimple(); - if (!IsSimpleLoadStore || Config.NoMemOps) + if (!IsSimpleLoadStore || !Config.VectorizeMemOps) return false; } else if (StoreInst *S = dyn_cast(I)) { // Vectorize simple stores if possbile: IsSimpleLoadStore = S->isSimple(); - if (!IsSimpleLoadStore || Config.NoMemOps) + if (!IsSimpleLoadStore || !Config.VectorizeMemOps) return false; } else if (CastInst *C = dyn_cast(I)) { // We can vectorize casts, but not casts of pointer types, etc. - if (Config.NoCasts) + if (!Config.VectorizeCasts) return false; Type *SrcTy = C->getSrcTy(); @@ -582,10 +582,12 @@ namespace { !(VectorType::isValidElementType(T2) || T2->isVectorTy())) return false; - if (Config.NoInts && (T1->isIntOrIntVectorTy() || T2->isIntOrIntVectorTy())) + if (!Config.VectorizeInts + && (T1->isIntOrIntVectorTy() || T2->isIntOrIntVectorTy())) return false; - if (Config.NoFloats && (T1->isFPOrFPVectorTy() || T2->isFPOrFPVectorTy())) + if (!Config.VectorizeFloats + && (T1->isFPOrFPVectorTy() || T2->isFPOrFPVectorTy())) return false; if (T1->getPrimitiveSizeInBits() > Config.VectorBits/2 || @@ -1887,12 +1889,12 @@ llvm::vectorizeBasicBlock(Pass *P, BasicBlock &BB, const VectorizeConfig &C) { //===----------------------------------------------------------------------===// VectorizeConfig::VectorizeConfig() { VectorBits = ::VectorBits; - NoInts = ::NoInts; - NoFloats = ::NoFloats; - NoCasts = ::NoCasts; - NoMath = ::NoMath; - NoFMA = ::NoFMA; - NoMemOps = ::NoMemOps; + VectorizeInts = !::NoInts; + VectorizeFloats = !::NoFloats; + VectorizeCasts = !::NoCasts; + VectorizeMath = !::NoMath; + VectorizeFMA = !::NoFMA; + VectorizeMemOps = !::NoMemOps; AlignedOnly = ::AlignedOnly; ReqChainDepth= ::ReqChainDepth; SearchLimit = ::SearchLimit; -- cgit v1.1 From fc3665c87519850f629c9565535e3be447e10add Mon Sep 17 00:00:00 2001 From: Hal Finkel Date: Fri, 13 Apr 2012 20:45:45 +0000 Subject: Add support to BBVectorize for vectorizing selects. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@154700 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Transforms/Vectorize/BBVectorize.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib/Transforms/Vectorize') diff --git a/lib/Transforms/Vectorize/BBVectorize.cpp b/lib/Transforms/Vectorize/BBVectorize.cpp index 286b54f..b2f8c8f 100644 --- a/lib/Transforms/Vectorize/BBVectorize.cpp +++ b/lib/Transforms/Vectorize/BBVectorize.cpp @@ -96,6 +96,10 @@ NoFMA("bb-vectorize-no-fma", cl::init(false), cl::Hidden, cl::desc("Don't try to vectorize the fused-multiply-add intrinsic")); static cl::opt +NoSelect("bb-vectorize-no-select", cl::init(false), cl::Hidden, + cl::desc("Don't try to vectorize select instructions")); + +static cl::opt NoMemOps("bb-vectorize-no-mem-ops", cl::init(false), cl::Hidden, cl::desc("Don't try to vectorize loads and stores")); @@ -552,6 +556,9 @@ namespace { Type *DestTy = C->getDestTy(); if (!DestTy->isSingleValueType() || DestTy->isPointerTy()) return false; + } else if (isa(I)) { + if (!Config.VectorizeSelect) + return false; } else if (!(I->isBinaryOp() || isa(I) || isa(I) || isa(I))) { return false; @@ -1894,6 +1901,7 @@ VectorizeConfig::VectorizeConfig() { VectorizeCasts = !::NoCasts; VectorizeMath = !::NoMath; VectorizeFMA = !::NoFMA; + VectorizeSelect = !::NoSelect; VectorizeMemOps = !::NoMemOps; AlignedOnly = ::AlignedOnly; ReqChainDepth= ::ReqChainDepth; -- cgit v1.1 From f3f5a1e6f77a842ccb24cc81766437da5197d712 Mon Sep 17 00:00:00 2001 From: Hal Finkel Date: Sat, 14 Apr 2012 07:32:43 +0000 Subject: Enhance BBVectorize to more-properly handle pointer values and vectorize GEPs. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@154734 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Transforms/Vectorize/BBVectorize.cpp | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) (limited to 'lib/Transforms/Vectorize') diff --git a/lib/Transforms/Vectorize/BBVectorize.cpp b/lib/Transforms/Vectorize/BBVectorize.cpp index b2f8c8f..ba03318 100644 --- a/lib/Transforms/Vectorize/BBVectorize.cpp +++ b/lib/Transforms/Vectorize/BBVectorize.cpp @@ -84,6 +84,10 @@ NoFloats("bb-vectorize-no-floats", cl::init(false), cl::Hidden, cl::desc("Don't try to vectorize floating-point values")); static cl::opt +NoPointers("bb-vectorize-no-pointers", cl::init(false), cl::Hidden, + cl::desc("Don't try to vectorize pointer values")); + +static cl::opt NoCasts("bb-vectorize-no-casts", cl::init(false), cl::Hidden, cl::desc("Don't try to vectorize casting (conversion) operations")); @@ -100,6 +104,10 @@ NoSelect("bb-vectorize-no-select", cl::init(false), cl::Hidden, cl::desc("Don't try to vectorize select instructions")); static cl::opt +NoGEP("bb-vectorize-no-gep", cl::init(false), cl::Hidden, + cl::desc("Don't try to vectorize getelementptr instructions")); + +static cl::opt NoMemOps("bb-vectorize-no-mem-ops", cl::init(false), cl::Hidden, cl::desc("Don't try to vectorize loads and stores")); @@ -550,15 +558,22 @@ namespace { return false; Type *SrcTy = C->getSrcTy(); - if (!SrcTy->isSingleValueType() || SrcTy->isPointerTy()) + if (!SrcTy->isSingleValueType()) return false; Type *DestTy = C->getDestTy(); - if (!DestTy->isSingleValueType() || DestTy->isPointerTy()) + if (!DestTy->isSingleValueType()) return false; } else if (isa(I)) { if (!Config.VectorizeSelect) return false; + } else if (GetElementPtrInst *G = dyn_cast(I)) { + if (!Config.VectorizeGEP) + return false; + + // Currently, vector GEPs exist only with one index. + if (G->getNumIndices() != 1) + return false; } else if (!(I->isBinaryOp() || isa(I) || isa(I) || isa(I))) { return false; @@ -597,6 +612,14 @@ namespace { && (T1->isFPOrFPVectorTy() || T2->isFPOrFPVectorTy())) return false; + if ((!Config.VectorizePointers || TD == 0) + && ((T1->isPointerTy() || + (T1->isVectorTy() && T1->getScalarType()->isPointerTy())) || + (T2->isPointerTy() || + (T2->isVectorTy() && T2->getScalarType()->isPointerTy())) + )) + return false; + if (T1->getPrimitiveSizeInBits() > Config.VectorBits/2 || T2->getPrimitiveSizeInBits() > Config.VectorBits/2) return false; @@ -1898,10 +1921,12 @@ VectorizeConfig::VectorizeConfig() { VectorBits = ::VectorBits; VectorizeInts = !::NoInts; VectorizeFloats = !::NoFloats; + VectorizePointers = !::NoPointers; VectorizeCasts = !::NoCasts; VectorizeMath = !::NoMath; VectorizeFMA = !::NoFMA; VectorizeSelect = !::NoSelect; + VectorizeGEP = !::NoGEP; VectorizeMemOps = !::NoMemOps; AlignedOnly = ::AlignedOnly; ReqChainDepth= ::ReqChainDepth; -- cgit v1.1 From bba23ed672c4cedd61a302497f45bf6f53fec7b2 Mon Sep 17 00:00:00 2001 From: Hal Finkel Date: Sat, 14 Apr 2012 07:32:50 +0000 Subject: Fix an error in BBVectorize important for vectorizing pointer types. When vectorizing pointer types it is important to realize that potential pairs cannot be connected via the address pointer argument of a load or store. This is because even after vectorization, the address is still a scalar because the address of the higher half of the pair is implicit from the address of the lower half (it need not be, and should not be, explicitly computed). git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@154735 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Transforms/Vectorize/BBVectorize.cpp | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'lib/Transforms/Vectorize') diff --git a/lib/Transforms/Vectorize/BBVectorize.cpp b/lib/Transforms/Vectorize/BBVectorize.cpp index ba03318..290d82d 100644 --- a/lib/Transforms/Vectorize/BBVectorize.cpp +++ b/lib/Transforms/Vectorize/BBVectorize.cpp @@ -858,16 +858,33 @@ namespace { std::vector &PairableInsts, std::multimap &ConnectedPairs, ValuePair P) { + StoreInst *SI, *SJ; + // For each possible pairing for this variable, look at the uses of // the first value... for (Value::use_iterator I = P.first->use_begin(), E = P.first->use_end(); I != E; ++I) { + if (isa(*I)) { + // A pair cannot be connected to a load because the load only takes one + // operand (the address) and it is a scalar even after vectorization. + continue; + } else if ((SI = dyn_cast(*I)) && + P.first == SI->getPointerOperand()) { + // Similarly, a pair cannot be connected to a store through its + // pointer operand. + continue; + } + VPIteratorPair IPairRange = CandidatePairs.equal_range(*I); // For each use of the first variable, look for uses of the second // variable... for (Value::use_iterator J = P.second->use_begin(), E2 = P.second->use_end(); J != E2; ++J) { + if ((SJ = dyn_cast(*J)) && + P.second == SJ->getPointerOperand()) + continue; + VPIteratorPair JPairRange = CandidatePairs.equal_range(*J); // Look for : @@ -883,6 +900,10 @@ namespace { // Look for cases where just the first value in the pair is used by // both members of another pair (splatting). for (Value::use_iterator J = P.first->use_begin(); J != E; ++J) { + if ((SJ = dyn_cast(*J)) && + P.first == SJ->getPointerOperand()) + continue; + if (isSecondInIteratorPair(*J, IPairRange)) ConnectedPairs.insert(VPPair(P, ValuePair(*I, *J))); } @@ -893,9 +914,19 @@ namespace { // both members of another pair (splatting). for (Value::use_iterator I = P.second->use_begin(), E = P.second->use_end(); I != E; ++I) { + if (isa(*I)) + continue; + else if ((SI = dyn_cast(*I)) && + P.second == SI->getPointerOperand()) + continue; + VPIteratorPair IPairRange = CandidatePairs.equal_range(*I); for (Value::use_iterator J = P.second->use_begin(); J != E; ++J) { + if ((SJ = dyn_cast(*J)) && + P.second == SJ->getPointerOperand()) + continue; + if (isSecondInIteratorPair(*J, IPairRange)) ConnectedPairs.insert(VPPair(P, ValuePair(*I, *J))); } -- cgit v1.1 From 831a6e86d50414aa715846707e1f9f54f98513cb Mon Sep 17 00:00:00 2001 From: Hal Finkel Date: Mon, 16 Apr 2012 03:49:42 +0000 Subject: Simplify checking for pointer types in BBVectorize (this change was suggested by Duncan). git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@154787 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Transforms/Vectorize/BBVectorize.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'lib/Transforms/Vectorize') diff --git a/lib/Transforms/Vectorize/BBVectorize.cpp b/lib/Transforms/Vectorize/BBVectorize.cpp index 290d82d..a247e2d 100644 --- a/lib/Transforms/Vectorize/BBVectorize.cpp +++ b/lib/Transforms/Vectorize/BBVectorize.cpp @@ -613,11 +613,8 @@ namespace { return false; if ((!Config.VectorizePointers || TD == 0) - && ((T1->isPointerTy() || - (T1->isVectorTy() && T1->getScalarType()->isPointerTy())) || - (T2->isPointerTy() || - (T2->isVectorTy() && T2->getScalarType()->isPointerTy())) - )) + && (T1->getScalarType()->isPointerTy() || + T2->getScalarType()->isPointerTy())) return false; if (T1->getPrimitiveSizeInBits() > Config.VectorBits/2 || -- cgit v1.1 From 05bc5087a25bbcf59936d71ebfc878b545ef3e5c Mon Sep 17 00:00:00 2001 From: Hal Finkel Date: Mon, 16 Apr 2012 12:39:17 +0000 Subject: Fix style violation in BBVectorize (pointed out by Bill Wendling) git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@154810 91177308-0d34-0410-b5e6-96231b3b80d8 --- lib/Transforms/Vectorize/BBVectorize.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib/Transforms/Vectorize') diff --git a/lib/Transforms/Vectorize/BBVectorize.cpp b/lib/Transforms/Vectorize/BBVectorize.cpp index a247e2d..9d62306 100644 --- a/lib/Transforms/Vectorize/BBVectorize.cpp +++ b/lib/Transforms/Vectorize/BBVectorize.cpp @@ -612,9 +612,9 @@ namespace { && (T1->isFPOrFPVectorTy() || T2->isFPOrFPVectorTy())) return false; - if ((!Config.VectorizePointers || TD == 0) - && (T1->getScalarType()->isPointerTy() || - T2->getScalarType()->isPointerTy())) + if ((!Config.VectorizePointers || TD == 0) && + (T1->getScalarType()->isPointerTy() || + T2->getScalarType()->isPointerTy())) return false; if (T1->getPrimitiveSizeInBits() > Config.VectorBits/2 || -- cgit v1.1