BBVectorize: Don't vectorize vector-manipulation chains

Don't choose a vectorization plan containing only shuffles and
vector inserts/extracts. Due to inperfections in the cost model,
these can lead to infinite recusion.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@167811 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 17 insertions, 0 deletions

diff --git a/lib/Transforms/Vectorize/BBVectorize.cpp b/lib/Transforms/Vectorize/BBVectorize.cpp
index 32a37ba..a2c3ad7 100644
--- a/lib/Transforms/Vectorize/BBVectorize.cpp
+++ b/lib/Transforms/Vectorize/BBVectorize.cpp
@@ -1703,10 +1703,20 @@ assert(n < 10 && "hrmm, really?");
         // The set of pairs that have already contributed to the total cost.
         DenseSet<ValuePair> IncomingPairs;
 
+        // If the cost model were perfect, this might not be necessary; but we
+        // need to make sure that we don't get stuck vectorizing our own
+        // shuffle chains.
+        bool HasNontrivialInsts = false;
+
         // The node weights represent the cost savings associated with
         // fusing the pair of instructions.
         for (DenseSet<ValuePair>::iterator S = PrunedTree.begin(),
              E = PrunedTree.end(); S != E; ++S) {
+          if (!isa<ShuffleVectorInst>(S->first) &&
+              !isa<InsertElementInst>(S->first) &&
+              !isa<ExtractElementInst>(S->first))
+            HasNontrivialInsts = true;
+
           bool FlipOrder = false;
 
           if (getDepthFactor(S->first)) {
@@ -1943,6 +1953,13 @@ assert(n < 10 && "hrmm, really?");
             }
           }
         }
+
+        if (!HasNontrivialInsts) {
+          DEBUG(if (DebugPairSelection) dbgs() <<
+                "\tNo non-trivial instructions in tree;"
+                " override to zero effective size\n");
+          EffSize = 0;
+        }
       } else {
         for (DenseSet<ValuePair>::iterator S = PrunedTree.begin(),
              E = PrunedTree.end(); S != E; ++S)