Merge in the following upstream patches to resolve Cortex-A57 crashes.

r214957
r215233
r216455
r216721
r217682
r217689
r217690
r217735

Change-Id: Ia53b88591471325df132caf26e1087510a65ce36

2 files changed, 49 insertions, 41 deletions

diff --git a/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp b/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp
index 195a48e..7742fea 100644
--- a/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp
+++ b/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp
@@ -73,8 +73,6 @@ static bool isMul(MachineInstr *MI) {
   case AArch64::FNMULSrr:
   case AArch64::FMULDrr:
   case AArch64::FNMULDrr:
-
-  case AArch64::FMULv2f32:
     return true;
   default:
     return false;
@@ -92,9 +90,6 @@ static bool isMla(MachineInstr *MI) {
   case AArch64::FMADDDrrr:
   case AArch64::FNMSUBDrrr:
   case AArch64::FNMADDDrrr:
-
-  case AArch64::FMLAv2f32:
-  case AArch64::FMLSv2f32:
     return true;
   default:
     return false;
@@ -139,8 +134,8 @@ private:
   bool colorChain(Chain *G, Color C, MachineBasicBlock &MBB);
   int scavengeRegister(Chain *G, Color C, MachineBasicBlock &MBB);
   void scanInstruction(MachineInstr *MI, unsigned Idx,
-                       std::map<unsigned, Chain*> &Chains,
-                       std::set<Chain*> &ChainSet);
+                       std::map<unsigned, Chain*> &Active,
+                       std::set<std::unique_ptr<Chain>> &AllChains);
   void maybeKillChain(MachineOperand &MO, unsigned Idx,
                       std::map<unsigned, Chain*> &RegChains);
   Color getColor(unsigned Register);
@@ -191,10 +186,10 @@ public:
   /// instruction can be more tricky.
   Color LastColor;
 
-  Chain(MachineInstr *MI, unsigned Idx, Color C) :
-  StartInst(MI), LastInst(MI), KillInst(NULL),
-  StartInstIdx(Idx), LastInstIdx(Idx), KillInstIdx(0),
-  LastColor(C) {
+  Chain(MachineInstr *MI, unsigned Idx, Color C)
+      : StartInst(MI), LastInst(MI), KillInst(nullptr),
+        StartInstIdx(Idx), LastInstIdx(Idx), KillInstIdx(0),
+        LastColor(C) {
     Insts.insert(MI);
   }
 
@@ -204,6 +199,9 @@ public:
     LastInst = MI;
     LastInstIdx = Idx;
     LastColor = C;
+    assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) &&
+           "Chain: broken invariant. A Chain can only be killed after its last "
+           "def");
 
     Insts.insert(MI);
   }
@@ -222,6 +220,9 @@ public:
     KillInst = MI;
     KillInstIdx = Idx;
     KillIsImmutable = Immutable;
+    assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) &&
+           "Chain: broken invariant. A Chain can only be killed after its last "
+           "def");
   }
 
   /// Return the first instruction in the chain.
@@ -232,7 +233,7 @@ public:
   MachineInstr *getKill() const { return KillInst; }
   /// Return an instruction that can be used as an iterator for the end
   /// of the chain. This is the maximum of KillInst (if set) and LastInst.
-  MachineInstr *getEnd() const {
+  MachineBasicBlock::iterator getEnd() const {
     return ++MachineBasicBlock::iterator(KillInst ? KillInst : LastInst);
   }
 
@@ -247,12 +248,12 @@ public:
   }
 
   /// Return true if this chain (StartInst..KillInst) overlaps with Other.
-  bool rangeOverlapsWith(Chain *Other) {
+  bool rangeOverlapsWith(const Chain &Other) const {
     unsigned End = KillInst ? KillInstIdx : LastInstIdx;
-    unsigned OtherEnd = Other->KillInst ?
-      Other->KillInstIdx : Other->LastInstIdx;
+    unsigned OtherEnd = Other.KillInst ?
+      Other.KillInstIdx : Other.LastInstIdx;
 
-    return StartInstIdx <= OtherEnd && Other->StartInstIdx <= End;
+    return StartInstIdx <= OtherEnd && Other.StartInstIdx <= End;
   }
 
   /// Return true if this chain starts before Other.
@@ -317,7 +318,7 @@ bool AArch64A57FPLoadBalancing::runOnBasicBlock(MachineBasicBlock &MBB) {
   // been killed yet. This is keyed by register - all chains can only have one
   // "link" register between each inst in the chain.
   std::map<unsigned, Chain*> ActiveChains;
-  std::set<Chain*> AllChains;
+  std::set<std::unique_ptr<Chain>> AllChains;
   unsigned Idx = 0;
   for (auto &MI : MBB)
     scanInstruction(&MI, Idx++, ActiveChains, AllChains);
@@ -332,15 +333,13 @@ bool AArch64A57FPLoadBalancing::runOnBasicBlock(MachineBasicBlock &MBB) {
   //       range of chains is quite small and they are clustered between loads
   //       and stores.
   EquivalenceClasses<Chain*> EC;
-  for (auto *I : AllChains)
-    EC.insert(I);
+  for (auto &I : AllChains)
+    EC.insert(I.get());
 
-  for (auto *I : AllChains) {
-    for (auto *J : AllChains) {
-      if (I != J && I->rangeOverlapsWith(J))
-        EC.unionSets(I, J);
-    }
-  }
+  for (auto &I : AllChains)
+    for (auto &J : AllChains)
+      if (I != J && I->rangeOverlapsWith(*J))
+        EC.unionSets(I.get(), J.get());
   DEBUG(dbgs() << "Created " << EC.getNumClasses() << " disjoint sets.\n");
 
   // Now we assume that every member of an equivalence class interferes
@@ -378,9 +377,6 @@ bool AArch64A57FPLoadBalancing::runOnBasicBlock(MachineBasicBlock &MBB) {
   for (auto &I : V)
     Changed |= colorChainSet(I, MBB, Parity);
 
-  for (auto *C : AllChains)
-    delete C;
-
   return Changed;
 }
 
@@ -579,12 +575,14 @@ bool AArch64A57FPLoadBalancing::colorChain(Chain *G, Color C,
 void AArch64A57FPLoadBalancing::
 scanInstruction(MachineInstr *MI, unsigned Idx, 
                 std::map<unsigned, Chain*> &ActiveChains,
-                std::set<Chain*> &AllChains) {
+                std::set<std::unique_ptr<Chain>> &AllChains) {
   // Inspect "MI", updating ActiveChains and AllChains.
 
   if (isMul(MI)) {
 
-    for (auto &I : MI->operands())
+    for (auto &I : MI->uses())
+      maybeKillChain(I, Idx, ActiveChains);
+    for (auto &I : MI->defs())
       maybeKillChain(I, Idx, ActiveChains);
 
     // Create a new chain. Multiplies don't require forwarding so can go on any
@@ -594,9 +592,9 @@ scanInstruction(MachineInstr *MI, unsigned Idx,
     DEBUG(dbgs() << "New chain started for register "
           << TRI->getName(DestReg) << " at " << *MI);
 
-    Chain *G = new Chain(MI, Idx, getColor(DestReg));
-    ActiveChains[DestReg] = G;
-    AllChains.insert(G);
+    auto G = llvm::make_unique<Chain>(MI, Idx, getColor(DestReg));
+    ActiveChains[DestReg] = G.get();
+    AllChains.insert(std::move(G));
 
   } else if (isMla(MI)) {
 
@@ -624,7 +622,10 @@ scanInstruction(MachineInstr *MI, unsigned Idx,
         DEBUG(dbgs() << "Instruction was successfully added to chain.\n");
         ActiveChains[AccumReg]->add(MI, Idx, getColor(DestReg));
         // Handle cases where the destination is not the same as the accumulator.
-        ActiveChains[DestReg] = ActiveChains[AccumReg];
+        if (DestReg != AccumReg) {
+          ActiveChains[DestReg] = ActiveChains[AccumReg];
+          ActiveChains.erase(AccumReg);
+        }
         return;
       }
 
@@ -635,15 +636,17 @@ scanInstruction(MachineInstr *MI, unsigned Idx,
 
     DEBUG(dbgs() << "Creating new chain for dest register "
           << TRI->getName(DestReg) << "\n");
-    Chain *G = new Chain(MI, Idx, getColor(DestReg));
-    ActiveChains[DestReg] = G;
-    AllChains.insert(G);
+    auto G = llvm::make_unique<Chain>(MI, Idx, getColor(DestReg));
+    ActiveChains[DestReg] = G.get();
+    AllChains.insert(std::move(G));
 
   } else {
 
     // Non-MUL or MLA instruction. Invalidate any chain in the uses or defs
     // lists.
-    for (auto &I : MI->operands())
+    for (auto &I : MI->uses())
+      maybeKillChain(I, Idx, ActiveChains);
+    for (auto &I : MI->defs())
       maybeKillChain(I, Idx, ActiveChains);
 
   }
@@ -669,13 +672,14 @@ maybeKillChain(MachineOperand &MO, unsigned Idx,
   } else if (MO.isRegMask()) {
 
     for (auto I = ActiveChains.begin(), E = ActiveChains.end();
-         I != E; ++I) {
+         I != E;) {
       if (MO.clobbersPhysReg(I->first)) {
         DEBUG(dbgs() << "Kill (regmask) seen for chain "
               << TRI->getName(I->first) << "\n");
         I->second->setKill(MI, Idx, /*Immutable=*/true);
-        ActiveChains.erase(I);
-      }
+        ActiveChains.erase(I++);
+      } else
+        ++I;
     }
 
   }
diff --git a/lib/Target/AArch64/AArch64ISelLowering.cpp b/lib/Target/AArch64/AArch64ISelLowering.cpp
index 28d0035..7c33423 100644
--- a/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -472,6 +472,10 @@ AArch64TargetLowering::AArch64TargetLowering(TargetMachine &TM)
       setOperationAction(ISD::FROUND, Ty, Legal);
     }
   }
+
+  // Prefer likely predicted branches to selects on out-of-order cores.
+  if (Subtarget->isCortexA57())
+    PredictableSelectIsExpensive = true;
 }
 
 void AArch64TargetLowering::addTypeForNEON(EVT VT, EVT PromotedBitwiseVT) {