From 16abd8ab144c62c7b1c821c8a083c2345d249d54 Mon Sep 17 00:00:00 2001 From: Stephen Hines Date: Fri, 24 Oct 2014 15:42:31 -0700 Subject: Merge in the following upstream patches to resolve Cortex-A57 crashes. r214957 r215233 r216455 r216721 r217682 r217689 r217690 r217735 Change-Id: Ia53b88591471325df132caf26e1087510a65ce36 --- lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp | 86 +++++++++++++----------- lib/Target/AArch64/AArch64ISelLowering.cpp | 4 ++ 2 files changed, 49 insertions(+), 41 deletions(-) diff --git a/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp b/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp index 195a48e..7742fea 100644 --- a/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp +++ b/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp @@ -73,8 +73,6 @@ static bool isMul(MachineInstr *MI) { case AArch64::FNMULSrr: case AArch64::FMULDrr: case AArch64::FNMULDrr: - - case AArch64::FMULv2f32: return true; default: return false; @@ -92,9 +90,6 @@ static bool isMla(MachineInstr *MI) { case AArch64::FMADDDrrr: case AArch64::FNMSUBDrrr: case AArch64::FNMADDDrrr: - - case AArch64::FMLAv2f32: - case AArch64::FMLSv2f32: return true; default: return false; @@ -139,8 +134,8 @@ private: bool colorChain(Chain *G, Color C, MachineBasicBlock &MBB); int scavengeRegister(Chain *G, Color C, MachineBasicBlock &MBB); void scanInstruction(MachineInstr *MI, unsigned Idx, - std::map &Chains, - std::set &ChainSet); + std::map &Active, + std::set> &AllChains); void maybeKillChain(MachineOperand &MO, unsigned Idx, std::map &RegChains); Color getColor(unsigned Register); @@ -191,10 +186,10 @@ public: /// instruction can be more tricky. Color LastColor; - Chain(MachineInstr *MI, unsigned Idx, Color C) : - StartInst(MI), LastInst(MI), KillInst(NULL), - StartInstIdx(Idx), LastInstIdx(Idx), KillInstIdx(0), - LastColor(C) { + Chain(MachineInstr *MI, unsigned Idx, Color C) + : StartInst(MI), LastInst(MI), KillInst(nullptr), + StartInstIdx(Idx), LastInstIdx(Idx), KillInstIdx(0), + LastColor(C) { Insts.insert(MI); } @@ -204,6 +199,9 @@ public: LastInst = MI; LastInstIdx = Idx; LastColor = C; + assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) && + "Chain: broken invariant. A Chain can only be killed after its last " + "def"); Insts.insert(MI); } @@ -222,6 +220,9 @@ public: KillInst = MI; KillInstIdx = Idx; KillIsImmutable = Immutable; + assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) && + "Chain: broken invariant. A Chain can only be killed after its last " + "def"); } /// Return the first instruction in the chain. @@ -232,7 +233,7 @@ public: MachineInstr *getKill() const { return KillInst; } /// Return an instruction that can be used as an iterator for the end /// of the chain. This is the maximum of KillInst (if set) and LastInst. - MachineInstr *getEnd() const { + MachineBasicBlock::iterator getEnd() const { return ++MachineBasicBlock::iterator(KillInst ? KillInst : LastInst); } @@ -247,12 +248,12 @@ public: } /// Return true if this chain (StartInst..KillInst) overlaps with Other. - bool rangeOverlapsWith(Chain *Other) { + bool rangeOverlapsWith(const Chain &Other) const { unsigned End = KillInst ? KillInstIdx : LastInstIdx; - unsigned OtherEnd = Other->KillInst ? - Other->KillInstIdx : Other->LastInstIdx; + unsigned OtherEnd = Other.KillInst ? + Other.KillInstIdx : Other.LastInstIdx; - return StartInstIdx <= OtherEnd && Other->StartInstIdx <= End; + return StartInstIdx <= OtherEnd && Other.StartInstIdx <= End; } /// Return true if this chain starts before Other. @@ -317,7 +318,7 @@ bool AArch64A57FPLoadBalancing::runOnBasicBlock(MachineBasicBlock &MBB) { // been killed yet. This is keyed by register - all chains can only have one // "link" register between each inst in the chain. std::map ActiveChains; - std::set AllChains; + std::set> AllChains; unsigned Idx = 0; for (auto &MI : MBB) scanInstruction(&MI, Idx++, ActiveChains, AllChains); @@ -332,15 +333,13 @@ bool AArch64A57FPLoadBalancing::runOnBasicBlock(MachineBasicBlock &MBB) { // range of chains is quite small and they are clustered between loads // and stores. EquivalenceClasses EC; - for (auto *I : AllChains) - EC.insert(I); + for (auto &I : AllChains) + EC.insert(I.get()); - for (auto *I : AllChains) { - for (auto *J : AllChains) { - if (I != J && I->rangeOverlapsWith(J)) - EC.unionSets(I, J); - } - } + for (auto &I : AllChains) + for (auto &J : AllChains) + if (I != J && I->rangeOverlapsWith(*J)) + EC.unionSets(I.get(), J.get()); DEBUG(dbgs() << "Created " << EC.getNumClasses() << " disjoint sets.\n"); // Now we assume that every member of an equivalence class interferes @@ -378,9 +377,6 @@ bool AArch64A57FPLoadBalancing::runOnBasicBlock(MachineBasicBlock &MBB) { for (auto &I : V) Changed |= colorChainSet(I, MBB, Parity); - for (auto *C : AllChains) - delete C; - return Changed; } @@ -579,12 +575,14 @@ bool AArch64A57FPLoadBalancing::colorChain(Chain *G, Color C, void AArch64A57FPLoadBalancing:: scanInstruction(MachineInstr *MI, unsigned Idx, std::map &ActiveChains, - std::set &AllChains) { + std::set> &AllChains) { // Inspect "MI", updating ActiveChains and AllChains. if (isMul(MI)) { - for (auto &I : MI->operands()) + for (auto &I : MI->uses()) + maybeKillChain(I, Idx, ActiveChains); + for (auto &I : MI->defs()) maybeKillChain(I, Idx, ActiveChains); // Create a new chain. Multiplies don't require forwarding so can go on any @@ -594,9 +592,9 @@ scanInstruction(MachineInstr *MI, unsigned Idx, DEBUG(dbgs() << "New chain started for register " << TRI->getName(DestReg) << " at " << *MI); - Chain *G = new Chain(MI, Idx, getColor(DestReg)); - ActiveChains[DestReg] = G; - AllChains.insert(G); + auto G = llvm::make_unique(MI, Idx, getColor(DestReg)); + ActiveChains[DestReg] = G.get(); + AllChains.insert(std::move(G)); } else if (isMla(MI)) { @@ -624,7 +622,10 @@ scanInstruction(MachineInstr *MI, unsigned Idx, DEBUG(dbgs() << "Instruction was successfully added to chain.\n"); ActiveChains[AccumReg]->add(MI, Idx, getColor(DestReg)); // Handle cases where the destination is not the same as the accumulator. - ActiveChains[DestReg] = ActiveChains[AccumReg]; + if (DestReg != AccumReg) { + ActiveChains[DestReg] = ActiveChains[AccumReg]; + ActiveChains.erase(AccumReg); + } return; } @@ -635,15 +636,17 @@ scanInstruction(MachineInstr *MI, unsigned Idx, DEBUG(dbgs() << "Creating new chain for dest register " << TRI->getName(DestReg) << "\n"); - Chain *G = new Chain(MI, Idx, getColor(DestReg)); - ActiveChains[DestReg] = G; - AllChains.insert(G); + auto G = llvm::make_unique(MI, Idx, getColor(DestReg)); + ActiveChains[DestReg] = G.get(); + AllChains.insert(std::move(G)); } else { // Non-MUL or MLA instruction. Invalidate any chain in the uses or defs // lists. - for (auto &I : MI->operands()) + for (auto &I : MI->uses()) + maybeKillChain(I, Idx, ActiveChains); + for (auto &I : MI->defs()) maybeKillChain(I, Idx, ActiveChains); } @@ -669,13 +672,14 @@ maybeKillChain(MachineOperand &MO, unsigned Idx, } else if (MO.isRegMask()) { for (auto I = ActiveChains.begin(), E = ActiveChains.end(); - I != E; ++I) { + I != E;) { if (MO.clobbersPhysReg(I->first)) { DEBUG(dbgs() << "Kill (regmask) seen for chain " << TRI->getName(I->first) << "\n"); I->second->setKill(MI, Idx, /*Immutable=*/true); - ActiveChains.erase(I); - } + ActiveChains.erase(I++); + } else + ++I; } } diff --git a/lib/Target/AArch64/AArch64ISelLowering.cpp b/lib/Target/AArch64/AArch64ISelLowering.cpp index 28d0035..7c33423 100644 --- a/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -472,6 +472,10 @@ AArch64TargetLowering::AArch64TargetLowering(TargetMachine &TM) setOperationAction(ISD::FROUND, Ty, Legal); } } + + // Prefer likely predicted branches to selects on out-of-order cores. + if (Subtarget->isCortexA57()) + PredictableSelectIsExpensive = true; } void AArch64TargetLowering::addTypeForNEON(EVT VT, EVT PromotedBitwiseVT) { -- cgit v1.1