diff options
28 files changed, 653 insertions, 169 deletions
diff --git a/include/llvm/CodeGen/MachineBasicBlock.h b/include/llvm/CodeGen/MachineBasicBlock.h index 1ec89ee..10a32d3 100644 --- a/include/llvm/CodeGen/MachineBasicBlock.h +++ b/include/llvm/CodeGen/MachineBasicBlock.h @@ -194,21 +194,21 @@ public: return tmp; } - IterTy getInsnIterator() const { + IterTy getInstrIterator() const { return MII; } }; - typedef Instructions::iterator insn_iterator; - typedef Instructions::const_iterator const_insn_iterator; - typedef std::reverse_iterator<insn_iterator> reverse_insn_iterator; + typedef Instructions::iterator instr_iterator; + typedef Instructions::const_iterator const_instr_iterator; + typedef std::reverse_iterator<instr_iterator> reverse_instr_iterator; typedef - std::reverse_iterator<const_insn_iterator> const_reverse_insn_iterator; + std::reverse_iterator<const_instr_iterator> const_reverse_instr_iterator; typedef - bundle_iterator<MachineInstr,insn_iterator> iterator; + bundle_iterator<MachineInstr,instr_iterator> iterator; typedef - bundle_iterator<const MachineInstr,const_insn_iterator> const_iterator; + bundle_iterator<const MachineInstr,const_instr_iterator> const_iterator; typedef std::reverse_iterator<const_iterator> const_reverse_iterator; typedef std::reverse_iterator<iterator> reverse_iterator; @@ -221,44 +221,44 @@ public: const MachineInstr& front() const { return Insts.front(); } const MachineInstr& back() const { return Insts.back(); } - insn_iterator insn_begin() { return Insts.begin(); } - const_insn_iterator insn_begin() const { return Insts.begin(); } - insn_iterator insn_end() { return Insts.end(); } - const_insn_iterator insn_end() const { return Insts.end(); } - reverse_insn_iterator insn_rbegin() { return Insts.rbegin(); } - const_reverse_insn_iterator insn_rbegin() const { return Insts.rbegin(); } - reverse_insn_iterator insn_rend () { return Insts.rend(); } - const_reverse_insn_iterator insn_rend () const { return Insts.rend(); } + instr_iterator instr_begin() { return Insts.begin(); } + const_instr_iterator instr_begin() const { return Insts.begin(); } + instr_iterator instr_end() { return Insts.end(); } + const_instr_iterator instr_end() const { return Insts.end(); } + reverse_instr_iterator instr_rbegin() { return Insts.rbegin(); } + const_reverse_instr_iterator instr_rbegin() const { return Insts.rbegin(); } + reverse_instr_iterator instr_rend () { return Insts.rend(); } + const_reverse_instr_iterator instr_rend () const { return Insts.rend(); } iterator begin() { return Insts.begin(); } const_iterator begin() const { return Insts.begin(); } iterator end() { - insn_iterator II = insn_end(); - if (II != insn_begin()) { + instr_iterator II = instr_end(); + if (II != instr_begin()) { while (II->isInsideBundle()) --II; } return II; } const_iterator end() const { - const_insn_iterator II = insn_end(); - if (II != insn_begin()) { + const_instr_iterator II = instr_end(); + if (II != instr_begin()) { while (II->isInsideBundle()) --II; } return II; } reverse_iterator rbegin() { - reverse_insn_iterator II = insn_rbegin(); - if (II != insn_rend()) { + reverse_instr_iterator II = instr_rbegin(); + if (II != instr_rend()) { while (II->isInsideBundle()) ++II; } return II; } const_reverse_iterator rbegin() const { - const_reverse_insn_iterator II = insn_rbegin(); - if (II != insn_rend()) { + const_reverse_instr_iterator II = instr_rbegin(); + if (II != instr_rend()) { while (II->isInsideBundle()) ++II; } @@ -442,9 +442,9 @@ public: iterator getFirstTerminator(); const_iterator getFirstTerminator() const; - /// getFirstInsnTerminator - Same getFirstTerminator but it ignores bundles - /// and return an insn_iterator instead. - insn_iterator getFirstInsnTerminator(); + /// getFirstInstrTerminator - Same getFirstTerminator but it ignores bundles + /// and return an instr_iterator instead. + instr_iterator getFirstInstrTerminator(); /// getLastNonDebugInstr - returns an iterator to the last non-debug /// instruction in the basic block, or end() @@ -464,68 +464,80 @@ public: void push_back(MachineInstr *MI) { Insts.push_back(MI); } template<typename IT> - void insert(insn_iterator I, IT S, IT E) { + void insert(instr_iterator I, IT S, IT E) { Insts.insert(I, S, E); } - insn_iterator insert(insn_iterator I, MachineInstr *M) { + instr_iterator insert(instr_iterator I, MachineInstr *M) { return Insts.insert(I, M); } - insn_iterator insertAfter(insn_iterator I, MachineInstr *M) { + instr_iterator insertAfter(instr_iterator I, MachineInstr *M) { return Insts.insertAfter(I, M); } template<typename IT> void insert(iterator I, IT S, IT E) { - Insts.insert(I.getInsnIterator(), S, E); + Insts.insert(I.getInstrIterator(), S, E); } iterator insert(iterator I, MachineInstr *M) { - return Insts.insert(I.getInsnIterator(), M); + return Insts.insert(I.getInstrIterator(), M); } iterator insertAfter(iterator I, MachineInstr *M) { - return Insts.insertAfter(I.getInsnIterator(), M); + return Insts.insertAfter(I.getInstrIterator(), M); } - // erase - Remove the specified element or range from the instruction list. - // These functions delete any instructions removed. - // - insn_iterator erase(insn_iterator I) { + /// erase - Remove the specified element or range from the instruction list. + /// These functions delete any instructions removed. + /// + instr_iterator erase(instr_iterator I) { return Insts.erase(I); } - insn_iterator erase(insn_iterator I, insn_iterator E) { + instr_iterator erase(instr_iterator I, instr_iterator E) { return Insts.erase(I, E); } - - iterator erase(iterator I) { - return Insts.erase(I.getInsnIterator()); + instr_iterator erase_instr(MachineInstr *I) { + instr_iterator MII(I); + return erase(MII); } + + iterator erase(iterator I); iterator erase(iterator I, iterator E) { - return Insts.erase(I.getInsnIterator(), E.getInsnIterator()); + return Insts.erase(I.getInstrIterator(), E.getInstrIterator()); + } + iterator erase(MachineInstr *I) { + iterator MII(I); + return erase(MII); } - iterator erase(MachineInstr *I) { iterator MII(I); return erase(MII); } - MachineInstr *remove(MachineInstr *I) { return Insts.remove(I); } - void clear() { Insts.clear(); } + /// remove - Remove the instruction from the instruction list. This function + /// does not delete the instruction. WARNING: Note, if the specified + /// instruction is a bundle this function will remove all the bundled + /// instructions as well. It is up to the caller to keep a list of the + /// bundled instructions and re-insert them if desired. This function is + /// *not recommended* for manipulating instructions with bundled. Use + /// splice instead. + MachineInstr *remove(MachineInstr *I); + void clear() { + Insts.clear(); + } /// splice - Take an instruction from MBB 'Other' at the position From, /// and insert it into this MBB right before 'where'. - void splice(insn_iterator where, MachineBasicBlock *Other, - insn_iterator From) { + void splice(instr_iterator where, MachineBasicBlock *Other, + instr_iterator From) { Insts.splice(where, Other->Insts, From); } - void splice(iterator where, MachineBasicBlock *Other, iterator From) { - Insts.splice(where.getInsnIterator(), Other->Insts, From.getInsnIterator()); - } + void splice(iterator where, MachineBasicBlock *Other, iterator From); /// splice - Take a block of instructions from MBB 'Other' in the range [From, /// To), and insert them into this MBB right before 'where'. - void splice(insn_iterator where, MachineBasicBlock *Other, insn_iterator From, - insn_iterator To) { + void splice(instr_iterator where, MachineBasicBlock *Other, instr_iterator From, + instr_iterator To) { Insts.splice(where, Other->Insts, From, To); } void splice(iterator where, MachineBasicBlock *Other, iterator From, iterator To) { - Insts.splice(where.getInsnIterator(), Other->Insts, - From.getInsnIterator(), To.getInsnIterator()); + Insts.splice(where.getInstrIterator(), Other->Insts, + From.getInstrIterator(), To.getInstrIterator()); } /// removeFromParent - This method unlinks 'this' from the containing @@ -552,9 +564,9 @@ public: /// findDebugLoc - find the next valid DebugLoc starting at MBBI, skipping /// any DBG_VALUE instructions. Return UnknownLoc if there is none. - DebugLoc findDebugLoc(insn_iterator MBBI); + DebugLoc findDebugLoc(instr_iterator MBBI); DebugLoc findDebugLoc(iterator MBBI) { - return findDebugLoc(MBBI.getInsnIterator()); + return findDebugLoc(MBBI.getInstrIterator()); } // Debugging methods. diff --git a/include/llvm/CodeGen/MachineInstr.h b/include/llvm/CodeGen/MachineInstr.h index 9b04c1d..904f1a6 100644 --- a/include/llvm/CodeGen/MachineInstr.h +++ b/include/llvm/CodeGen/MachineInstr.h @@ -175,6 +175,11 @@ public: Flags = flags; } + /// clearFlag - Clear a MI flag. + void clearFlag(MIFlag Flag) { + Flags &= ~((uint8_t)Flag); + } + /// isInsideBundle - Return true if MI is in a bundle (but not the first MI /// in a bundle). /// @@ -215,6 +220,15 @@ public: return getFlag(InsideBundle); } + /// setIsInsideBundle - Set InsideBundle bit. + /// + void setIsInsideBundle(bool Val = true) { + if (Val) + setFlag(InsideBundle); + else + clearFlag(InsideBundle); + } + /// getDebugLoc - Returns the debug location id of this MachineInstr. /// DebugLoc getDebugLoc() const { return debugLoc; } @@ -589,6 +603,9 @@ public: bool isRegSequence() const { return getOpcode() == TargetOpcode::REG_SEQUENCE; } + bool isBundle() const { + return getOpcode() == TargetOpcode::BUNDLE; + } bool isCopy() const { return getOpcode() == TargetOpcode::COPY; } @@ -608,6 +625,9 @@ public: getOperand(0).getSubReg() == getOperand(1).getSubReg(); } + /// getBundleSize - Return the number of instructions inside the MI bundle. + unsigned getBundleSize() const; + /// readsRegister - Return true if the MachineInstr reads the specified /// register. If TargetRegisterInfo is passed, then it also checks if there /// is a read of a super-register. diff --git a/include/llvm/CodeGen/MachineInstrBuilder.h b/include/llvm/CodeGen/MachineInstrBuilder.h index b989027..8025642 100644 --- a/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/include/llvm/CodeGen/MachineInstrBuilder.h @@ -209,6 +209,30 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, return MachineInstrBuilder(MI).addReg(DestReg, RegState::Define); } +inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, + MachineBasicBlock::instr_iterator I, + DebugLoc DL, + const MCInstrDesc &MCID, + unsigned DestReg) { + MachineInstr *MI = BB.getParent()->CreateMachineInstr(MCID, DL); + BB.insert(I, MI); + return MachineInstrBuilder(MI).addReg(DestReg, RegState::Define); +} + +inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, + MachineInstr *I, + DebugLoc DL, + const MCInstrDesc &MCID, + unsigned DestReg) { + if (I->isInsideBundle()) { + MachineBasicBlock::instr_iterator MII = I; + return BuildMI(BB, MII, DL, MCID, DestReg); + } + + MachineBasicBlock::iterator MII = I; + return BuildMI(BB, MII, DL, MCID, DestReg); +} + /// BuildMI - This version of the builder inserts the newly-built /// instruction before the given position in the given MachineBasicBlock, and /// does NOT take a destination register. @@ -222,6 +246,28 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, return MachineInstrBuilder(MI); } +inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, + MachineBasicBlock::instr_iterator I, + DebugLoc DL, + const MCInstrDesc &MCID) { + MachineInstr *MI = BB.getParent()->CreateMachineInstr(MCID, DL); + BB.insert(I, MI); + return MachineInstrBuilder(MI); +} + +inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB, + MachineInstr *I, + DebugLoc DL, + const MCInstrDesc &MCID) { + if (I->isInsideBundle()) { + MachineBasicBlock::instr_iterator MII = I; + return BuildMI(BB, MII, DL, MCID); + } + + MachineBasicBlock::iterator MII = I; + return BuildMI(BB, MII, DL, MCID); +} + /// BuildMI - This version of the builder inserts the newly-built /// instruction at the end of the given MachineBasicBlock, and does NOT take a /// destination register. diff --git a/include/llvm/CodeGen/MachineInstrBundle.h b/include/llvm/CodeGen/MachineInstrBundle.h new file mode 100644 index 0000000..8c150e6 --- /dev/null +++ b/include/llvm/CodeGen/MachineInstrBundle.h @@ -0,0 +1,34 @@ +//===-- CodeGen/MachineInstBundle.h - MI bundle utilities -------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file provide utility functions to manipulate machine instruction +// bundles. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_MACHINEINSTRBUNDLE_H +#define LLVM_CODEGEN_MACHINEINSTRBUNDLE_H + +#include "llvm/CodeGen/MachineBasicBlock.h" + +namespace llvm { + +/// FinalizeBundle - Finalize a machine instruction bundle which includes +/// a sequence of instructions starting from FirstMI to LastMI (inclusive). +/// This routine adds a BUNDLE instruction to represent the bundle, it adds +/// IsInternalRead markers to MachineOperands which are defined inside the +/// bundle, and it copies externally visible defs and uses to the BUNDLE +/// instruction. +void FinalizeBundle(MachineBasicBlock &MBB, + MachineBasicBlock::instr_iterator FirstMI, + MachineBasicBlock::instr_iterator LastMI); + +} // End llvm namespace + +#endif diff --git a/include/llvm/CodeGen/Passes.h b/include/llvm/CodeGen/Passes.h index 45b3353..b4f6cac 100644 --- a/include/llvm/CodeGen/Passes.h +++ b/include/llvm/CodeGen/Passes.h @@ -238,6 +238,10 @@ namespace llvm { /// FunctionPass *createExecutionDependencyFixPass(const TargetRegisterClass *RC); + /// createUnpackMachineBundles - This pass unpack machine instruction bundles. + /// + FunctionPass *createUnpackMachineBundlesPass(); + } // End llvm namespace #endif diff --git a/include/llvm/InitializePasses.h b/include/llvm/InitializePasses.h index 565b64b..ce87f16 100644 --- a/include/llvm/InitializePasses.h +++ b/include/llvm/InitializePasses.h @@ -231,6 +231,7 @@ void initializeUnreachableMachineBlockElimPass(PassRegistry&); void initializeVerifierPass(PassRegistry&); void initializeVirtRegMapPass(PassRegistry&); void initializeInstSimplifierPass(PassRegistry&); +void initializeUnpackMachineBundlesPass(PassRegistry&); } diff --git a/lib/CodeGen/BranchFolding.cpp b/lib/CodeGen/BranchFolding.cpp index 0d88e6c..89894c3 100644 --- a/lib/CodeGen/BranchFolding.cpp +++ b/lib/CodeGen/BranchFolding.cpp @@ -926,8 +926,9 @@ bool BranchFolder::TailMergeBlocks(MachineFunction &MF) { if (MergePotentials.size() >= 2) MadeChange |= TryTailMergeBlocks(IBB, PredBB); // Reinsert an unconditional branch if needed. - // The 1 below can occur as a result of removing blocks in TryTailMergeBlocks. - PredBB = prior(I); // this may have been changed in TryTailMergeBlocks + // The 1 below can occur as a result of removing blocks in + // TryTailMergeBlocks. + PredBB = prior(I); // this may have been changed in TryTailMergeBlocks if (MergePotentials.size() == 1 && MergePotentials.begin()->getBlock() != PredBB) FixTail(MergePotentials.begin()->getBlock(), IBB, TII); diff --git a/lib/CodeGen/MachineBasicBlock.cpp b/lib/CodeGen/MachineBasicBlock.cpp index 4dc8173..6734916 100644 --- a/lib/CodeGen/MachineBasicBlock.cpp +++ b/lib/CodeGen/MachineBasicBlock.cpp @@ -73,8 +73,8 @@ void ilist_traits<MachineBasicBlock>::addNodeToList(MachineBasicBlock *N) { // Make sure the instructions have their operands in the reginfo lists. MachineRegisterInfo &RegInfo = MF.getRegInfo(); - for (MachineBasicBlock::insn_iterator I = N->insn_begin(), E = N->insn_end(); - I != E; ++I) + for (MachineBasicBlock::instr_iterator + I = N->instr_begin(), E = N->instr_end(); I != E; ++I) I->AddRegOperandsToUseLists(RegInfo); LeakDetector::removeGarbageObject(N); @@ -141,7 +141,7 @@ void ilist_traits<MachineInstr>::deleteNode(MachineInstr* MI) { } MachineBasicBlock::iterator MachineBasicBlock::getFirstNonPHI() { - insn_iterator I = insn_begin(); + instr_iterator I = instr_begin(); while (I != end() && I->isPHI()) ++I; assert(!I->isInsideBundle() && "First non-phi MI cannot be inside a bundle!"); @@ -178,18 +178,18 @@ MachineBasicBlock::getFirstTerminator() const { return I; } -MachineBasicBlock::insn_iterator MachineBasicBlock::getFirstInsnTerminator() { - insn_iterator I = insn_end(); - while (I != insn_begin() && ((--I)->isTerminator() || I->isDebugValue())) +MachineBasicBlock::instr_iterator MachineBasicBlock::getFirstInstrTerminator() { + instr_iterator I = instr_end(); + while (I != instr_begin() && ((--I)->isTerminator() || I->isDebugValue())) ; /*noop */ - while (I != insn_end() && !I->isTerminator()) + while (I != instr_end() && !I->isTerminator()) ++I; return I; } MachineBasicBlock::iterator MachineBasicBlock::getLastNonDebugInstr() { // Skip over end-of-block dbg_value instructions. - insn_iterator B = insn_begin(), I = insn_end(); + instr_iterator B = instr_begin(), I = instr_end(); while (I != B) { --I; // Return instruction that starts a bundle. @@ -204,7 +204,7 @@ MachineBasicBlock::iterator MachineBasicBlock::getLastNonDebugInstr() { MachineBasicBlock::const_iterator MachineBasicBlock::getLastNonDebugInstr() const { // Skip over end-of-block dbg_value instructions. - const_insn_iterator B = insn_begin(), I = insn_end(); + const_instr_iterator B = instr_begin(), I = instr_end(); while (I != B) { --I; // Return instruction that starts a bundle. @@ -283,13 +283,15 @@ void MachineBasicBlock::print(raw_ostream &OS, SlotIndexes *Indexes) const { OS << '\n'; } - for (const_iterator I = begin(); I != end(); ++I) { + for (const_instr_iterator I = instr_begin(); I != instr_end(); ++I) { if (Indexes) { if (Indexes->hasIndex(I)) OS << Indexes->getInstructionIndex(I); OS << '\t'; } OS << '\t'; + if (I->isInsideBundle()) + OS << " * "; I->print(OS, &getParent()->getTarget()); } @@ -495,8 +497,8 @@ MachineBasicBlock::transferSuccessorsAndUpdatePHIs(MachineBasicBlock *fromMBB) { fromMBB->removeSuccessor(Succ); // Fix up any PHI nodes in the successor. - for (MachineBasicBlock::insn_iterator MI = Succ->insn_begin(), - ME = Succ->insn_end(); MI != ME && MI->isPHI(); ++MI) + for (MachineBasicBlock::instr_iterator MI = Succ->instr_begin(), + ME = Succ->instr_end(); MI != ME && MI->isPHI(); ++MI) for (unsigned i = 2, e = MI->getNumOperands()+1; i != e; i += 2) { MachineOperand &MO = MI->getOperand(i); if (MO.getMBB() == fromMBB) @@ -598,7 +600,7 @@ MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { // Collect a list of virtual registers killed by the terminators. SmallVector<unsigned, 4> KilledRegs; if (LV) - for (insn_iterator I = getFirstInsnTerminator(), E = insn_end(); + for (instr_iterator I = getFirstInstrTerminator(), E = instr_end(); I != E; ++I) { MachineInstr *MI = I; for (MachineInstr::mop_iterator OI = MI->operands_begin(), @@ -626,8 +628,9 @@ MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { } // Fix PHI nodes in Succ so they refer to NMBB instead of this - for (MachineBasicBlock::insn_iterator - i = Succ->insn_begin(),e = Succ->insn_end(); i != e && i->isPHI(); ++i) + for (MachineBasicBlock::instr_iterator + i = Succ->instr_begin(),e = Succ->instr_end(); + i != e && i->isPHI(); ++i) for (unsigned ni = 1, ne = i->getNumOperands(); ni != ne; ni += 2) if (i->getOperand(ni+1).getMBB() == this) i->getOperand(ni+1).setMBB(NMBB); @@ -642,7 +645,7 @@ MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { // Restore kills of virtual registers that were killed by the terminators. while (!KilledRegs.empty()) { unsigned Reg = KilledRegs.pop_back_val(); - for (insn_iterator I = insn_end(), E = insn_begin(); I != E;) { + for (instr_iterator I = instr_end(), E = instr_begin(); I != E;) { if (!(--I)->addRegisterKilled(Reg, NULL, /* addIfNotFound= */ false)) continue; LV->getVarInfo(Reg).Kills.push_back(I); @@ -711,6 +714,41 @@ MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { return NMBB; } +MachineBasicBlock::iterator +MachineBasicBlock::erase(MachineBasicBlock::iterator I) { + if (I->isBundle()) { + MachineBasicBlock::iterator E = llvm::next(I); + return Insts.erase(I.getInstrIterator(), E.getInstrIterator()); + } + + return Insts.erase(I.getInstrIterator()); +} + +MachineInstr *MachineBasicBlock::remove(MachineInstr *I) { + if (I->isBundle()) { + MachineBasicBlock::instr_iterator MII = I; ++MII; + while (MII != end() && MII->isInsideBundle()) { + MachineInstr *MI = &*MII++; + Insts.remove(MI); + } + } + + return Insts.remove(I); +} + +void MachineBasicBlock::splice(MachineBasicBlock::iterator where, + MachineBasicBlock *Other, + MachineBasicBlock::iterator From) { + if (From->isBundle()) { + MachineBasicBlock::iterator To = llvm::next(From); + Insts.splice(where.getInstrIterator(), Other->Insts, + From.getInstrIterator(), To.getInstrIterator()); + return; + } + + Insts.splice(where.getInstrIterator(), Other->Insts, From.getInstrIterator()); +} + /// removeFromParent - This method unlinks 'this' from the containing function, /// and returns it, but does not delete it. MachineBasicBlock *MachineBasicBlock::removeFromParent() { @@ -734,8 +772,8 @@ void MachineBasicBlock::ReplaceUsesOfBlockWith(MachineBasicBlock *Old, MachineBasicBlock *New) { assert(Old != New && "Cannot replace self with self!"); - MachineBasicBlock::insn_iterator I = insn_end(); - while (I != insn_begin()) { + MachineBasicBlock::instr_iterator I = instr_end(); + while (I != instr_begin()) { --I; if (!I->isTerminator()) break; @@ -816,9 +854,9 @@ bool MachineBasicBlock::CorrectExtraCFGEdges(MachineBasicBlock *DestA, /// findDebugLoc - find the next valid DebugLoc starting at MBBI, skipping /// any DBG_VALUE instructions. Return UnknownLoc if there is none. DebugLoc -MachineBasicBlock::findDebugLoc(insn_iterator MBBI) { +MachineBasicBlock::findDebugLoc(instr_iterator MBBI) { DebugLoc DL; - insn_iterator E = insn_end(); + instr_iterator E = instr_end(); if (MBBI == E) return DL; diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp index d16e5d4..ec5a1cd 100644 --- a/lib/CodeGen/MachineInstr.cpp +++ b/lib/CodeGen/MachineInstr.cpp @@ -750,11 +750,11 @@ void MachineInstr::addMemOperand(MachineFunction &MF, bool MachineInstr::hasProperty(unsigned MCFlag, QueryType Type) const { - if (Type == IgnoreBundle || getOpcode() != TargetOpcode::BUNDLE) + if (Type == IgnoreBundle || !isBundle()) return getDesc().getFlags() & (1 << MCFlag); const MachineBasicBlock *MBB = getParent(); - MachineBasicBlock::const_insn_iterator MII = *this; ++MII; + MachineBasicBlock::const_instr_iterator MII = *this; ++MII; while (MII != MBB->end() && MII->isInsideBundle()) { if (MII->getDesc().getFlags() & (1 << MCFlag)) { if (Type == AnyInBundle) @@ -777,6 +777,19 @@ bool MachineInstr::isIdenticalTo(const MachineInstr *Other, Other->getNumOperands() != getNumOperands()) return false; + if (isBundle()) { + // Both instructions are bundles, compare MIs inside the bundle. + MachineBasicBlock::const_instr_iterator I1 = *this; + MachineBasicBlock::const_instr_iterator E1 = getParent()->instr_end(); + MachineBasicBlock::const_instr_iterator I2 = *Other; + MachineBasicBlock::const_instr_iterator E2= Other->getParent()->instr_end(); + while (++I1 != E1 && I1->isInsideBundle()) { + ++I2; + if (I2 == E2 || !I2->isInsideBundle() || !I1->isIdenticalTo(I2, Check)) + return false; + } + } + // Check operands to make sure they match. for (unsigned i = 0, e = getNumOperands(); i != e; ++i) { const MachineOperand &MO = getOperand(i); @@ -825,10 +838,11 @@ MachineInstr *MachineInstr::removeFromParent() { assert(getParent() && "Not embedded in a basic block!"); // If it's a bundle then remove the MIs inside the bundle as well. - if (getOpcode() == TargetOpcode::BUNDLE) { + if (isBundle()) { MachineBasicBlock *MBB = getParent(); - MachineBasicBlock::insn_iterator MII = *this; ++MII; - while (MII != MBB->end() && MII->isInsideBundle()) { + MachineBasicBlock::instr_iterator MII = *this; ++MII; + MachineBasicBlock::instr_iterator E = MBB->instr_end(); + while (MII != E && MII->isInsideBundle()) { MachineInstr *MI = &*MII; ++MII; MBB->remove(MI); @@ -844,10 +858,11 @@ MachineInstr *MachineInstr::removeFromParent() { void MachineInstr::eraseFromParent() { assert(getParent() && "Not embedded in a basic block!"); // If it's a bundle then remove the MIs inside the bundle as well. - if (getOpcode() == TargetOpcode::BUNDLE) { + if (isBundle()) { MachineBasicBlock *MBB = getParent(); - MachineBasicBlock::insn_iterator MII = *this; ++MII; - while (MII != MBB->end() && MII->isInsideBundle()) { + MachineBasicBlock::instr_iterator MII = *this; ++MII; + MachineBasicBlock::instr_iterator E = MBB->instr_end(); + while (MII != E && MII->isInsideBundle()) { MachineInstr *MI = &*MII; ++MII; MBB->erase(MI); @@ -942,6 +957,20 @@ MachineInstr::getRegClassConstraint(unsigned OpIdx, return NULL; } +/// getBundleSize - Return the number of instructions inside the MI bundle. +unsigned MachineInstr::getBundleSize() const { + assert(isBundle() && "Expecting a bundle"); + + MachineBasicBlock::const_instr_iterator I = *this; + unsigned Size = 0; + while ((++I)->isInsideBundle()) { + ++Size; + } + assert(Size > 1 && "Malformed bundle"); + + return Size; +} + /// findRegisterUseOperandIdx() - Returns the MachineOperand that is a use of /// the specific register or -1 if it is not found. It further tightens /// the search criteria to a use that kills the register if isKill is true. @@ -1024,9 +1053,6 @@ MachineInstr::findRegisterDefOperandIdx(unsigned Reg, bool isDead, bool Overlap, /// operand list that is used to represent the predicate. It returns -1 if /// none is found. int MachineInstr::findFirstPredOperandIdx() const { - assert(getOpcode() != TargetOpcode::BUNDLE && - "MachineInstr::findFirstPredOperandIdx() can't handle bundles"); - // Don't call MCID.findFirstPredOperandIdx() because this variant // is sometimes called on an instruction that's not yet complete, and // so the number of operands is less than the MCID indicates. In @@ -1176,8 +1202,7 @@ void MachineInstr::copyKillDeadInfo(const MachineInstr *MI) { /// copyPredicates - Copies predicate operand(s) from MI. void MachineInstr::copyPredicates(const MachineInstr *MI) { - assert(getOpcode() != TargetOpcode::BUNDLE && - "MachineInstr::copyPredicates() can't handle bundles"); + assert(!isBundle() && "MachineInstr::copyPredicates() can't handle bundles"); const MCInstrDesc &MCID = MI->getDesc(); if (!MCID.isPredicable()) diff --git a/lib/CodeGen/MachineInstrBundle.cpp b/lib/CodeGen/MachineInstrBundle.cpp new file mode 100644 index 0000000..b766d08 --- /dev/null +++ b/lib/CodeGen/MachineInstrBundle.cpp @@ -0,0 +1,180 @@ +//===-- lib/CodeGen/MachineInstrBundle.cpp --------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/MachineInstrBundle.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +using namespace llvm; + +namespace { + class UnpackMachineBundles : public MachineFunctionPass { + public: + static char ID; // Pass identification + UnpackMachineBundles() : MachineFunctionPass(ID) { + initializeUnpackMachineBundlesPass(*PassRegistry::getPassRegistry()); + } + + virtual bool runOnMachineFunction(MachineFunction &MF); + }; +} // end anonymous namespace + +char UnpackMachineBundles::ID = 0; +INITIALIZE_PASS(UnpackMachineBundles, "unpack-mi-bundle", + "Unpack machine instruction bundles", false, false) + +FunctionPass *llvm::createUnpackMachineBundlesPass() { + return new UnpackMachineBundles(); +} + +bool UnpackMachineBundles::runOnMachineFunction(MachineFunction &MF) { + bool Changed = false; + for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) { + MachineBasicBlock *MBB = &*I; + + for (MachineBasicBlock::instr_iterator MII = MBB->instr_begin(), + MIE = MBB->instr_end(); MII != MIE; ) { + MachineInstr *MI = &*MII; + + // Remove BUNDLE instruction and the InsideBundle flags from bundled + // instructions. + if (MI->isBundle()) { + while (++MII != MIE && MII->isInsideBundle()) { + MII->setIsInsideBundle(false); + for (unsigned i = 0, e = MII->getNumOperands(); i != e; ++i) { + MachineOperand &MO = MII->getOperand(i); + if (MO.isReg() && MO.isInternalRead()) + MO.setIsInternalRead(false); + } + } + MI->eraseFromParent(); + + Changed = true; + continue; + } + + ++MII; + } + } + + return Changed; +} + +/// FinalizeBundle - Finalize a machine instruction bundle which includes +/// a sequence of instructions starting from FirstMI to LastMI (inclusive). +/// This routine adds a BUNDLE instruction to represent the bundle, it adds +/// IsInternalRead markers to MachineOperands which are defined inside the +/// bundle, and it copies externally visible defs and uses to the BUNDLE +/// instruction. +void llvm::FinalizeBundle(MachineBasicBlock &MBB, + MachineBasicBlock::instr_iterator FirstMI, + MachineBasicBlock::instr_iterator LastMI) { + const TargetMachine &TM = MBB.getParent()->getTarget(); + const TargetInstrInfo *TII = TM.getInstrInfo(); + const TargetRegisterInfo *TRI = TM.getRegisterInfo(); + + MachineInstrBuilder MIB = BuildMI(MBB, FirstMI, FirstMI->getDebugLoc(), + TII->get(TargetOpcode::BUNDLE)); + + SmallVector<unsigned, 8> LocalDefs; + SmallSet<unsigned, 8> LocalDefSet; + SmallSet<unsigned, 8> DeadDefSet; + SmallSet<unsigned, 8> KilledDefSet; + SmallVector<unsigned, 8> ExternUses; + SmallSet<unsigned, 8> ExternUseSet; + SmallSet<unsigned, 8> KilledUseSet; + SmallSet<unsigned, 8> UndefUseSet; + SmallVector<MachineOperand*, 4> Defs; + do { + for (unsigned i = 0, e = FirstMI->getNumOperands(); i != e; ++i) { + MachineOperand &MO = FirstMI->getOperand(i); + if (!MO.isReg()) + continue; + if (MO.isDef()) { + Defs.push_back(&MO); + continue; + } + + unsigned Reg = MO.getReg(); + if (!Reg) + continue; + assert(TargetRegisterInfo::isPhysicalRegister(Reg)); + if (LocalDefSet.count(Reg)) { + MO.setIsInternalRead(); + if (MO.isKill()) + // Internal def is now killed. + KilledDefSet.insert(Reg); + } else { + if (ExternUseSet.insert(Reg)) { + ExternUses.push_back(Reg); + if (MO.isUndef()) + UndefUseSet.insert(Reg); + } + if (MO.isKill()) + // External def is now killed. + KilledUseSet.insert(Reg); + } + } + + for (unsigned i = 0, e = Defs.size(); i != e; ++i) { + MachineOperand &MO = *Defs[i]; + unsigned Reg = MO.getReg(); + if (!Reg) + continue; + + if (LocalDefSet.insert(Reg)) { + LocalDefs.push_back(Reg); + if (MO.isDead()) { + DeadDefSet.insert(Reg); + } + } else { + // Re-defined inside the bundle, it's no longer killed. + KilledDefSet.erase(Reg); + if (!MO.isDead()) + // Previously defined but dead. + DeadDefSet.erase(Reg); + } + + if (!MO.isDead()) { + for (const unsigned *SubRegs = TRI->getSubRegisters(Reg); + unsigned SubReg = *SubRegs; ++SubRegs) { + if (LocalDefSet.insert(SubReg)) + LocalDefs.push_back(SubReg); + } + } + } + + FirstMI->setIsInsideBundle(); + Defs.clear(); + } while (FirstMI++ != LastMI); + + SmallSet<unsigned, 8> Added; + for (unsigned i = 0, e = LocalDefs.size(); i != e; ++i) { + unsigned Reg = LocalDefs[i]; + if (Added.insert(Reg)) { + // If it's not live beyond end of the bundle, mark it dead. + bool isDead = DeadDefSet.count(Reg) || KilledDefSet.count(Reg); + MIB.addReg(Reg, getDefRegState(true) | getDeadRegState(isDead) | + getImplRegState(true)); + } + } + + for (unsigned i = 0, e = ExternUses.size(); i != e; ++i) { + unsigned Reg = ExternUses[i]; + bool isKill = KilledUseSet.count(Reg); + bool isUndef = UndefUseSet.count(Reg); + MIB.addReg(Reg, getKillRegState(isKill) | getUndefRegState(isUndef) | + getImplRegState(true)); + } +} diff --git a/lib/CodeGen/MachineVerifier.cpp b/lib/CodeGen/MachineVerifier.cpp index 1abc61b..0a2c2f8 100644 --- a/lib/CodeGen/MachineVerifier.cpp +++ b/lib/CodeGen/MachineVerifier.cpp @@ -279,13 +279,17 @@ bool MachineVerifier::runOnMachineFunction(MachineFunction &MF) { for (MachineFunction::const_iterator MFI = MF.begin(), MFE = MF.end(); MFI!=MFE; ++MFI) { visitMachineBasicBlockBefore(MFI); - for (MachineBasicBlock::const_iterator MBBI = MFI->begin(), - MBBE = MFI->end(); MBBI != MBBE; ++MBBI) { + for (MachineBasicBlock::const_instr_iterator MBBI = MFI->instr_begin(), + MBBE = MFI->instr_end(); MBBI != MBBE; ++MBBI) { if (MBBI->getParent() != MFI) { report("Bad instruction parent pointer", MFI); *OS << "Instruction: " << *MBBI; continue; } + // Skip BUNDLE instruction for now. FIXME: We should add code to verify + // the BUNDLE's specifically. + if (MBBI->isBundle()) + continue; visitMachineInstrBefore(MBBI); for (unsigned I = 0, E = MBBI->getNumOperands(); I != E; ++I) visitMachineOperand(&MBBI->getOperand(I), I); diff --git a/lib/CodeGen/PostRASchedulerList.cpp b/lib/CodeGen/PostRASchedulerList.cpp index 126a368..fa832c8 100644 --- a/lib/CodeGen/PostRASchedulerList.cpp +++ b/lib/CodeGen/PostRASchedulerList.cpp @@ -212,7 +212,8 @@ bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) { RegClassInfo.runOnMachineFunction(Fn); // Check for explicit enable/disable of post-ra scheduling. - TargetSubtargetInfo::AntiDepBreakMode AntiDepMode = TargetSubtargetInfo::ANTIDEP_NONE; + TargetSubtargetInfo::AntiDepBreakMode AntiDepMode = + TargetSubtargetInfo::ANTIDEP_NONE; SmallVector<TargetRegisterClass*, 4> CriticalPathRCs; if (EnablePostRAScheduler.getPosition() > 0) { if (!EnablePostRAScheduler) @@ -271,6 +272,8 @@ bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) { } I = MI; --Count; + if (MI->isBundle()) + Count -= MI->getBundleSize(); } assert(Count == 0 && "Instruction count mismatch!"); assert((MBB->begin() == Current || CurrentCount != 0) && diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp index 2bb173b..aedc2a1 100644 --- a/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -277,8 +277,9 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) { continue; if (DefSU != SU && (Kind != SDep::Output || !MO.isDead() || - !DefSU->getInstr()->registerDefIsDead(Reg))) + !DefSU->getInstr()->registerDefIsDead(Reg))) { DefSU->addPred(SDep(SU, Kind, AOLatency, /*Reg=*/Reg)); + } } for (const unsigned *Alias = TRI->getAliasSet(Reg); *Alias; ++Alias) { std::vector<SUnit *> &MemDefList = Defs[*Alias]; @@ -657,22 +658,16 @@ std::string ScheduleDAGInstrs::getGraphNodeLabel(const SUnit *SU) const { // EmitSchedule - Emit the machine code in scheduled order. MachineBasicBlock *ScheduleDAGInstrs::EmitSchedule() { - // For MachineInstr-based scheduling, we're rescheduling the instructions in - // the block, so start by removing them from the block. - while (Begin != InsertPos) { - MachineBasicBlock::iterator I = Begin; - ++Begin; - BB->remove(I); - } + Begin = InsertPos; // If first instruction was a DBG_VALUE then put it back. if (FirstDbgValue) - BB->insert(InsertPos, FirstDbgValue); + BB->splice(InsertPos, BB, FirstDbgValue); // Then re-insert them according to the given schedule. for (unsigned i = 0, e = Sequence.size(); i != e; i++) { if (SUnit *SU = Sequence[i]) - BB->insert(InsertPos, SU->getInstr()); + BB->splice(InsertPos, BB, SU->getInstr()); else // Null SUnit* is a noop. EmitNoop(); @@ -689,7 +684,7 @@ MachineBasicBlock *ScheduleDAGInstrs::EmitSchedule() { std::pair<MachineInstr *, MachineInstr *> P = *prior(DI); MachineInstr *DbgValue = P.first; MachineBasicBlock::iterator OrigPrivMI = P.second; - BB->insertAfter(OrigPrivMI, DbgValue); + BB->splice(++OrigPrivMI, BB, DbgValue); } DbgValues.clear(); FirstDbgValue = NULL; diff --git a/lib/CodeGen/TargetInstrInfoImpl.cpp b/lib/CodeGen/TargetInstrInfoImpl.cpp index ccbda98..72daabf 100644 --- a/lib/CodeGen/TargetInstrInfoImpl.cpp +++ b/lib/CodeGen/TargetInstrInfoImpl.cpp @@ -121,7 +121,7 @@ MachineInstr *TargetInstrInfoImpl::commuteInstruction(MachineInstr *MI, bool TargetInstrInfoImpl::findCommutedOpIndices(MachineInstr *MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const { - assert(MI->getOpcode() != TargetOpcode::BUNDLE && + assert(!MI->isBundle() && "TargetInstrInfoImpl::findCommutedOpIndices() can't handle bundles"); const MCInstrDesc &MCID = MI->getDesc(); @@ -156,7 +156,7 @@ bool TargetInstrInfoImpl::PredicateInstruction(MachineInstr *MI, const SmallVectorImpl<MachineOperand> &Pred) const { bool MadeChange = false; - assert(MI->getOpcode() != TargetOpcode::BUNDLE && + assert(!MI->isBundle() && "TargetInstrInfoImpl::PredicateInstruction() can't handle bundles"); const MCInstrDesc &MCID = MI->getDesc(); diff --git a/lib/Target/ARM/ARMBaseInstrInfo.cpp b/lib/Target/ARM/ARMBaseInstrInfo.cpp index b0dc172..5ee2dc8 100644 --- a/lib/Target/ARM/ARMBaseInstrInfo.cpp +++ b/lib/Target/ARM/ARMBaseInstrInfo.cpp @@ -439,6 +439,22 @@ ReverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const { return false; } +bool ARMBaseInstrInfo::isPredicated(const MachineInstr *MI) const { + if (MI->isBundle()) { + MachineBasicBlock::const_instr_iterator I = MI; + MachineBasicBlock::const_instr_iterator E = MI->getParent()->instr_end(); + while (++I != E && I->isInsideBundle()) { + int PIdx = I->findFirstPredOperandIdx(); + if (PIdx != -1 && I->getOperand(PIdx).getImm() != ARMCC::AL) + return true; + } + return false; + } + + int PIdx = MI->findFirstPredOperandIdx(); + return PIdx != -1 && MI->getOperand(PIdx).getImm() != ARMCC::AL; +} + bool ARMBaseInstrInfo:: PredicateInstruction(MachineInstr *MI, const SmallVectorImpl<MachineOperand> &Pred) const { @@ -547,7 +563,7 @@ unsigned ARMBaseInstrInfo::GetInstSizeInBytes(const MachineInstr *MI) const { return getInlineAsmLength(MI->getOperand(0).getSymbolName(), *MAI); if (MI->isLabel()) return 0; - unsigned Opc = MI->getOpcode(); + unsigned Opc = MI->getOpcode(); switch (Opc) { case TargetOpcode::IMPLICIT_DEF: case TargetOpcode::KILL: @@ -555,6 +571,8 @@ unsigned ARMBaseInstrInfo::GetInstSizeInBytes(const MachineInstr *MI) const { case TargetOpcode::EH_LABEL: case TargetOpcode::DBG_VALUE: return 0; + case TargetOpcode::BUNDLE: + return getInstBundleLength(MI); case ARM::MOVi16_ga_pcrel: case ARM::MOVTi16_ga_pcrel: case ARM::t2MOVi16_ga_pcrel: @@ -621,6 +639,17 @@ unsigned ARMBaseInstrInfo::GetInstSizeInBytes(const MachineInstr *MI) const { return 0; // Not reached } +unsigned ARMBaseInstrInfo::getInstBundleLength(const MachineInstr *MI) const { + unsigned Size = 0; + MachineBasicBlock::const_instr_iterator I = MI; + MachineBasicBlock::const_instr_iterator E = MI->getParent()->instr_end(); + while (++I != E && I->isInsideBundle()) { + assert(!I->isBundle() && "No nested bundle!"); + Size += GetInstSizeInBytes(&*I); + } + return Size; +} + void ARMBaseInstrInfo::copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, DebugLoc DL, unsigned DestReg, unsigned SrcReg, @@ -1955,7 +1984,7 @@ bool ARMBaseInstrInfo::FoldImmediate(MachineInstr *UseMI, bool isKill = UseMI->getOperand(OpIdx).isKill(); unsigned NewReg = MRI->createVirtualRegister(MRI->getRegClass(Reg)); AddDefaultCC(AddDefaultPred(BuildMI(*UseMI->getParent(), - *UseMI, UseMI->getDebugLoc(), + UseMI, UseMI->getDebugLoc(), get(NewUseOpc), NewReg) .addReg(Reg1, getKillRegState(isKill)) .addImm(SOImmValV1))); @@ -2330,6 +2359,57 @@ ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, return UseCycle; } +static const MachineInstr *getBundledDefMI(const TargetRegisterInfo *TRI, + const MachineInstr *MI, + unsigned &DefIdx, unsigned &Dist) { + Dist = 0; + + MachineBasicBlock::const_iterator I = MI; ++I; + MachineBasicBlock::const_instr_iterator II = + llvm::prior(I.getInstrIterator()); + assert(II->isInsideBundle() && "Empty bundle?"); + + int Idx = -1; + unsigned Reg = MI->getOperand(DefIdx).getReg(); + while (II->isInsideBundle()) { + Idx = II->findRegisterDefOperandIdx(Reg, false, true, TRI); + if (Idx != -1) + break; + --II; + ++Dist; + } + + assert(Idx != -1 && "Cannot find bundled definition!"); + DefIdx = Idx; + return II; +} + +static const MachineInstr *getBundledUseMI(const TargetRegisterInfo *TRI, + const MachineInstr *MI, + unsigned &UseIdx, unsigned &Dist) { + Dist = 0; + + MachineBasicBlock::const_instr_iterator II = MI; ++II; + assert(II->isInsideBundle() && "Empty bundle?"); + MachineBasicBlock::const_instr_iterator E = MI->getParent()->instr_end(); + + // FIXME: This doesn't properly handle multiple uses. + int Idx = -1; + unsigned Reg = MI->getOperand(UseIdx).getReg(); + while (II != E && II->isInsideBundle()) { + Idx = II->findRegisterUseOperandIdx(Reg, false, TRI); + if (Idx != -1) + break; + if (II->getOpcode() != ARM::t2IT) + ++Dist; + ++II; + } + + assert(Idx != -1 && "Cannot find bundled definition!"); + UseIdx = Idx; + return II; +} + int ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, const MachineInstr *DefMI, unsigned DefIdx, @@ -2341,8 +2421,8 @@ ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, if (!ItinData || ItinData->isEmpty()) return DefMI->mayLoad() ? 3 : 1; - const MCInstrDesc &DefMCID = DefMI->getDesc(); - const MCInstrDesc &UseMCID = UseMI->getDesc(); + const MCInstrDesc *DefMCID = &DefMI->getDesc(); + const MCInstrDesc *UseMCID = &UseMI->getDesc(); const MachineOperand &DefMO = DefMI->getOperand(DefIdx); if (DefMO.getReg() == ARM::CPSR) { if (DefMI->getOpcode() == ARM::FMSTAT) { @@ -2353,20 +2433,50 @@ ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, // CPSR set and branch can be paired in the same cycle. if (UseMI->isBranch()) return 0; + + // Otherwise it takes the instruction latency (generally one). + int Latency = getInstrLatency(ItinData, DefMI); + // For Thumb2, prefer scheduling CPSR setting instruction close to its uses. + // Instructions which are otherwise scheduled between them may incur a code + // size penalty (not able to use the CPSR setting 16-bit instructions). + if (Latency > 0 && Subtarget.isThumb2()) + --Latency; + return Latency; } unsigned DefAlign = DefMI->hasOneMemOperand() ? (*DefMI->memoperands_begin())->getAlignment() : 0; unsigned UseAlign = UseMI->hasOneMemOperand() ? (*UseMI->memoperands_begin())->getAlignment() : 0; - int Latency = getOperandLatency(ItinData, DefMCID, DefIdx, DefAlign, - UseMCID, UseIdx, UseAlign); + + unsigned DefAdj = 0; + if (DefMI->isBundle()) { + DefMI = getBundledDefMI(&getRegisterInfo(), DefMI, DefIdx, DefAdj); + if (DefMI->isCopyLike() || DefMI->isInsertSubreg() || + DefMI->isRegSequence() || DefMI->isImplicitDef()) + return 1; + DefMCID = &DefMI->getDesc(); + } + unsigned UseAdj = 0; + if (UseMI->isBundle()) { + UseMI = getBundledUseMI(&getRegisterInfo(), UseMI, UseIdx, UseAdj); + UseMCID = &UseMI->getDesc(); + } + + int Latency = getOperandLatency(ItinData, *DefMCID, DefIdx, DefAlign, + *UseMCID, UseIdx, UseAlign); + int Adj = DefAdj + UseAdj; + if (Adj) { + Latency -= (int)(DefAdj + UseAdj); + if (Latency < 1) + return 1; + } if (Latency > 1 && (Subtarget.isCortexA8() || Subtarget.isCortexA9())) { // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2] // variants are one cycle cheaper. - switch (DefMCID.getOpcode()) { + switch (DefMCID->getOpcode()) { default: break; case ARM::LDRrs: case ARM::LDRBrs: { @@ -2391,7 +2501,7 @@ ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, } if (DefAlign < 8 && Subtarget.isCortexA9()) - switch (DefMCID.getOpcode()) { + switch (DefMCID->getOpcode()) { default: break; case ARM::VLD1q8: case ARM::VLD1q16: @@ -2697,6 +2807,17 @@ int ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, if (!ItinData || ItinData->isEmpty()) return 1; + if (MI->isBundle()) { + int Latency = 0; + MachineBasicBlock::const_instr_iterator I = MI; + MachineBasicBlock::const_instr_iterator E = MI->getParent()->instr_end(); + while (++I != E && I->isInsideBundle()) { + if (I->getOpcode() != ARM::t2IT) + Latency += getInstrLatency(ItinData, I, PredCost); + } + return Latency; + } + const MCInstrDesc &MCID = MI->getDesc(); unsigned Class = MCID.getSchedClass(); unsigned UOps = ItinData->Itineraries[Class].NumMicroOps; diff --git a/lib/Target/ARM/ARMBaseInstrInfo.h b/lib/Target/ARM/ARMBaseInstrInfo.h index 0f9f321..4ce7461 100644 --- a/lib/Target/ARM/ARMBaseInstrInfo.h +++ b/lib/Target/ARM/ARMBaseInstrInfo.h @@ -69,10 +69,7 @@ public: bool ReverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const; // Predication support. - bool isPredicated(const MachineInstr *MI) const { - int PIdx = MI->findFirstPredOperandIdx(); - return PIdx != -1 && MI->getOperand(PIdx).getImm() != ARMCC::AL; - } + bool isPredicated(const MachineInstr *MI) const; ARMCC::CondCodes getPredicate(const MachineInstr *MI) const { int PIdx = MI->findFirstPredOperandIdx(); @@ -219,6 +216,8 @@ public: void setExecutionDomain(MachineInstr *MI, unsigned Domain) const; private: + unsigned getInstBundleLength(const MachineInstr *MI) const; + int getVLDMDefCycle(const InstrItineraryData *ItinData, const MCInstrDesc &DefMCID, unsigned DefClass, diff --git a/lib/Target/ARM/ARMFastISel.cpp b/lib/Target/ARM/ARMFastISel.cpp index 0a4faa2..5cb24ad 100644 --- a/lib/Target/ARM/ARMFastISel.cpp +++ b/lib/Target/ARM/ARMFastISel.cpp @@ -702,7 +702,7 @@ unsigned ARMFastISel::TargetMaterializeAlloca(const AllocaInst *AI) { TargetRegisterClass* RC = TLI.getRegClassFor(VT); unsigned ResultReg = createResultReg(RC); unsigned Opc = isThumb2 ? ARM::t2ADDri : ARM::ADDri; - AddOptionalDefs(BuildMI(*FuncInfo.MBB, *FuncInfo.InsertPt, DL, + AddOptionalDefs(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(Opc), ResultReg) .addFrameIndex(SI->second) .addImm(0)); @@ -898,7 +898,7 @@ void ARMFastISel::ARMSimplifyAddress(Address &Addr, EVT VT, bool useAM3) { ARM::GPRRegisterClass; unsigned ResultReg = createResultReg(RC); unsigned Opc = isThumb2 ? ARM::t2ADDri : ARM::ADDri; - AddOptionalDefs(BuildMI(*FuncInfo.MBB, *FuncInfo.InsertPt, DL, + AddOptionalDefs(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(Opc), ResultReg) .addFrameIndex(Addr.Base.FI) .addImm(0)); diff --git a/lib/Target/ARM/ARMHazardRecognizer.cpp b/lib/Target/ARM/ARMHazardRecognizer.cpp index 2393544..a5fd15b 100644 --- a/lib/Target/ARM/ARMHazardRecognizer.cpp +++ b/lib/Target/ARM/ARMHazardRecognizer.cpp @@ -38,9 +38,6 @@ ARMHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { MachineInstr *MI = SU->getInstr(); if (!MI->isDebugValue()) { - if (ITBlockSize && MI != ITBlockMIs[ITBlockSize-1]) - return Hazard; - // Look for special VMLA / VMLS hazards. A VMUL / VADD / VSUB following // a VMLA / VMLS will cause 4 cycle stall. const MCInstrDesc &MCID = MI->getDesc(); @@ -76,30 +73,11 @@ ARMHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { void ARMHazardRecognizer::Reset() { LastMI = 0; FpMLxStalls = 0; - ITBlockSize = 0; ScoreboardHazardRecognizer::Reset(); } void ARMHazardRecognizer::EmitInstruction(SUnit *SU) { MachineInstr *MI = SU->getInstr(); - unsigned Opcode = MI->getOpcode(); - if (ITBlockSize) { - --ITBlockSize; - } else if (Opcode == ARM::t2IT) { - unsigned Mask = MI->getOperand(1).getImm(); - unsigned NumTZ = CountTrailingZeros_32(Mask); - assert(NumTZ <= 3 && "Invalid IT mask!"); - ITBlockSize = 4 - NumTZ; - MachineBasicBlock::iterator I = MI; - for (unsigned i = 0; i < ITBlockSize; ++i) { - // Advance to the next instruction, skipping any dbg_value instructions. - do { - ++I; - } while (I->isDebugValue()); - ITBlockMIs[ITBlockSize-1-i] = &*I; - } - } - if (!MI->isDebugValue()) { LastMI = MI; FpMLxStalls = 0; diff --git a/lib/Target/ARM/ARMHazardRecognizer.h b/lib/Target/ARM/ARMHazardRecognizer.h index 053f874..98bfc4c 100644 --- a/lib/Target/ARM/ARMHazardRecognizer.h +++ b/lib/Target/ARM/ARMHazardRecognizer.h @@ -34,8 +34,6 @@ class ARMHazardRecognizer : public ScoreboardHazardRecognizer { MachineInstr *LastMI; unsigned FpMLxStalls; - unsigned ITBlockSize; // No. of MIs in current IT block yet to be scheduled. - MachineInstr *ITBlockMIs[4]; public: ARMHazardRecognizer(const InstrItineraryData *ItinData, @@ -44,7 +42,7 @@ public: const ARMSubtarget &sti, const ScheduleDAG *DAG) : ScoreboardHazardRecognizer(ItinData, DAG, "post-RA-sched"), TII(tii), - TRI(tri), STI(sti), LastMI(0), ITBlockSize(0) {} + TRI(tri), STI(sti), LastMI(0) {} virtual HazardType getHazardType(SUnit *SU, int Stalls); virtual void Reset(); diff --git a/lib/Target/ARM/ARMTargetMachine.cpp b/lib/Target/ARM/ARMTargetMachine.cpp index 963cc47..61b75cb 100644 --- a/lib/Target/ARM/ARMTargetMachine.cpp +++ b/lib/Target/ARM/ARMTargetMachine.cpp @@ -147,10 +147,16 @@ bool ARMBaseTargetMachine::addPreSched2(PassManagerBase &PM) { } bool ARMBaseTargetMachine::addPreEmitPass(PassManagerBase &PM) { - if (Subtarget.isThumb2() && !Subtarget.prefers32BitThumb()) - PM.add(createThumb2SizeReductionPass()); + if (Subtarget.isThumb2()) { + if (!Subtarget.prefers32BitThumb()) + PM.add(createThumb2SizeReductionPass()); + + // Constant island pass work on unbundled instructions. + PM.add(createUnpackMachineBundlesPass()); + } PM.add(createARMConstantIslandPass()); + return true; } diff --git a/lib/Target/ARM/MLxExpansionPass.cpp b/lib/Target/ARM/MLxExpansionPass.cpp index a187f8e..000a37f 100644 --- a/lib/Target/ARM/MLxExpansionPass.cpp +++ b/lib/Target/ARM/MLxExpansionPass.cpp @@ -222,14 +222,14 @@ MLxExpansion::ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI, const MCInstrDesc &MCID2 = TII->get(AddSubOpc); unsigned TmpReg = MRI->createVirtualRegister(TII->getRegClass(MCID1, 0, TRI)); - MachineInstrBuilder MIB = BuildMI(MBB, *MI, MI->getDebugLoc(), MCID1, TmpReg) + MachineInstrBuilder MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID1, TmpReg) .addReg(Src1Reg, getKillRegState(Src1Kill)) .addReg(Src2Reg, getKillRegState(Src2Kill)); if (HasLane) MIB.addImm(LaneImm); MIB.addImm(Pred).addReg(PredReg); - MIB = BuildMI(MBB, *MI, MI->getDebugLoc(), MCID2) + MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID2) .addReg(DstReg, getDefRegState(true) | getDeadRegState(DstDead)); if (NegAcc) { diff --git a/lib/Target/ARM/Thumb2ITBlockPass.cpp b/lib/Target/ARM/Thumb2ITBlockPass.cpp index d1acb6f..55b4d30 100644 --- a/lib/Target/ARM/Thumb2ITBlockPass.cpp +++ b/lib/Target/ARM/Thumb2ITBlockPass.cpp @@ -13,6 +13,7 @@ #include "Thumb2InstrInfo.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineInstrBundle.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" @@ -237,6 +238,9 @@ bool Thumb2ITBlockPass::InsertITInstructions(MachineBasicBlock &MBB) { // Last instruction in IT block kills ITSTATE. LastITMI->findRegisterUseOperand(ARM::ITSTATE)->setIsKill(); + // Finalize the bundle. + FinalizeBundle(MBB, InsertPos.getInstrIterator(), LastITMI); + Modified = true; ++NumITs; } diff --git a/lib/Target/ARM/Thumb2SizeReduction.cpp b/lib/Target/ARM/Thumb2SizeReduction.cpp index c0e3ac6..e206288 100644 --- a/lib/Target/ARM/Thumb2SizeReduction.cpp +++ b/lib/Target/ARM/Thumb2SizeReduction.cpp @@ -452,7 +452,7 @@ Thumb2SizeReduce::ReduceLoadStore(MachineBasicBlock &MBB, MachineInstr *MI, // Add the 16-bit load / store instruction. DebugLoc dl = MI->getDebugLoc(); - MachineInstrBuilder MIB = BuildMI(MBB, *MI, dl, TII->get(Opc)); + MachineInstrBuilder MIB = BuildMI(MBB, MI, dl, TII->get(Opc)); if (!isLdStMul) { MIB.addOperand(MI->getOperand(0)); MIB.addOperand(MI->getOperand(1)); @@ -478,7 +478,7 @@ Thumb2SizeReduce::ReduceLoadStore(MachineBasicBlock &MBB, MachineInstr *MI, DEBUG(errs() << "Converted 32-bit: " << *MI << " to 16-bit: " << *MIB); - MBB.erase(MI); + MBB.erase_instr(MI); ++NumLdSts; return true; } @@ -513,7 +513,7 @@ Thumb2SizeReduce::ReduceSpecial(MachineBasicBlock &MBB, MachineInstr *MI, MI->getOperand(MCID.getNumOperands()-1).getReg() == ARM::CPSR) return false; - MachineInstrBuilder MIB = BuildMI(MBB, *MI, MI->getDebugLoc(), + MachineInstrBuilder MIB = BuildMI(MBB, MI, MI->getDebugLoc(), TII->get(ARM::tADDrSPi)) .addOperand(MI->getOperand(0)) .addOperand(MI->getOperand(1)) @@ -525,7 +525,7 @@ Thumb2SizeReduce::ReduceSpecial(MachineBasicBlock &MBB, MachineInstr *MI, DEBUG(errs() << "Converted 32-bit: " << *MI << " to 16-bit: " <<*MIB); - MBB.erase(MI); + MBB.erase_instr(MI); ++NumNarrows; return true; } @@ -653,7 +653,7 @@ Thumb2SizeReduce::ReduceTo2Addr(MachineBasicBlock &MBB, MachineInstr *MI, // Add the 16-bit instruction. DebugLoc dl = MI->getDebugLoc(); - MachineInstrBuilder MIB = BuildMI(MBB, *MI, dl, NewMCID); + MachineInstrBuilder MIB = BuildMI(MBB, MI, dl, NewMCID); MIB.addOperand(MI->getOperand(0)); if (NewMCID.hasOptionalDef()) { if (HasCC) @@ -677,7 +677,7 @@ Thumb2SizeReduce::ReduceTo2Addr(MachineBasicBlock &MBB, MachineInstr *MI, DEBUG(errs() << "Converted 32-bit: " << *MI << " to 16-bit: " << *MIB); - MBB.erase(MI); + MBB.erase_instr(MI); ++Num2Addrs; return true; } @@ -744,7 +744,7 @@ Thumb2SizeReduce::ReduceToNarrow(MachineBasicBlock &MBB, MachineInstr *MI, // Add the 16-bit instruction. DebugLoc dl = MI->getDebugLoc(); - MachineInstrBuilder MIB = BuildMI(MBB, *MI, dl, NewMCID); + MachineInstrBuilder MIB = BuildMI(MBB, MI, dl, NewMCID); MIB.addOperand(MI->getOperand(0)); if (NewMCID.hasOptionalDef()) { if (HasCC) @@ -784,7 +784,7 @@ Thumb2SizeReduce::ReduceToNarrow(MachineBasicBlock &MBB, MachineInstr *MI, DEBUG(errs() << "Converted 32-bit: " << *MI << " to 16-bit: " << *MIB); - MBB.erase(MI); + MBB.erase_instr(MI); ++NumNarrows; return true; } @@ -829,16 +829,22 @@ bool Thumb2SizeReduce::ReduceMBB(MachineBasicBlock &MBB) { // Yes, CPSR could be livein. bool LiveCPSR = MBB.isLiveIn(ARM::CPSR); MachineInstr *CPSRDef = 0; + MachineInstr *BundleMI = 0; // If this BB loops back to itself, conservatively avoid narrowing the // first instruction that does partial flag update. bool IsSelfLoop = MBB.isSuccessor(&MBB); - MachineBasicBlock::iterator MII = MBB.begin(), E = MBB.end(); - MachineBasicBlock::iterator NextMII; + MachineBasicBlock::instr_iterator MII = MBB.instr_begin(), E = MBB.instr_end(); + MachineBasicBlock::instr_iterator NextMII; for (; MII != E; MII = NextMII) { NextMII = llvm::next(MII); MachineInstr *MI = &*MII; + if (MI->isBundle()) { + BundleMI = MI; + continue; + } + LiveCPSR = UpdateCPSRUse(*MI, LiveCPSR); unsigned Opcode = MI->getOpcode(); @@ -849,7 +855,7 @@ bool Thumb2SizeReduce::ReduceMBB(MachineBasicBlock &MBB) { if (Entry.Special) { if (ReduceSpecial(MBB, MI, Entry, LiveCPSR, CPSRDef, IsSelfLoop)) { Modified = true; - MachineBasicBlock::iterator I = prior(NextMII); + MachineBasicBlock::instr_iterator I = prior(NextMII); MI = &*I; } goto ProcessNext; @@ -859,7 +865,7 @@ bool Thumb2SizeReduce::ReduceMBB(MachineBasicBlock &MBB) { if (Entry.NarrowOpc2 && ReduceTo2Addr(MBB, MI, Entry, LiveCPSR, CPSRDef, IsSelfLoop)) { Modified = true; - MachineBasicBlock::iterator I = prior(NextMII); + MachineBasicBlock::instr_iterator I = prior(NextMII); MI = &*I; goto ProcessNext; } @@ -868,12 +874,21 @@ bool Thumb2SizeReduce::ReduceMBB(MachineBasicBlock &MBB) { if (Entry.NarrowOpc1 && ReduceToNarrow(MBB, MI, Entry, LiveCPSR, CPSRDef, IsSelfLoop)) { Modified = true; - MachineBasicBlock::iterator I = prior(NextMII); + MachineBasicBlock::instr_iterator I = prior(NextMII); MI = &*I; } } ProcessNext: + if (LiveCPSR && + NextMII != E && MI->isInsideBundle() && !NextMII->isInsideBundle() && + BundleMI->killsRegister(ARM::CPSR)) + // FIXME: Since post-ra scheduler operates on bundles, the CPSR kill + // marker is only on the BUNDLE instruction. Process the BUNDLE + // instruction as we finish with the bundled instruction to work around + // the inconsistency. + LiveCPSR = false; + bool DefCPSR = false; LiveCPSR = UpdateCPSRDef(*MI, LiveCPSR, DefCPSR); if (MI->isCall()) { diff --git a/lib/Target/X86/X86InstrInfo.cpp b/lib/Target/X86/X86InstrInfo.cpp index 37ac7f2..7d1b9a1 100644 --- a/lib/Target/X86/X86InstrInfo.cpp +++ b/lib/Target/X86/X86InstrInfo.cpp @@ -1528,9 +1528,9 @@ X86InstrInfo::convertToThreeAddressWithLEA(unsigned MIOpc, leaInReg2 = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass); // Build and insert into an implicit UNDEF value. This is OK because // well be shifting and then extracting the lower 16-bits. - BuildMI(*MFI, MIB, MI->getDebugLoc(), get(X86::IMPLICIT_DEF), leaInReg2); + BuildMI(*MFI, &*MIB, MI->getDebugLoc(), get(X86::IMPLICIT_DEF),leaInReg2); InsMI2 = - BuildMI(*MFI, MIB, MI->getDebugLoc(), get(TargetOpcode::COPY)) + BuildMI(*MFI, &*MIB, MI->getDebugLoc(), get(TargetOpcode::COPY)) .addReg(leaInReg2, RegState::Define, X86::sub_16bit) .addReg(Src2, getKillRegState(isKill2)); addRegReg(MIB, leaInReg, true, leaInReg2, true); diff --git a/test/CodeGen/ARM/debug-info-d16-reg.ll b/test/CodeGen/ARM/debug-info-d16-reg.ll index 8c9095e..325eea0 100644 --- a/test/CodeGen/ARM/debug-info-d16-reg.ll +++ b/test/CodeGen/ARM/debug-info-d16-reg.ll @@ -1,4 +1,4 @@ -; RUN: llc < %s - | FileCheck %s +; RUN: llc < %s | FileCheck %s ; Radar 9309221 ; Test dwarf reg no for d16 ;CHECK: DW_OP_regx diff --git a/test/CodeGen/ARM/long_shift.ll b/test/CodeGen/ARM/long_shift.ll index d5aac2e..a99a7ec 100644 --- a/test/CodeGen/ARM/long_shift.ll +++ b/test/CodeGen/ARM/long_shift.ll @@ -25,8 +25,8 @@ define i32 @f2(i64 %x, i64 %y) { ; CHECK: lsr{{.*}}r2 ; CHECK-NEXT: rsb r3, r2, #32 ; CHECK-NEXT: sub r2, r2, #32 -; CHECK-NEXT: cmp r2, #0 ; CHECK-NEXT: orr r0, r0, r1, lsl r3 +; CHECK-NEXT: cmp r2, #0 ; CHECK-NEXT: asrge r0, r1, r2 %a = ashr i64 %x, %y %b = trunc i64 %a to i32 @@ -38,8 +38,8 @@ define i32 @f3(i64 %x, i64 %y) { ; CHECK: lsr{{.*}}r2 ; CHECK-NEXT: rsb r3, r2, #32 ; CHECK-NEXT: sub r2, r2, #32 -; CHECK-NEXT: cmp r2, #0 ; CHECK-NEXT: orr r0, r0, r1, lsl r3 +; CHECK-NEXT: cmp r2, #0 ; CHECK-NEXT: lsrge r0, r1, r2 %a = lshr i64 %x, %y %b = trunc i64 %a to i32 diff --git a/test/CodeGen/Thumb2/machine-licm.ll b/test/CodeGen/Thumb2/machine-licm.ll index 4e16c9a..8285742 100644 --- a/test/CodeGen/Thumb2/machine-licm.ll +++ b/test/CodeGen/Thumb2/machine-licm.ll @@ -94,8 +94,8 @@ bb.nph: bb: ; preds = %bb, %bb.nph ; CHECK: bb -; CHECK: eor.w {{(r[0-9])|(lr)}}, {{(r[0-9])|(lr)}}, [[REGISTER]] ; CHECK: eor.w +; CHECK: eor.w {{(r[0-9])|(lr)}}, {{(r[0-9])|(lr)}}, [[REGISTER]] ; CHECK-NOT: eor ; CHECK: and %data_addr.013 = phi i8 [ %data, %bb.nph ], [ %8, %bb ] ; <i8> [#uses=2] diff --git a/test/CodeGen/Thumb2/thumb2-select_xform.ll b/test/CodeGen/Thumb2/thumb2-select_xform.ll index ceefabb..74729fd 100644 --- a/test/CodeGen/Thumb2/thumb2-select_xform.ll +++ b/test/CodeGen/Thumb2/thumb2-select_xform.ll @@ -3,8 +3,8 @@ define i32 @t1(i32 %a, i32 %b, i32 %c) nounwind { ; CHECK: t1 ; CHECK: mvn r0, #-2147483648 -; CHECK: add r0, r1 ; CHECK: cmp r2, #10 +; CHECK: add r0, r1 ; CHECK: it gt ; CHECK: movgt r0, r1 %tmp1 = icmp sgt i32 %c, 10 |