diff options
Diffstat (limited to 'lib/CodeGen')
55 files changed, 2441 insertions, 3732 deletions
diff --git a/lib/CodeGen/AggressiveAntiDepBreaker.cpp b/lib/CodeGen/AggressiveAntiDepBreaker.cpp index c23351b..6f45216 100644 --- a/lib/CodeGen/AggressiveAntiDepBreaker.cpp +++ b/lib/CodeGen/AggressiveAntiDepBreaker.cpp @@ -404,7 +404,7 @@ void AggressiveAntiDepBreaker::PrescanInstruction(MachineInstr *MI, // Note register reference... const TargetRegisterClass *RC = NULL; if (i < MI->getDesc().getNumOperands()) - RC = MI->getDesc().OpInfo[i].getRegClass(TRI); + RC = TII->getRegClass(MI->getDesc(), i, TRI); AggressiveAntiDepState::RegisterReference RR = { &MO, RC }; RegRefs.insert(std::make_pair(Reg, RR)); } @@ -479,7 +479,7 @@ void AggressiveAntiDepBreaker::ScanInstruction(MachineInstr *MI, // Note register reference... const TargetRegisterClass *RC = NULL; if (i < MI->getDesc().getNumOperands()) - RC = MI->getDesc().OpInfo[i].getRegClass(TRI); + RC = TII->getRegClass(MI->getDesc(), i, TRI); AggressiveAntiDepState::RegisterReference RR = { &MO, RC }; RegRefs.insert(std::make_pair(Reg, RR)); } diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index bfee679..2cdf272 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -584,6 +584,8 @@ static bool EmitDebugValueComment(const MachineInstr *MI, AsmPrinter &AP) { } } else if (MI->getOperand(0).isImm()) { OS << MI->getOperand(0).getImm(); + } else if (MI->getOperand(0).isCImm()) { + MI->getOperand(0).getCImm()->getValue().print(OS, false /*isSigned*/); } else { assert(MI->getOperand(0).isReg() && "Unknown operand type"); if (MI->getOperand(0).getReg() == 0) { diff --git a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp index bff1a35..1fe035e 100644 --- a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp @@ -491,7 +491,7 @@ bool CompileUnit::addConstantFPValue(DIE *Die, const MachineOperand &MO) { } /// addConstantValue - Add constant value entry in variable DIE. -bool CompileUnit::addConstantValue(DIE *Die, ConstantInt *CI, +bool CompileUnit::addConstantValue(DIE *Die, const ConstantInt *CI, bool Unsigned) { unsigned CIBitWidth = CI->getBitWidth(); if (CIBitWidth <= 64) { diff --git a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h index 60a9b28..213c7fc 100644 --- a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h +++ b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h @@ -181,7 +181,7 @@ public: /// addConstantValue - Add constant value entry in variable DIE. bool addConstantValue(DIE *Die, const MachineOperand &MO, DIType Ty); - bool addConstantValue(DIE *Die, ConstantInt *CI, bool Unsigned); + bool addConstantValue(DIE *Die, const ConstantInt *CI, bool Unsigned); /// addConstantFPValue - Add constant value entry in variable DIE. bool addConstantFPValue(DIE *Die, const MachineOperand &MO); diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp index 8845bfa..f85a82d 100644 --- a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp @@ -618,6 +618,21 @@ DIE *DwarfDebug::constructInlinedScopeDIE(DbgScope *Scope) { return ScopeDIE; } +/// isUnsignedDIType - Return true if type encoding is unsigned. +static bool isUnsignedDIType(DIType Ty) { + DIDerivedType DTy(Ty); + if (DTy.Verify()) + return isUnsignedDIType(DTy.getTypeDerivedFrom()); + + DIBasicType BTy(Ty); + if (BTy.Verify()) { + unsigned Encoding = BTy.getEncoding(); + if (Encoding == dwarf::DW_ATE_unsigned || + Encoding == dwarf::DW_ATE_unsigned_char) + return true; + } + return false; +} /// constructVariableDIE - Construct a DIE for the given DbgVariable. DIE *DwarfDebug::constructVariableDIE(DbgVariable *DV, DbgScope *Scope) { @@ -718,6 +733,11 @@ DIE *DwarfDebug::constructVariableDIE(DbgVariable *DV, DbgScope *Scope) { else if (DVInsn->getOperand(0).isFPImm()) updated = VariableCU->addConstantFPValue(VariableDie, DVInsn->getOperand(0)); + else if (DVInsn->getOperand(0).isCImm()) + updated = + VariableCU->addConstantValue(VariableDie, + DVInsn->getOperand(0).getCImm(), + isUnsignedDIType(DV->getType())); } else { VariableCU->addVariableAddress(DV, VariableDie, Asm->getDebugValueLocation(DVInsn)); @@ -913,22 +933,6 @@ CompileUnit *DwarfDebug::getCompileUnit(const MDNode *N) const { return I->second; } -/// isUnsignedDIType - Return true if type encoding is unsigned. -static bool isUnsignedDIType(DIType Ty) { - DIDerivedType DTy(Ty); - if (DTy.Verify()) - return isUnsignedDIType(DTy.getTypeDerivedFrom()); - - DIBasicType BTy(Ty); - if (BTy.Verify()) { - unsigned Encoding = BTy.getEncoding(); - if (Encoding == dwarf::DW_ATE_unsigned || - Encoding == dwarf::DW_ATE_unsigned_char) - return true; - } - return false; -} - // Return const exprssion if value is a GEP to access merged global // constant. e.g. // i8* getelementptr ({ i8, i8, i8, i8 }* @_MergedGlobals, i32 0, i32 0) @@ -1017,7 +1021,7 @@ void DwarfDebug::constructGlobalVariableDIE(const MDNode *N) { } else { TheCU->addBlock(VariableDIE, dwarf::DW_AT_location, 0, Block); } - } else if (ConstantInt *CI = + } else if (const ConstantInt *CI = dyn_cast_or_null<ConstantInt>(GV.getConstant())) TheCU->addConstantValue(VariableDIE, CI, isUnsignedDIType(GTy)); else if (const ConstantExpr *CE = getMergedGlobalExpr(N->getOperand(11))) { diff --git a/lib/CodeGen/BranchFolding.cpp b/lib/CodeGen/BranchFolding.cpp index d95f77e..4df7b46 100644 --- a/lib/CodeGen/BranchFolding.cpp +++ b/lib/CodeGen/BranchFolding.cpp @@ -421,10 +421,10 @@ static unsigned EstimateRuntime(MachineBasicBlock::iterator I, for (; I != E; ++I) { if (I->isDebugValue()) continue; - const TargetInstrDesc &TID = I->getDesc(); - if (TID.isCall()) + const MCInstrDesc &MCID = I->getDesc(); + if (MCID.isCall()) Time += 10; - else if (TID.mayLoad() || TID.mayStore()) + else if (MCID.mayLoad() || MCID.mayStore()) Time += 2; else ++Time; diff --git a/lib/CodeGen/CMakeLists.txt b/lib/CodeGen/CMakeLists.txt index aef4ff2..92319c8 100644 --- a/lib/CodeGen/CMakeLists.txt +++ b/lib/CodeGen/CMakeLists.txt @@ -59,7 +59,6 @@ add_llvm_library(LLVMCodeGen Passes.cpp PeepholeOptimizer.cpp PostRASchedulerList.cpp - PreAllocSplitting.cpp ProcessImplicitDefs.cpp PrologEpilogInserter.cpp PseudoSourceValue.cpp @@ -79,7 +78,6 @@ add_llvm_library(LLVMCodeGen ScoreboardHazardRecognizer.cpp ShadowStackGC.cpp ShrinkWrapping.cpp - SimpleRegisterCoalescing.cpp SjLjEHPrepare.cpp SlotIndexes.cpp Spiller.cpp diff --git a/lib/CodeGen/CalcSpillWeights.cpp b/lib/CodeGen/CalcSpillWeights.cpp index 5d722ee..e6b3bbc 100644 --- a/lib/CodeGen/CalcSpillWeights.cpp +++ b/lib/CodeGen/CalcSpillWeights.cpp @@ -188,6 +188,7 @@ void VirtRegAuxInfo::CalculateWeightAndHint(LiveInterval &li) { void VirtRegAuxInfo::CalculateRegClass(unsigned reg) { MachineRegisterInfo &MRI = MF.getRegInfo(); + const TargetInstrInfo *TII = MF.getTarget().getInstrInfo(); const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo(); const TargetRegisterClass *OldRC = MRI.getRegClass(reg); const TargetRegisterClass *NewRC = TRI->getLargestLegalSuperClass(OldRC); @@ -202,8 +203,11 @@ void VirtRegAuxInfo::CalculateRegClass(unsigned reg) { // TRI doesn't have accurate enough information to model this yet. if (I.getOperand().getSubReg()) return; + // Inline asm instuctions don't remember their constraints. + if (I->isInlineAsm()) + return; const TargetRegisterClass *OpRC = - I->getDesc().getRegClass(I.getOperandNo(), TRI); + TII->getRegClass(I->getDesc(), I.getOperandNo(), TRI); if (OpRC) NewRC = getCommonSubClass(NewRC, OpRC); if (!NewRC || NewRC == OldRC) diff --git a/lib/CodeGen/CodeGen.cpp b/lib/CodeGen/CodeGen.cpp index 515e6f9..489746c 100644 --- a/lib/CodeGen/CodeGen.cpp +++ b/lib/CodeGen/CodeGen.cpp @@ -37,13 +37,11 @@ void llvm::initializeCodeGen(PassRegistry &Registry) { initializeOptimizePHIsPass(Registry); initializePHIEliminationPass(Registry); initializePeepholeOptimizerPass(Registry); - initializePreAllocSplittingPass(Registry); initializeProcessImplicitDefsPass(Registry); initializePEIPass(Registry); initializeRALinScanPass(Registry); - initializeRegisterCoalescerAnalysisGroup(Registry); + initializeRegisterCoalescerPass(Registry); initializeRenderMachineFunctionPass(Registry); - initializeSimpleRegisterCoalescingPass(Registry); initializeSlotIndexesPass(Registry); initializeLoopSplitterPass(Registry); initializeStackProtectorPass(Registry); diff --git a/lib/CodeGen/CriticalAntiDepBreaker.cpp b/lib/CodeGen/CriticalAntiDepBreaker.cpp index 4cac453..84c4d59 100644 --- a/lib/CodeGen/CriticalAntiDepBreaker.cpp +++ b/lib/CodeGen/CriticalAntiDepBreaker.cpp @@ -207,7 +207,7 @@ void CriticalAntiDepBreaker::PrescanInstruction(MachineInstr *MI) { const TargetRegisterClass *NewRC = 0; if (i < MI->getDesc().getNumOperands()) - NewRC = MI->getDesc().OpInfo[i].getRegClass(TRI); + NewRC = TII->getRegClass(MI->getDesc(), i, TRI); // For now, only allow the register to be changed if its register // class is consistent across all uses. @@ -295,7 +295,7 @@ void CriticalAntiDepBreaker::ScanInstruction(MachineInstr *MI, const TargetRegisterClass *NewRC = 0; if (i < MI->getDesc().getNumOperands()) - NewRC = MI->getDesc().OpInfo[i].getRegClass(TRI); + NewRC = TII->getRegClass(MI->getDesc(), i, TRI); // For now, only allow the register to be changed if its register // class is consistent across all uses. diff --git a/lib/CodeGen/DeadMachineInstructionElim.cpp b/lib/CodeGen/DeadMachineInstructionElim.cpp index fdc1d91..6de6c0c 100644 --- a/lib/CodeGen/DeadMachineInstructionElim.cpp +++ b/lib/CodeGen/DeadMachineInstructionElim.cpp @@ -110,9 +110,14 @@ bool DeadMachineInstructionElim::runOnMachineFunction(MachineFunction &MF) { LivePhysRegs.set(Reg); } - // FIXME: Add live-ins from sucessors to LivePhysRegs. Normally, physregs - // are not live across blocks, but some targets (x86) can have flags live - // out of a block. + // Add live-ins from sucessors to LivePhysRegs. Normally, physregs are not + // live across blocks, but some targets (x86) can have flags live out of a + // block. + for (MachineBasicBlock::succ_iterator S = MBB->succ_begin(), + E = MBB->succ_end(); S != E; S++) + for (MachineBasicBlock::livein_iterator LI = (*S)->livein_begin(); + LI != (*S)->livein_end(); LI++) + LivePhysRegs.set(*LI); // Now scan the instructions and delete dead ones, tracking physreg // liveness as we go. diff --git a/lib/CodeGen/ExpandISelPseudos.cpp b/lib/CodeGen/ExpandISelPseudos.cpp index ebc2fc9..a67140e 100644 --- a/lib/CodeGen/ExpandISelPseudos.cpp +++ b/lib/CodeGen/ExpandISelPseudos.cpp @@ -62,8 +62,8 @@ bool ExpandISelPseudos::runOnMachineFunction(MachineFunction &MF) { MachineInstr *MI = MBBI++; // If MI is a pseudo, expand it. - const TargetInstrDesc &TID = MI->getDesc(); - if (TID.usesCustomInsertionHook()) { + const MCInstrDesc &MCID = MI->getDesc(); + if (MCID.usesCustomInsertionHook()) { Changed = true; MachineBasicBlock *NewMBB = TLI->EmitInstrWithCustomInserter(MI, MBB); diff --git a/lib/CodeGen/IfConversion.cpp b/lib/CodeGen/IfConversion.cpp index 8b2c981..c918bf6 100644 --- a/lib/CodeGen/IfConversion.cpp +++ b/lib/CodeGen/IfConversion.cpp @@ -18,8 +18,8 @@ #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/MC/MCInstrItineraries.h" #include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetInstrItineraries.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetRegisterInfo.h" @@ -651,12 +651,12 @@ void IfConverter::ScanInstructions(BBInfo &BBI) { if (I->isDebugValue()) continue; - const TargetInstrDesc &TID = I->getDesc(); - if (TID.isNotDuplicable()) + const MCInstrDesc &MCID = I->getDesc(); + if (MCID.isNotDuplicable()) BBI.CannotBeCopied = true; bool isPredicated = TII->isPredicated(I); - bool isCondBr = BBI.IsBrAnalyzable && TID.isConditionalBranch(); + bool isCondBr = BBI.IsBrAnalyzable && MCID.isConditionalBranch(); if (!isCondBr) { if (!isPredicated) { @@ -1414,9 +1414,9 @@ void IfConverter::CopyAndPredicateBlock(BBInfo &ToBBI, BBInfo &FromBBI, for (MachineBasicBlock::iterator I = FromBBI.BB->begin(), E = FromBBI.BB->end(); I != E; ++I) { - const TargetInstrDesc &TID = I->getDesc(); + const MCInstrDesc &MCID = I->getDesc(); // Do not copy the end of the block branches. - if (IgnoreBr && TID.isBranch()) + if (IgnoreBr && MCID.isBranch()) break; MachineInstr *MI = MF.CloneMachineInstr(I); diff --git a/lib/CodeGen/InlineSpiller.cpp b/lib/CodeGen/InlineSpiller.cpp index 19ae333..0273891 100644 --- a/lib/CodeGen/InlineSpiller.cpp +++ b/lib/CodeGen/InlineSpiller.cpp @@ -180,11 +180,7 @@ Spiller *createInlineSpiller(MachineFunctionPass &pass, /// isFullCopyOf - If MI is a COPY to or from Reg, return the other register, /// otherwise return 0. static unsigned isFullCopyOf(const MachineInstr *MI, unsigned Reg) { - if (!MI->isCopy()) - return 0; - if (MI->getOperand(0).getSubReg() != 0) - return 0; - if (MI->getOperand(1).getSubReg() != 0) + if (!MI->isFullCopy()) return 0; if (MI->getOperand(0).getReg() == Reg) return MI->getOperand(1).getReg(); diff --git a/lib/CodeGen/MachineBasicBlock.cpp b/lib/CodeGen/MachineBasicBlock.cpp index c0f71d2..8f0fb46 100644 --- a/lib/CodeGen/MachineBasicBlock.cpp +++ b/lib/CodeGen/MachineBasicBlock.cpp @@ -22,7 +22,6 @@ #include "llvm/MC/MCContext.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetData.h" -#include "llvm/Target/TargetInstrDesc.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Assembly/Writer.h" diff --git a/lib/CodeGen/MachineCSE.cpp b/lib/CodeGen/MachineCSE.cpp index f97ccf6..3a60a37 100644 --- a/lib/CodeGen/MachineCSE.cpp +++ b/lib/CodeGen/MachineCSE.cpp @@ -260,12 +260,12 @@ bool MachineCSE::isCSECandidate(MachineInstr *MI) { return false; // Ignore stuff that we obviously can't move. - const TargetInstrDesc &TID = MI->getDesc(); - if (TID.mayStore() || TID.isCall() || TID.isTerminator() || + const MCInstrDesc &MCID = MI->getDesc(); + if (MCID.mayStore() || MCID.isCall() || MCID.isTerminator() || MI->hasUnmodeledSideEffects()) return false; - if (TID.mayLoad()) { + if (MCID.mayLoad()) { // Okay, this instruction does a load. As a refinement, we allow the target // to decide whether the loaded value is actually a constant. If so, we can // actually use it as a load. diff --git a/lib/CodeGen/MachineFunction.cpp b/lib/CodeGen/MachineFunction.cpp index 50750a5..cd25156 100644 --- a/lib/CodeGen/MachineFunction.cpp +++ b/lib/CodeGen/MachineFunction.cpp @@ -152,10 +152,10 @@ void MachineFunction::RenumberBlocks(MachineBasicBlock *MBB) { /// of `new MachineInstr'. /// MachineInstr * -MachineFunction::CreateMachineInstr(const TargetInstrDesc &TID, +MachineFunction::CreateMachineInstr(const MCInstrDesc &MCID, DebugLoc DL, bool NoImp) { return new (InstructionRecycler.Allocate<MachineInstr>(Allocator)) - MachineInstr(TID, DL, NoImp); + MachineInstr(MCID, DL, NoImp); } /// CloneMachineInstr - Create a new MachineInstr which is a copy of the diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp index 36b0b83..0995106 100644 --- a/lib/CodeGen/MachineInstr.cpp +++ b/lib/CodeGen/MachineInstr.cpp @@ -24,10 +24,10 @@ #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/MC/MCInstrDesc.h" #include "llvm/MC/MCSymbol.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetInstrDesc.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/DebugInfo.h" @@ -267,6 +267,9 @@ void MachineOperand::print(raw_ostream &OS, const TargetMachine *TM) const { case MachineOperand::MO_Immediate: OS << getImm(); break; + case MachineOperand::MO_CImmediate: + getCImm()->getValue().print(OS, false); + break; case MachineOperand::MO_FPImmediate: if (getFPImm()->getType()->isFloatTy()) OS << getFPImm()->getValueAPF().convertToFloat(); @@ -454,9 +457,9 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const MachineMemOperand &MMO) { //===----------------------------------------------------------------------===// /// MachineInstr ctor - This constructor creates a dummy MachineInstr with -/// TID NULL and no operands. +/// MCID NULL and no operands. MachineInstr::MachineInstr() - : TID(0), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), + : MCID(0), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), MemRefs(0), MemRefsEnd(0), Parent(0) { // Make sure that we get added to a machine basicblock @@ -464,23 +467,23 @@ MachineInstr::MachineInstr() } void MachineInstr::addImplicitDefUseOperands() { - if (TID->ImplicitDefs) - for (const unsigned *ImpDefs = TID->ImplicitDefs; *ImpDefs; ++ImpDefs) + if (MCID->ImplicitDefs) + for (const unsigned *ImpDefs = MCID->ImplicitDefs; *ImpDefs; ++ImpDefs) addOperand(MachineOperand::CreateReg(*ImpDefs, true, true)); - if (TID->ImplicitUses) - for (const unsigned *ImpUses = TID->ImplicitUses; *ImpUses; ++ImpUses) + if (MCID->ImplicitUses) + for (const unsigned *ImpUses = MCID->ImplicitUses; *ImpUses; ++ImpUses) addOperand(MachineOperand::CreateReg(*ImpUses, false, true)); } /// MachineInstr ctor - This constructor creates a MachineInstr and adds the /// implicit operands. It reserves space for the number of operands specified by -/// the TargetInstrDesc. -MachineInstr::MachineInstr(const TargetInstrDesc &tid, bool NoImp) - : TID(&tid), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), +/// the MCInstrDesc. +MachineInstr::MachineInstr(const MCInstrDesc &tid, bool NoImp) + : MCID(&tid), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), MemRefs(0), MemRefsEnd(0), Parent(0) { if (!NoImp) - NumImplicitOps = TID->getNumImplicitDefs() + TID->getNumImplicitUses(); - Operands.reserve(NumImplicitOps + TID->getNumOperands()); + NumImplicitOps = MCID->getNumImplicitDefs() + MCID->getNumImplicitUses(); + Operands.reserve(NumImplicitOps + MCID->getNumOperands()); if (!NoImp) addImplicitDefUseOperands(); // Make sure that we get added to a machine basicblock @@ -488,13 +491,13 @@ MachineInstr::MachineInstr(const TargetInstrDesc &tid, bool NoImp) } /// MachineInstr ctor - As above, but with a DebugLoc. -MachineInstr::MachineInstr(const TargetInstrDesc &tid, const DebugLoc dl, +MachineInstr::MachineInstr(const MCInstrDesc &tid, const DebugLoc dl, bool NoImp) - : TID(&tid), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), + : MCID(&tid), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), MemRefs(0), MemRefsEnd(0), Parent(0), debugLoc(dl) { if (!NoImp) - NumImplicitOps = TID->getNumImplicitDefs() + TID->getNumImplicitUses(); - Operands.reserve(NumImplicitOps + TID->getNumOperands()); + NumImplicitOps = MCID->getNumImplicitDefs() + MCID->getNumImplicitUses(); + Operands.reserve(NumImplicitOps + MCID->getNumOperands()); if (!NoImp) addImplicitDefUseOperands(); // Make sure that we get added to a machine basicblock @@ -504,12 +507,12 @@ MachineInstr::MachineInstr(const TargetInstrDesc &tid, const DebugLoc dl, /// MachineInstr ctor - Work exactly the same as the ctor two above, except /// that the MachineInstr is created and added to the end of the specified /// basic block. -MachineInstr::MachineInstr(MachineBasicBlock *MBB, const TargetInstrDesc &tid) - : TID(&tid), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), +MachineInstr::MachineInstr(MachineBasicBlock *MBB, const MCInstrDesc &tid) + : MCID(&tid), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), MemRefs(0), MemRefsEnd(0), Parent(0) { assert(MBB && "Cannot use inserting ctor with null basic block!"); - NumImplicitOps = TID->getNumImplicitDefs() + TID->getNumImplicitUses(); - Operands.reserve(NumImplicitOps + TID->getNumOperands()); + NumImplicitOps = MCID->getNumImplicitDefs() + MCID->getNumImplicitUses(); + Operands.reserve(NumImplicitOps + MCID->getNumOperands()); addImplicitDefUseOperands(); // Make sure that we get added to a machine basicblock LeakDetector::addGarbageObject(this); @@ -519,12 +522,12 @@ MachineInstr::MachineInstr(MachineBasicBlock *MBB, const TargetInstrDesc &tid) /// MachineInstr ctor - As above, but with a DebugLoc. /// MachineInstr::MachineInstr(MachineBasicBlock *MBB, const DebugLoc dl, - const TargetInstrDesc &tid) - : TID(&tid), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), + const MCInstrDesc &tid) + : MCID(&tid), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), MemRefs(0), MemRefsEnd(0), Parent(0), debugLoc(dl) { assert(MBB && "Cannot use inserting ctor with null basic block!"); - NumImplicitOps = TID->getNumImplicitDefs() + TID->getNumImplicitUses(); - Operands.reserve(NumImplicitOps + TID->getNumOperands()); + NumImplicitOps = MCID->getNumImplicitDefs() + MCID->getNumImplicitUses(); + Operands.reserve(NumImplicitOps + MCID->getNumOperands()); addImplicitDefUseOperands(); // Make sure that we get added to a machine basicblock LeakDetector::addGarbageObject(this); @@ -534,7 +537,7 @@ MachineInstr::MachineInstr(MachineBasicBlock *MBB, const DebugLoc dl, /// MachineInstr ctor - Copies MachineInstr arg exactly /// MachineInstr::MachineInstr(MachineFunction &MF, const MachineInstr &MI) - : TID(&MI.getDesc()), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), + : MCID(&MI.getDesc()), NumImplicitOps(0), Flags(0), AsmPrinterFlags(0), MemRefs(MI.MemRefs), MemRefsEnd(MI.MemRefsEnd), Parent(0), debugLoc(MI.getDebugLoc()) { Operands.reserve(MI.getNumOperands()); @@ -621,7 +624,7 @@ void MachineInstr::addOperand(const MachineOperand &Op) { Operands.back().AddRegOperandToRegInfo(RegInfo); // If the register operand is flagged as early, mark the operand as such unsigned OpNo = Operands.size() - 1; - if (TID->getOperandConstraint(OpNo, TOI::EARLY_CLOBBER) != -1) + if (MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1) Operands[OpNo].setIsEarlyClobber(true); } return; @@ -643,7 +646,7 @@ void MachineInstr::addOperand(const MachineOperand &Op) { if (Operands[OpNo].isReg()) { Operands[OpNo].AddRegOperandToRegInfo(0); // If the register operand is flagged as early, mark the operand as such - if (TID->getOperandConstraint(OpNo, TOI::EARLY_CLOBBER) != -1) + if (MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1) Operands[OpNo].setIsEarlyClobber(true); } @@ -668,7 +671,7 @@ void MachineInstr::addOperand(const MachineOperand &Op) { if (Operands[OpNo].isReg()) { Operands[OpNo].AddRegOperandToRegInfo(RegInfo); // If the register operand is flagged as early, mark the operand as such - if (TID->getOperandConstraint(OpNo, TOI::EARLY_CLOBBER) != -1) + if (MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1) Operands[OpNo].setIsEarlyClobber(true); } @@ -691,7 +694,7 @@ void MachineInstr::addOperand(const MachineOperand &Op) { // If the register operand is flagged as early, mark the operand as such if (Operands[OpNo].isReg() - && TID->getOperandConstraint(OpNo, TOI::EARLY_CLOBBER) != -1) + && MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1) Operands[OpNo].setIsEarlyClobber(true); } } @@ -817,8 +820,8 @@ void MachineInstr::eraseFromParent() { /// OperandComplete - Return true if it's illegal to add a new operand /// bool MachineInstr::OperandsComplete() const { - unsigned short NumOperands = TID->getNumOperands(); - if (!TID->isVariadic() && getNumOperands()-NumImplicitOps >= NumOperands) + unsigned short NumOperands = MCID->getNumOperands(); + if (!MCID->isVariadic() && getNumOperands()-NumImplicitOps >= NumOperands) return true; // Broken: we have all the operands of this instruction! return false; } @@ -826,8 +829,8 @@ bool MachineInstr::OperandsComplete() const { /// getNumExplicitOperands - Returns the number of non-implicit operands. /// unsigned MachineInstr::getNumExplicitOperands() const { - unsigned NumOperands = TID->getNumOperands(); - if (!TID->isVariadic()) + unsigned NumOperands = MCID->getNumOperands(); + if (!MCID->isVariadic()) return NumOperands; for (unsigned i = NumOperands, e = getNumOperands(); i != e; ++i) { @@ -928,10 +931,10 @@ MachineInstr::findRegisterDefOperandIdx(unsigned Reg, bool isDead, bool Overlap, /// operand list that is used to represent the predicate. It returns -1 if /// none is found. int MachineInstr::findFirstPredOperandIdx() const { - const TargetInstrDesc &TID = getDesc(); - if (TID.isPredicable()) { + const MCInstrDesc &MCID = getDesc(); + if (MCID.isPredicable()) { for (unsigned i = 0, e = getNumOperands(); i != e; ++i) - if (TID.OpInfo[i].isPredicate()) + if (MCID.OpInfo[i].isPredicate()) return i; } @@ -987,11 +990,11 @@ isRegTiedToUseOperand(unsigned DefOpIdx, unsigned *UseOpIdx) const { } assert(getOperand(DefOpIdx).isDef() && "DefOpIdx is not a def!"); - const TargetInstrDesc &TID = getDesc(); - for (unsigned i = 0, e = TID.getNumOperands(); i != e; ++i) { + const MCInstrDesc &MCID = getDesc(); + for (unsigned i = 0, e = MCID.getNumOperands(); i != e; ++i) { const MachineOperand &MO = getOperand(i); if (MO.isReg() && MO.isUse() && - TID.getOperandConstraint(i, TOI::TIED_TO) == (int)DefOpIdx) { + MCID.getOperandConstraint(i, MCOI::TIED_TO) == (int)DefOpIdx) { if (UseOpIdx) *UseOpIdx = (unsigned)i; return true; @@ -1047,13 +1050,13 @@ isRegTiedToDefOperand(unsigned UseOpIdx, unsigned *DefOpIdx) const { return false; } - const TargetInstrDesc &TID = getDesc(); - if (UseOpIdx >= TID.getNumOperands()) + const MCInstrDesc &MCID = getDesc(); + if (UseOpIdx >= MCID.getNumOperands()) return false; const MachineOperand &MO = getOperand(UseOpIdx); if (!MO.isReg() || !MO.isUse()) return false; - int DefIdx = TID.getOperandConstraint(UseOpIdx, TOI::TIED_TO); + int DefIdx = MCID.getOperandConstraint(UseOpIdx, MCOI::TIED_TO); if (DefIdx == -1) return false; if (DefOpIdx) @@ -1093,11 +1096,11 @@ void MachineInstr::copyKillDeadInfo(const MachineInstr *MI) { /// copyPredicates - Copies predicate operand(s) from MI. void MachineInstr::copyPredicates(const MachineInstr *MI) { - const TargetInstrDesc &TID = MI->getDesc(); - if (!TID.isPredicable()) + const MCInstrDesc &MCID = MI->getDesc(); + if (!MCID.isPredicable()) return; for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { - if (TID.OpInfo[i].isPredicate()) { + if (MCID.OpInfo[i].isPredicate()) { // Predicated operands must be last operands. addOperand(MI->getOperand(i)); } @@ -1134,13 +1137,13 @@ bool MachineInstr::isSafeToMove(const TargetInstrInfo *TII, AliasAnalysis *AA, bool &SawStore) const { // Ignore stuff that we obviously can't move. - if (TID->mayStore() || TID->isCall()) { + if (MCID->mayStore() || MCID->isCall()) { SawStore = true; return false; } if (isLabel() || isDebugValue() || - TID->isTerminator() || hasUnmodeledSideEffects()) + MCID->isTerminator() || hasUnmodeledSideEffects()) return false; // See if this instruction does a load. If so, we have to guarantee that the @@ -1148,7 +1151,7 @@ bool MachineInstr::isSafeToMove(const TargetInstrInfo *TII, // destination. The check for isInvariantLoad gives the targe the chance to // classify the load as always returning a constant, e.g. a constant pool // load. - if (TID->mayLoad() && !isInvariantLoad(AA)) + if (MCID->mayLoad() && !isInvariantLoad(AA)) // Otherwise, this is a real load. If there is a store between the load and // end of block, or if the load is volatile, we can't move it. return !SawStore && !hasVolatileMemoryRef(); @@ -1188,9 +1191,9 @@ bool MachineInstr::isSafeToReMat(const TargetInstrInfo *TII, /// have no volatile memory references. bool MachineInstr::hasVolatileMemoryRef() const { // An instruction known never to access memory won't have a volatile access. - if (!TID->mayStore() && - !TID->mayLoad() && - !TID->isCall() && + if (!MCID->mayStore() && + !MCID->mayLoad() && + !MCID->isCall() && !hasUnmodeledSideEffects()) return false; @@ -1214,7 +1217,7 @@ bool MachineInstr::hasVolatileMemoryRef() const { /// *all* loads the instruction does are invariant (if it does multiple loads). bool MachineInstr::isInvariantLoad(AliasAnalysis *AA) const { // If the instruction doesn't load at all, it isn't an invariant load. - if (!TID->mayLoad()) + if (!MCID->mayLoad()) return false; // If the instruction has lost its memoperands, conservatively assume that @@ -1364,6 +1367,8 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const { // Print the rest of the operands. bool OmittedAnyCallClobbers = false; bool FirstOp = true; + unsigned AsmDescOp = ~0u; + unsigned AsmOpCount = 0; if (isInlineAsm()) { // Print asm string. @@ -1377,7 +1382,7 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const { if (ExtraInfo & InlineAsm::Extra_IsAlignStack) OS << " [alignstack]"; - StartOp = InlineAsm::MIOp_FirstOperand; + StartOp = AsmDescOp = InlineAsm::MIOp_FirstOperand; FirstOp = false; } @@ -1416,10 +1421,10 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const { if (FirstOp) FirstOp = false; else OS << ","; OS << " "; if (i < getDesc().NumOperands) { - const TargetOperandInfo &TOI = getDesc().OpInfo[i]; - if (TOI.isPredicate()) + const MCOperandInfo &MCOI = getDesc().OpInfo[i]; + if (MCOI.isPredicate()) OS << "pred:"; - if (TOI.isOptionalDef()) + if (MCOI.isOptionalDef()) OS << "opt:"; } if (isDebugValue() && MO.isMetadata()) { @@ -1431,6 +1436,26 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const { MO.print(OS, TM); } else if (TM && (isInsertSubreg() || isRegSequence()) && MO.isImm()) { OS << TM->getRegisterInfo()->getSubRegIndexName(MO.getImm()); + } else if (i == AsmDescOp && MO.isImm()) { + // Pretty print the inline asm operand descriptor. + OS << '$' << AsmOpCount++; + unsigned Flag = MO.getImm(); + switch (InlineAsm::getKind(Flag)) { + case InlineAsm::Kind_RegUse: OS << ":[reguse]"; break; + case InlineAsm::Kind_RegDef: OS << ":[regdef]"; break; + case InlineAsm::Kind_RegDefEarlyClobber: OS << ":[regdef-ec]"; break; + case InlineAsm::Kind_Clobber: OS << ":[clobber]"; break; + case InlineAsm::Kind_Imm: OS << ":[imm]"; break; + case InlineAsm::Kind_Mem: OS << ":[mem]"; break; + default: OS << ":[??" << InlineAsm::getKind(Flag) << ']'; break; + } + + unsigned TiedTo = 0; + if (InlineAsm::isUseOperandTiedToDef(Flag, TiedTo)) + OS << " [tiedto:$" << TiedTo << ']'; + + // Compute the index of the next operand descriptor. + AsmDescOp += 1 + InlineAsm::getNumOperandRegisters(Flag); } else MO.print(OS, TM); } diff --git a/lib/CodeGen/MachineLICM.cpp b/lib/CodeGen/MachineLICM.cpp index b315702..722ceb2 100644 --- a/lib/CodeGen/MachineLICM.cpp +++ b/lib/CodeGen/MachineLICM.cpp @@ -28,10 +28,10 @@ #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/MC/MCInstrItineraries.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetInstrItineraries.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/ADT/DenseMap.h" @@ -1018,9 +1018,9 @@ MachineInstr *MachineLICM::ExtractHoistableLoad(MachineInstr *MI) { /*UnfoldStore=*/false, &LoadRegIndex); if (NewOpc == 0) return 0; - const TargetInstrDesc &TID = TII->get(NewOpc); - if (TID.getNumDefs() != 1) return 0; - const TargetRegisterClass *RC = TID.OpInfo[LoadRegIndex].getRegClass(TRI); + const MCInstrDesc &MID = TII->get(NewOpc); + if (MID.getNumDefs() != 1) return 0; + const TargetRegisterClass *RC = TII->getRegClass(MID, LoadRegIndex, TRI); // Ok, we're unfolding. Create a temporary register and do the unfold. unsigned Reg = MRI->createVirtualRegister(RC); diff --git a/lib/CodeGen/MachineRegisterInfo.cpp b/lib/CodeGen/MachineRegisterInfo.cpp index 08ff5bb..4b3e64c 100644 --- a/lib/CodeGen/MachineRegisterInfo.cpp +++ b/lib/CodeGen/MachineRegisterInfo.cpp @@ -20,7 +20,6 @@ using namespace llvm; MachineRegisterInfo::MachineRegisterInfo(const TargetRegisterInfo &TRI) { VRegInfo.reserve(256); RegAllocHints.reserve(256); - RegClass2VRegMap = new std::vector<unsigned>[TRI.getNumRegClasses()]; UsedPhysRegs.resize(TRI.getNumRegs()); // Create the physreg use/def lists. @@ -38,25 +37,13 @@ MachineRegisterInfo::~MachineRegisterInfo() { "PhysRegUseDefLists has entries after all instructions are deleted"); #endif delete [] PhysRegUseDefLists; - delete [] RegClass2VRegMap; } /// setRegClass - Set the register class of the specified virtual register. /// void MachineRegisterInfo::setRegClass(unsigned Reg, const TargetRegisterClass *RC) { - const TargetRegisterClass *OldRC = VRegInfo[Reg].first; VRegInfo[Reg].first = RC; - - // Remove from old register class's vregs list. This may be slow but - // fortunately this operation is rarely needed. - std::vector<unsigned> &VRegs = RegClass2VRegMap[OldRC->getID()]; - std::vector<unsigned>::iterator I = - std::find(VRegs.begin(), VRegs.end(), Reg); - VRegs.erase(I); - - // Add to new register class's vregs list. - RegClass2VRegMap[RC->getID()].push_back(Reg); } const TargetRegisterClass * @@ -95,7 +82,6 @@ MachineRegisterInfo::createVirtualRegister(const TargetRegisterClass *RegClass){ if (ArrayBase && &VRegInfo[FirstVirtReg] != ArrayBase) // The vector reallocated, handle this now. HandleVRegListReallocation(); - RegClass2VRegMap[RegClass->getID()].push_back(Reg); return Reg; } diff --git a/lib/CodeGen/MachineVerifier.cpp b/lib/CodeGen/MachineVerifier.cpp index 471463b..7a55852 100644 --- a/lib/CodeGen/MachineVerifier.cpp +++ b/lib/CodeGen/MachineVerifier.cpp @@ -62,6 +62,7 @@ namespace { raw_ostream *OS; const MachineFunction *MF; const TargetMachine *TM; + const TargetInstrInfo *TII; const TargetRegisterInfo *TRI; const MachineRegisterInfo *MRI; @@ -255,6 +256,7 @@ bool MachineVerifier::runOnMachineFunction(MachineFunction &MF) { this->MF = &MF; TM = &MF.getTarget(); + TII = TM->getInstrInfo(); TRI = TM->getRegisterInfo(); MRI = &MF.getRegInfo(); @@ -387,8 +389,6 @@ static bool matchPair(MachineBasicBlock::const_succ_iterator i, void MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) { - const TargetInstrInfo *TII = MF->getTarget().getInstrInfo(); - // Count the number of landing pad successors. SmallPtrSet<MachineBasicBlock*, 4> LandingPadSuccs; for (MachineBasicBlock::const_succ_iterator I = MBB->succ_begin(), @@ -541,19 +541,19 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) { } void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) { - const TargetInstrDesc &TI = MI->getDesc(); - if (MI->getNumOperands() < TI.getNumOperands()) { + const MCInstrDesc &MCID = MI->getDesc(); + if (MI->getNumOperands() < MCID.getNumOperands()) { report("Too few operands", MI); - *OS << TI.getNumOperands() << " operands expected, but " + *OS << MCID.getNumOperands() << " operands expected, but " << MI->getNumExplicitOperands() << " given.\n"; } // Check the MachineMemOperands for basic consistency. for (MachineInstr::mmo_iterator I = MI->memoperands_begin(), E = MI->memoperands_end(); I != E; ++I) { - if ((*I)->isLoad() && !TI.mayLoad()) + if ((*I)->isLoad() && !MCID.mayLoad()) report("Missing mayLoad flag", MI); - if ((*I)->isStore() && !TI.mayStore()) + if ((*I)->isStore() && !MCID.mayStore()) report("Missing mayStore flag", MI); } @@ -575,29 +575,30 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) { void MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { const MachineInstr *MI = MO->getParent(); - const TargetInstrDesc &TI = MI->getDesc(); - const TargetOperandInfo &TOI = TI.OpInfo[MONum]; + const MCInstrDesc &MCID = MI->getDesc(); + const MCOperandInfo &MCOI = MCID.OpInfo[MONum]; - // The first TI.NumDefs operands must be explicit register defines - if (MONum < TI.getNumDefs()) { + // The first MCID.NumDefs operands must be explicit register defines + if (MONum < MCID.getNumDefs()) { if (!MO->isReg()) report("Explicit definition must be a register", MO, MONum); else if (!MO->isDef()) report("Explicit definition marked as use", MO, MONum); else if (MO->isImplicit()) report("Explicit definition marked as implicit", MO, MONum); - } else if (MONum < TI.getNumOperands()) { + } else if (MONum < MCID.getNumOperands()) { // Don't check if it's the last operand in a variadic instruction. See, // e.g., LDM_RET in the arm back end. - if (MO->isReg() && !(TI.isVariadic() && MONum == TI.getNumOperands()-1)) { - if (MO->isDef() && !TOI.isOptionalDef()) + if (MO->isReg() && + !(MCID.isVariadic() && MONum == MCID.getNumOperands()-1)) { + if (MO->isDef() && !MCOI.isOptionalDef()) report("Explicit operand marked as def", MO, MONum); if (MO->isImplicit()) report("Explicit operand marked as implicit", MO, MONum); } } else { // ARM adds %reg0 operands to indicate predicates. We'll allow that. - if (MO->isReg() && !MO->isImplicit() && !TI.isVariadic() && MO->getReg()) + if (MO->isReg() && !MO->isImplicit() && !MCID.isVariadic() && MO->getReg()) report("Extra explicit operand on non-variadic instruction", MO, MONum); } @@ -709,7 +710,7 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { } // Check register classes. - if (MONum < TI.getNumOperands() && !MO->isImplicit()) { + if (MONum < MCID.getNumOperands() && !MO->isImplicit()) { unsigned SubIdx = MO->getSubReg(); if (TargetRegisterInfo::isPhysicalRegister(Reg)) { @@ -723,7 +724,7 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { } sr = s; } - if (const TargetRegisterClass *DRC = TOI.getRegClass(TRI)) { + if (const TargetRegisterClass *DRC = TII->getRegClass(MCID,MONum,TRI)) { if (!DRC->contains(sr)) { report("Illegal physical register for instruction", MO, MONum); *OS << TRI->getName(sr) << " is not a " @@ -743,7 +744,7 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { } RC = SRC; } - if (const TargetRegisterClass *DRC = TOI.getRegClass(TRI)) { + if (const TargetRegisterClass *DRC = TII->getRegClass(MCID,MONum,TRI)) { if (!RC->hasSuperClassEq(DRC)) { report("Illegal virtual register for instruction", MO, MONum); *OS << "Expected a " << DRC->getName() << " register, but got a " @@ -765,11 +766,11 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { LiveInts && !LiveInts->isNotInMIMap(MI)) { LiveInterval &LI = LiveStks->getInterval(MO->getIndex()); SlotIndex Idx = LiveInts->getInstructionIndex(MI); - if (TI.mayLoad() && !LI.liveAt(Idx.getUseIndex())) { + if (MCID.mayLoad() && !LI.liveAt(Idx.getUseIndex())) { report("Instruction loads from dead spill slot", MO, MONum); *OS << "Live stack: " << LI << '\n'; } - if (TI.mayStore() && !LI.liveAt(Idx.getDefIndex())) { + if (MCID.mayStore() && !LI.liveAt(Idx.getDefIndex())) { report("Instruction stores to dead spill slot", MO, MONum); *OS << "Live stack: " << LI << '\n'; } diff --git a/lib/CodeGen/PeepholeOptimizer.cpp b/lib/CodeGen/PeepholeOptimizer.cpp index c105bb0..c523e39 100644 --- a/lib/CodeGen/PeepholeOptimizer.cpp +++ b/lib/CodeGen/PeepholeOptimizer.cpp @@ -353,10 +353,10 @@ bool PeepholeOptimizer::OptimizeCmpInstr(MachineInstr *MI, bool PeepholeOptimizer::isMoveImmediate(MachineInstr *MI, SmallSet<unsigned, 4> &ImmDefRegs, DenseMap<unsigned, MachineInstr*> &ImmDefMIs) { - const TargetInstrDesc &TID = MI->getDesc(); - if (!TID.isMoveImmediate()) + const MCInstrDesc &MCID = MI->getDesc(); + if (!MCID.isMoveImmediate()) return false; - if (TID.getNumDefs() != 1) + if (MCID.getNumDefs() != 1) return false; unsigned Reg = MI->getOperand(0).getReg(); if (TargetRegisterInfo::isVirtualRegister(Reg)) { @@ -429,16 +429,16 @@ bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) { continue; } - const TargetInstrDesc &TID = MI->getDesc(); + const MCInstrDesc &MCID = MI->getDesc(); - if (TID.isBitcast()) { + if (MCID.isBitcast()) { if (OptimizeBitcastInstr(MI, MBB)) { // MI is deleted. Changed = true; MII = First ? I->begin() : llvm::next(PMII); continue; } - } else if (TID.isCompare()) { + } else if (MCID.isCompare()) { if (OptimizeCmpInstr(MI, MBB)) { // MI is deleted. Changed = true; diff --git a/lib/CodeGen/PreAllocSplitting.cpp b/lib/CodeGen/PreAllocSplitting.cpp deleted file mode 100644 index d6e31da..0000000 --- a/lib/CodeGen/PreAllocSplitting.cpp +++ /dev/null @@ -1,1430 +0,0 @@ -//===-- PreAllocSplitting.cpp - Pre-allocation Interval Spltting Pass. ----===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file implements the machine instruction level pre-register allocation -// live interval splitting pass. It finds live interval barriers, i.e. -// instructions which will kill all physical registers in certain register -// classes, and split all live intervals which cross the barrier. -// -//===----------------------------------------------------------------------===// - -#define DEBUG_TYPE "pre-alloc-split" -#include "VirtRegMap.h" -#include "llvm/CodeGen/CalcSpillWeights.h" -#include "llvm/CodeGen/LiveIntervalAnalysis.h" -#include "llvm/CodeGen/LiveStackAnalysis.h" -#include "llvm/CodeGen/MachineDominators.h" -#include "llvm/CodeGen/MachineFrameInfo.h" -#include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineLoopInfo.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/Passes.h" -#include "llvm/CodeGen/RegisterCoalescer.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" -#include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/DepthFirstIterator.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/Statistic.h" -using namespace llvm; - -static cl::opt<int> PreSplitLimit("pre-split-limit", cl::init(-1), cl::Hidden); -static cl::opt<int> DeadSplitLimit("dead-split-limit", cl::init(-1), - cl::Hidden); -static cl::opt<int> RestoreFoldLimit("restore-fold-limit", cl::init(-1), - cl::Hidden); - -STATISTIC(NumSplits, "Number of intervals split"); -STATISTIC(NumRemats, "Number of intervals split by rematerialization"); -STATISTIC(NumFolds, "Number of intervals split with spill folding"); -STATISTIC(NumRestoreFolds, "Number of intervals split with restore folding"); -STATISTIC(NumRenumbers, "Number of intervals renumbered into new registers"); -STATISTIC(NumDeadSpills, "Number of dead spills removed"); - -namespace { - class PreAllocSplitting : public MachineFunctionPass { - MachineFunction *CurrMF; - const TargetMachine *TM; - const TargetInstrInfo *TII; - const TargetRegisterInfo* TRI; - MachineFrameInfo *MFI; - MachineRegisterInfo *MRI; - SlotIndexes *SIs; - LiveIntervals *LIs; - LiveStacks *LSs; - VirtRegMap *VRM; - - // Barrier - Current barrier being processed. - MachineInstr *Barrier; - - // BarrierMBB - Basic block where the barrier resides in. - MachineBasicBlock *BarrierMBB; - - // Barrier - Current barrier index. - SlotIndex BarrierIdx; - - // CurrLI - Current live interval being split. - LiveInterval *CurrLI; - - // CurrSLI - Current stack slot live interval. - LiveInterval *CurrSLI; - - // CurrSValNo - Current val# for the stack slot live interval. - VNInfo *CurrSValNo; - - // IntervalSSMap - A map from live interval to spill slots. - DenseMap<unsigned, int> IntervalSSMap; - - // Def2SpillMap - A map from a def instruction index to spill index. - DenseMap<SlotIndex, SlotIndex> Def2SpillMap; - - public: - static char ID; - PreAllocSplitting() : MachineFunctionPass(ID) { - initializePreAllocSplittingPass(*PassRegistry::getPassRegistry()); - } - - virtual bool runOnMachineFunction(MachineFunction &MF); - - virtual void getAnalysisUsage(AnalysisUsage &AU) const { - AU.setPreservesCFG(); - AU.addRequired<SlotIndexes>(); - AU.addPreserved<SlotIndexes>(); - AU.addRequired<LiveIntervals>(); - AU.addPreserved<LiveIntervals>(); - AU.addRequired<LiveStacks>(); - AU.addPreserved<LiveStacks>(); - AU.addPreserved<RegisterCoalescer>(); - AU.addPreserved<CalculateSpillWeights>(); - AU.addPreservedID(StrongPHIEliminationID); - AU.addPreservedID(PHIEliminationID); - AU.addRequired<MachineDominatorTree>(); - AU.addRequired<MachineLoopInfo>(); - AU.addRequired<VirtRegMap>(); - AU.addPreserved<MachineDominatorTree>(); - AU.addPreserved<MachineLoopInfo>(); - AU.addPreserved<VirtRegMap>(); - MachineFunctionPass::getAnalysisUsage(AU); - } - - virtual void releaseMemory() { - IntervalSSMap.clear(); - Def2SpillMap.clear(); - } - - virtual const char *getPassName() const { - return "Pre-Register Allocaton Live Interval Splitting"; - } - - /// print - Implement the dump method. - virtual void print(raw_ostream &O, const Module* M = 0) const { - LIs->print(O, M); - } - - - private: - - MachineBasicBlock::iterator - findSpillPoint(MachineBasicBlock*, MachineInstr*, MachineInstr*, - SmallPtrSet<MachineInstr*, 4>&); - - MachineBasicBlock::iterator - findRestorePoint(MachineBasicBlock*, MachineInstr*, SlotIndex, - SmallPtrSet<MachineInstr*, 4>&); - - int CreateSpillStackSlot(unsigned, const TargetRegisterClass *); - - bool IsAvailableInStack(MachineBasicBlock*, unsigned, - SlotIndex, SlotIndex, - SlotIndex&, int&) const; - - void UpdateSpillSlotInterval(VNInfo*, SlotIndex, SlotIndex); - - bool SplitRegLiveInterval(LiveInterval*); - - bool SplitRegLiveIntervals(const TargetRegisterClass **, - SmallPtrSet<LiveInterval*, 8>&); - - bool createsNewJoin(LiveRange* LR, MachineBasicBlock* DefMBB, - MachineBasicBlock* BarrierMBB); - bool Rematerialize(unsigned vreg, VNInfo* ValNo, - MachineInstr* DefMI, - MachineBasicBlock::iterator RestorePt, - SmallPtrSet<MachineInstr*, 4>& RefsInMBB); - MachineInstr* FoldSpill(unsigned vreg, const TargetRegisterClass* RC, - MachineInstr* DefMI, - MachineInstr* Barrier, - MachineBasicBlock* MBB, - int& SS, - SmallPtrSet<MachineInstr*, 4>& RefsInMBB); - MachineInstr* FoldRestore(unsigned vreg, - const TargetRegisterClass* RC, - MachineInstr* Barrier, - MachineBasicBlock* MBB, - int SS, - SmallPtrSet<MachineInstr*, 4>& RefsInMBB); - void RenumberValno(VNInfo* VN); - void ReconstructLiveInterval(LiveInterval* LI); - bool removeDeadSpills(SmallPtrSet<LiveInterval*, 8>& split); - unsigned getNumberOfNonSpills(SmallPtrSet<MachineInstr*, 4>& MIs, - unsigned Reg, int FrameIndex, bool& TwoAddr); - VNInfo* PerformPHIConstruction(MachineBasicBlock::iterator Use, - MachineBasicBlock* MBB, LiveInterval* LI, - SmallPtrSet<MachineInstr*, 4>& Visited, - DenseMap<MachineBasicBlock*, SmallPtrSet<MachineInstr*, 2> >& Defs, - DenseMap<MachineBasicBlock*, SmallPtrSet<MachineInstr*, 2> >& Uses, - DenseMap<MachineInstr*, VNInfo*>& NewVNs, - DenseMap<MachineBasicBlock*, VNInfo*>& LiveOut, - DenseMap<MachineBasicBlock*, VNInfo*>& Phis, - bool IsTopLevel, bool IsIntraBlock); - VNInfo* PerformPHIConstructionFallBack(MachineBasicBlock::iterator Use, - MachineBasicBlock* MBB, LiveInterval* LI, - SmallPtrSet<MachineInstr*, 4>& Visited, - DenseMap<MachineBasicBlock*, SmallPtrSet<MachineInstr*, 2> >& Defs, - DenseMap<MachineBasicBlock*, SmallPtrSet<MachineInstr*, 2> >& Uses, - DenseMap<MachineInstr*, VNInfo*>& NewVNs, - DenseMap<MachineBasicBlock*, VNInfo*>& LiveOut, - DenseMap<MachineBasicBlock*, VNInfo*>& Phis, - bool IsTopLevel, bool IsIntraBlock); -}; -} // end anonymous namespace - -char PreAllocSplitting::ID = 0; - -INITIALIZE_PASS_BEGIN(PreAllocSplitting, "pre-alloc-splitting", - "Pre-Register Allocation Live Interval Splitting", - false, false) -INITIALIZE_PASS_DEPENDENCY(SlotIndexes) -INITIALIZE_PASS_DEPENDENCY(LiveIntervals) -INITIALIZE_PASS_DEPENDENCY(LiveStacks) -INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) -INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) -INITIALIZE_PASS_DEPENDENCY(VirtRegMap) -INITIALIZE_PASS_END(PreAllocSplitting, "pre-alloc-splitting", - "Pre-Register Allocation Live Interval Splitting", - false, false) - -char &llvm::PreAllocSplittingID = PreAllocSplitting::ID; - -/// findSpillPoint - Find a gap as far away from the given MI that's suitable -/// for spilling the current live interval. The index must be before any -/// defs and uses of the live interval register in the mbb. Return begin() if -/// none is found. -MachineBasicBlock::iterator -PreAllocSplitting::findSpillPoint(MachineBasicBlock *MBB, MachineInstr *MI, - MachineInstr *DefMI, - SmallPtrSet<MachineInstr*, 4> &RefsInMBB) { - MachineBasicBlock::iterator Pt = MBB->begin(); - - MachineBasicBlock::iterator MII = MI; - MachineBasicBlock::iterator EndPt = DefMI - ? MachineBasicBlock::iterator(DefMI) : MBB->begin(); - - while (MII != EndPt && !RefsInMBB.count(MII) && - MII->getOpcode() != TRI->getCallFrameSetupOpcode()) - --MII; - if (MII == EndPt || RefsInMBB.count(MII)) return Pt; - - while (MII != EndPt && !RefsInMBB.count(MII)) { - // We can't insert the spill between the barrier (a call), and its - // corresponding call frame setup. - if (MII->getOpcode() == TRI->getCallFrameDestroyOpcode()) { - while (MII->getOpcode() != TRI->getCallFrameSetupOpcode()) { - --MII; - if (MII == EndPt) { - return Pt; - } - } - continue; - } else { - Pt = MII; - } - - if (RefsInMBB.count(MII)) - return Pt; - - - --MII; - } - - return Pt; -} - -/// findRestorePoint - Find a gap in the instruction index map that's suitable -/// for restoring the current live interval value. The index must be before any -/// uses of the live interval register in the mbb. Return end() if none is -/// found. -MachineBasicBlock::iterator -PreAllocSplitting::findRestorePoint(MachineBasicBlock *MBB, MachineInstr *MI, - SlotIndex LastIdx, - SmallPtrSet<MachineInstr*, 4> &RefsInMBB) { - // FIXME: Allow spill to be inserted to the beginning of the mbb. Update mbb - // begin index accordingly. - MachineBasicBlock::iterator Pt = MBB->end(); - MachineBasicBlock::iterator EndPt = MBB->getFirstTerminator(); - - // We start at the call, so walk forward until we find the call frame teardown - // since we can't insert restores before that. Bail if we encounter a use - // during this time. - MachineBasicBlock::iterator MII = MI; - if (MII == EndPt) return Pt; - - while (MII != EndPt && !RefsInMBB.count(MII) && - MII->getOpcode() != TRI->getCallFrameDestroyOpcode()) - ++MII; - if (MII == EndPt || RefsInMBB.count(MII)) return Pt; - ++MII; - - // FIXME: Limit the number of instructions to examine to reduce - // compile time? - while (MII != EndPt) { - SlotIndex Index = LIs->getInstructionIndex(MII); - if (Index > LastIdx) - break; - - // We can't insert a restore between the barrier (a call) and its - // corresponding call frame teardown. - if (MII->getOpcode() == TRI->getCallFrameSetupOpcode()) { - do { - if (MII == EndPt || RefsInMBB.count(MII)) return Pt; - ++MII; - } while (MII->getOpcode() != TRI->getCallFrameDestroyOpcode()); - } else { - Pt = MII; - } - - if (RefsInMBB.count(MII)) - return Pt; - - ++MII; - } - - return Pt; -} - -/// CreateSpillStackSlot - Create a stack slot for the live interval being -/// split. If the live interval was previously split, just reuse the same -/// slot. -int PreAllocSplitting::CreateSpillStackSlot(unsigned Reg, - const TargetRegisterClass *RC) { - int SS; - DenseMap<unsigned, int>::iterator I = IntervalSSMap.find(Reg); - if (I != IntervalSSMap.end()) { - SS = I->second; - } else { - SS = MFI->CreateSpillStackObject(RC->getSize(), RC->getAlignment()); - IntervalSSMap[Reg] = SS; - } - - // Create live interval for stack slot. - CurrSLI = &LSs->getOrCreateInterval(SS, RC); - if (CurrSLI->hasAtLeastOneValue()) - CurrSValNo = CurrSLI->getValNumInfo(0); - else - CurrSValNo = CurrSLI->getNextValue(SlotIndex(), 0, - LSs->getVNInfoAllocator()); - return SS; -} - -/// IsAvailableInStack - Return true if register is available in a split stack -/// slot at the specified index. -bool -PreAllocSplitting::IsAvailableInStack(MachineBasicBlock *DefMBB, - unsigned Reg, SlotIndex DefIndex, - SlotIndex RestoreIndex, - SlotIndex &SpillIndex, - int& SS) const { - if (!DefMBB) - return false; - - DenseMap<unsigned, int>::const_iterator I = IntervalSSMap.find(Reg); - if (I == IntervalSSMap.end()) - return false; - DenseMap<SlotIndex, SlotIndex>::const_iterator - II = Def2SpillMap.find(DefIndex); - if (II == Def2SpillMap.end()) - return false; - - // If last spill of def is in the same mbb as barrier mbb (where restore will - // be), make sure it's not below the intended restore index. - // FIXME: Undo the previous spill? - assert(LIs->getMBBFromIndex(II->second) == DefMBB); - if (DefMBB == BarrierMBB && II->second >= RestoreIndex) - return false; - - SS = I->second; - SpillIndex = II->second; - return true; -} - -/// UpdateSpillSlotInterval - Given the specified val# of the register live -/// interval being split, and the spill and restore indicies, update the live -/// interval of the spill stack slot. -void -PreAllocSplitting::UpdateSpillSlotInterval(VNInfo *ValNo, SlotIndex SpillIndex, - SlotIndex RestoreIndex) { - assert(LIs->getMBBFromIndex(RestoreIndex) == BarrierMBB && - "Expect restore in the barrier mbb"); - - MachineBasicBlock *MBB = LIs->getMBBFromIndex(SpillIndex); - if (MBB == BarrierMBB) { - // Intra-block spill + restore. We are done. - LiveRange SLR(SpillIndex, RestoreIndex, CurrSValNo); - CurrSLI->addRange(SLR); - return; - } - - SmallPtrSet<MachineBasicBlock*, 4> Processed; - SlotIndex EndIdx = LIs->getMBBEndIdx(MBB); - LiveRange SLR(SpillIndex, EndIdx, CurrSValNo); - CurrSLI->addRange(SLR); - Processed.insert(MBB); - - // Start from the spill mbb, figure out the extend of the spill slot's - // live interval. - SmallVector<MachineBasicBlock*, 4> WorkList; - const LiveRange *LR = CurrLI->getLiveRangeContaining(SpillIndex); - if (LR->end > EndIdx) - // If live range extend beyond end of mbb, add successors to work list. - for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(), - SE = MBB->succ_end(); SI != SE; ++SI) - WorkList.push_back(*SI); - - while (!WorkList.empty()) { - MachineBasicBlock *MBB = WorkList.back(); - WorkList.pop_back(); - if (Processed.count(MBB)) - continue; - SlotIndex Idx = LIs->getMBBStartIdx(MBB); - LR = CurrLI->getLiveRangeContaining(Idx); - if (LR && LR->valno == ValNo) { - EndIdx = LIs->getMBBEndIdx(MBB); - if (Idx <= RestoreIndex && RestoreIndex < EndIdx) { - // Spill slot live interval stops at the restore. - LiveRange SLR(Idx, RestoreIndex, CurrSValNo); - CurrSLI->addRange(SLR); - } else if (LR->end > EndIdx) { - // Live range extends beyond end of mbb, process successors. - LiveRange SLR(Idx, EndIdx.getNextIndex(), CurrSValNo); - CurrSLI->addRange(SLR); - for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(), - SE = MBB->succ_end(); SI != SE; ++SI) - WorkList.push_back(*SI); - } else { - LiveRange SLR(Idx, LR->end, CurrSValNo); - CurrSLI->addRange(SLR); - } - Processed.insert(MBB); - } - } -} - -/// PerformPHIConstruction - From properly set up use and def lists, use a PHI -/// construction algorithm to compute the ranges and valnos for an interval. -VNInfo* -PreAllocSplitting::PerformPHIConstruction(MachineBasicBlock::iterator UseI, - MachineBasicBlock* MBB, LiveInterval* LI, - SmallPtrSet<MachineInstr*, 4>& Visited, - DenseMap<MachineBasicBlock*, SmallPtrSet<MachineInstr*, 2> >& Defs, - DenseMap<MachineBasicBlock*, SmallPtrSet<MachineInstr*, 2> >& Uses, - DenseMap<MachineInstr*, VNInfo*>& NewVNs, - DenseMap<MachineBasicBlock*, VNInfo*>& LiveOut, - DenseMap<MachineBasicBlock*, VNInfo*>& Phis, - bool IsTopLevel, bool IsIntraBlock) { - // Return memoized result if it's available. - if (IsTopLevel && Visited.count(UseI) && NewVNs.count(UseI)) - return NewVNs[UseI]; - else if (!IsTopLevel && IsIntraBlock && NewVNs.count(UseI)) - return NewVNs[UseI]; - else if (!IsIntraBlock && LiveOut.count(MBB)) - return LiveOut[MBB]; - - // Check if our block contains any uses or defs. - bool ContainsDefs = Defs.count(MBB); - bool ContainsUses = Uses.count(MBB); - - VNInfo* RetVNI = 0; - - // Enumerate the cases of use/def contaning blocks. - if (!ContainsDefs && !ContainsUses) { - return PerformPHIConstructionFallBack(UseI, MBB, LI, Visited, Defs, Uses, - NewVNs, LiveOut, Phis, - IsTopLevel, IsIntraBlock); - } else if (ContainsDefs && !ContainsUses) { - SmallPtrSet<MachineInstr*, 2>& BlockDefs = Defs[MBB]; - - // Search for the def in this block. If we don't find it before the - // instruction we care about, go to the fallback case. Note that that - // should never happen: this cannot be intrablock, so use should - // always be an end() iterator. - assert(UseI == MBB->end() && "No use marked in intrablock"); - - MachineBasicBlock::iterator Walker = UseI; - --Walker; - while (Walker != MBB->begin()) { - if (BlockDefs.count(Walker)) - break; - --Walker; - } - - // Once we've found it, extend its VNInfo to our instruction. - SlotIndex DefIndex = LIs->getInstructionIndex(Walker); - DefIndex = DefIndex.getDefIndex(); - SlotIndex EndIndex = LIs->getMBBEndIdx(MBB); - - RetVNI = NewVNs[Walker]; - LI->addRange(LiveRange(DefIndex, EndIndex, RetVNI)); - } else if (!ContainsDefs && ContainsUses) { - SmallPtrSet<MachineInstr*, 2>& BlockUses = Uses[MBB]; - - // Search for the use in this block that precedes the instruction we care - // about, going to the fallback case if we don't find it. - MachineBasicBlock::iterator Walker = UseI; - bool found = false; - while (Walker != MBB->begin()) { - --Walker; - if (BlockUses.count(Walker)) { - found = true; - break; - } - } - - if (!found) - return PerformPHIConstructionFallBack(UseI, MBB, LI, Visited, Defs, - Uses, NewVNs, LiveOut, Phis, - IsTopLevel, IsIntraBlock); - - SlotIndex UseIndex = LIs->getInstructionIndex(Walker); - UseIndex = UseIndex.getUseIndex(); - SlotIndex EndIndex; - if (IsIntraBlock) { - EndIndex = LIs->getInstructionIndex(UseI).getDefIndex(); - } else - EndIndex = LIs->getMBBEndIdx(MBB); - - // Now, recursively phi construct the VNInfo for the use we found, - // and then extend it to include the instruction we care about - RetVNI = PerformPHIConstruction(Walker, MBB, LI, Visited, Defs, Uses, - NewVNs, LiveOut, Phis, false, true); - - LI->addRange(LiveRange(UseIndex, EndIndex, RetVNI)); - - // FIXME: Need to set kills properly for inter-block stuff. - } else if (ContainsDefs && ContainsUses) { - SmallPtrSet<MachineInstr*, 2>& BlockDefs = Defs[MBB]; - SmallPtrSet<MachineInstr*, 2>& BlockUses = Uses[MBB]; - - // This case is basically a merging of the two preceding case, with the - // special note that checking for defs must take precedence over checking - // for uses, because of two-address instructions. - MachineBasicBlock::iterator Walker = UseI; - bool foundDef = false; - bool foundUse = false; - while (Walker != MBB->begin()) { - --Walker; - if (BlockDefs.count(Walker)) { - foundDef = true; - break; - } else if (BlockUses.count(Walker)) { - foundUse = true; - break; - } - } - - if (!foundDef && !foundUse) - return PerformPHIConstructionFallBack(UseI, MBB, LI, Visited, Defs, - Uses, NewVNs, LiveOut, Phis, - IsTopLevel, IsIntraBlock); - - SlotIndex StartIndex = LIs->getInstructionIndex(Walker); - StartIndex = foundDef ? StartIndex.getDefIndex() : StartIndex.getUseIndex(); - SlotIndex EndIndex; - if (IsIntraBlock) { - EndIndex = LIs->getInstructionIndex(UseI).getDefIndex(); - } else - EndIndex = LIs->getMBBEndIdx(MBB); - - if (foundDef) - RetVNI = NewVNs[Walker]; - else - RetVNI = PerformPHIConstruction(Walker, MBB, LI, Visited, Defs, Uses, - NewVNs, LiveOut, Phis, false, true); - - LI->addRange(LiveRange(StartIndex, EndIndex, RetVNI)); - } - - // Memoize results so we don't have to recompute them. - if (!IsIntraBlock) LiveOut[MBB] = RetVNI; - else { - if (!NewVNs.count(UseI)) - NewVNs[UseI] = RetVNI; - Visited.insert(UseI); - } - - return RetVNI; -} - -/// PerformPHIConstructionFallBack - PerformPHIConstruction fall back path. -/// -VNInfo* -PreAllocSplitting::PerformPHIConstructionFallBack(MachineBasicBlock::iterator UseI, - MachineBasicBlock* MBB, LiveInterval* LI, - SmallPtrSet<MachineInstr*, 4>& Visited, - DenseMap<MachineBasicBlock*, SmallPtrSet<MachineInstr*, 2> >& Defs, - DenseMap<MachineBasicBlock*, SmallPtrSet<MachineInstr*, 2> >& Uses, - DenseMap<MachineInstr*, VNInfo*>& NewVNs, - DenseMap<MachineBasicBlock*, VNInfo*>& LiveOut, - DenseMap<MachineBasicBlock*, VNInfo*>& Phis, - bool IsTopLevel, bool IsIntraBlock) { - // NOTE: Because this is the fallback case from other cases, we do NOT - // assume that we are not intrablock here. - if (Phis.count(MBB)) return Phis[MBB]; - - SlotIndex StartIndex = LIs->getMBBStartIdx(MBB); - VNInfo *RetVNI = Phis[MBB] = - LI->getNextValue(SlotIndex(), /*FIXME*/ 0, - LIs->getVNInfoAllocator()); - - if (!IsIntraBlock) LiveOut[MBB] = RetVNI; - - // If there are no uses or defs between our starting point and the - // beginning of the block, then recursive perform phi construction - // on our predecessors. - DenseMap<MachineBasicBlock*, VNInfo*> IncomingVNs; - for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), - PE = MBB->pred_end(); PI != PE; ++PI) { - VNInfo* Incoming = PerformPHIConstruction((*PI)->end(), *PI, LI, - Visited, Defs, Uses, NewVNs, - LiveOut, Phis, false, false); - if (Incoming != 0) - IncomingVNs[*PI] = Incoming; - } - - if (MBB->pred_size() == 1 && !RetVNI->hasPHIKill()) { - VNInfo* OldVN = RetVNI; - VNInfo* NewVN = IncomingVNs.begin()->second; - VNInfo* MergedVN = LI->MergeValueNumberInto(OldVN, NewVN); - if (MergedVN == OldVN) std::swap(OldVN, NewVN); - - for (DenseMap<MachineBasicBlock*, VNInfo*>::iterator LOI = LiveOut.begin(), - LOE = LiveOut.end(); LOI != LOE; ++LOI) - if (LOI->second == OldVN) - LOI->second = MergedVN; - for (DenseMap<MachineInstr*, VNInfo*>::iterator NVI = NewVNs.begin(), - NVE = NewVNs.end(); NVI != NVE; ++NVI) - if (NVI->second == OldVN) - NVI->second = MergedVN; - for (DenseMap<MachineBasicBlock*, VNInfo*>::iterator PI = Phis.begin(), - PE = Phis.end(); PI != PE; ++PI) - if (PI->second == OldVN) - PI->second = MergedVN; - RetVNI = MergedVN; - } else { - // Otherwise, merge the incoming VNInfos with a phi join. Create a new - // VNInfo to represent the joined value. - for (DenseMap<MachineBasicBlock*, VNInfo*>::iterator I = - IncomingVNs.begin(), E = IncomingVNs.end(); I != E; ++I) { - I->second->setHasPHIKill(true); - } - } - - SlotIndex EndIndex; - if (IsIntraBlock) { - EndIndex = LIs->getInstructionIndex(UseI).getDefIndex(); - } else - EndIndex = LIs->getMBBEndIdx(MBB); - LI->addRange(LiveRange(StartIndex, EndIndex, RetVNI)); - - // Memoize results so we don't have to recompute them. - if (!IsIntraBlock) - LiveOut[MBB] = RetVNI; - else { - if (!NewVNs.count(UseI)) - NewVNs[UseI] = RetVNI; - Visited.insert(UseI); - } - - return RetVNI; -} - -/// ReconstructLiveInterval - Recompute a live interval from scratch. -void PreAllocSplitting::ReconstructLiveInterval(LiveInterval* LI) { - VNInfo::Allocator& Alloc = LIs->getVNInfoAllocator(); - - // Clear the old ranges and valnos; - LI->clear(); - - // Cache the uses and defs of the register - typedef DenseMap<MachineBasicBlock*, SmallPtrSet<MachineInstr*, 2> > RegMap; - RegMap Defs, Uses; - - // Keep track of the new VNs we're creating. - DenseMap<MachineInstr*, VNInfo*> NewVNs; - SmallPtrSet<VNInfo*, 2> PhiVNs; - - // Cache defs, and create a new VNInfo for each def. - for (MachineRegisterInfo::def_iterator DI = MRI->def_begin(LI->reg), - DE = MRI->def_end(); DI != DE; ++DI) { - Defs[(*DI).getParent()].insert(&*DI); - - SlotIndex DefIdx = LIs->getInstructionIndex(&*DI); - DefIdx = DefIdx.getDefIndex(); - - assert(!DI->isPHI() && "PHI instr in code during pre-alloc splitting."); - VNInfo* NewVN = LI->getNextValue(DefIdx, 0, Alloc); - - // If the def is a move, set the copy field. - if (DI->isCopyLike() && DI->getOperand(0).getReg() == LI->reg) - NewVN->setCopy(&*DI); - - NewVNs[&*DI] = NewVN; - } - - // Cache uses as a separate pass from actually processing them. - for (MachineRegisterInfo::use_iterator UI = MRI->use_begin(LI->reg), - UE = MRI->use_end(); UI != UE; ++UI) - Uses[(*UI).getParent()].insert(&*UI); - - // Now, actually process every use and use a phi construction algorithm - // to walk from it to its reaching definitions, building VNInfos along - // the way. - DenseMap<MachineBasicBlock*, VNInfo*> LiveOut; - DenseMap<MachineBasicBlock*, VNInfo*> Phis; - SmallPtrSet<MachineInstr*, 4> Visited; - for (MachineRegisterInfo::use_iterator UI = MRI->use_begin(LI->reg), - UE = MRI->use_end(); UI != UE; ++UI) { - PerformPHIConstruction(&*UI, UI->getParent(), LI, Visited, Defs, - Uses, NewVNs, LiveOut, Phis, true, true); - } - - // Add ranges for dead defs - for (MachineRegisterInfo::def_iterator DI = MRI->def_begin(LI->reg), - DE = MRI->def_end(); DI != DE; ++DI) { - SlotIndex DefIdx = LIs->getInstructionIndex(&*DI); - DefIdx = DefIdx.getDefIndex(); - - if (LI->liveAt(DefIdx)) continue; - - VNInfo* DeadVN = NewVNs[&*DI]; - LI->addRange(LiveRange(DefIdx, DefIdx.getNextSlot(), DeadVN)); - } -} - -/// RenumberValno - Split the given valno out into a new vreg, allowing it to -/// be allocated to a different register. This function creates a new vreg, -/// copies the valno and its live ranges over to the new vreg's interval, -/// removes them from the old interval, and rewrites all uses and defs of -/// the original reg to the new vreg within those ranges. -void PreAllocSplitting::RenumberValno(VNInfo* VN) { - SmallVector<VNInfo*, 4> Stack; - SmallVector<VNInfo*, 4> VNsToCopy; - Stack.push_back(VN); - - // Walk through and copy the valno we care about, and any other valnos - // that are two-address redefinitions of the one we care about. These - // will need to be rewritten as well. We also check for safety of the - // renumbering here, by making sure that none of the valno involved has - // phi kills. - while (!Stack.empty()) { - VNInfo* OldVN = Stack.back(); - Stack.pop_back(); - - // Bail out if we ever encounter a valno that has a PHI kill. We can't - // renumber these. - if (OldVN->hasPHIKill()) return; - - VNsToCopy.push_back(OldVN); - - // Locate two-address redefinitions - for (MachineRegisterInfo::def_iterator DI = MRI->def_begin(CurrLI->reg), - DE = MRI->def_end(); DI != DE; ++DI) { - if (!DI->isRegTiedToUseOperand(DI.getOperandNo())) continue; - SlotIndex DefIdx = LIs->getInstructionIndex(&*DI).getDefIndex(); - VNInfo* NextVN = CurrLI->findDefinedVNInfoForRegInt(DefIdx); - if (std::find(VNsToCopy.begin(), VNsToCopy.end(), NextVN) != - VNsToCopy.end()) - Stack.push_back(NextVN); - } - } - - // Create the new vreg - unsigned NewVReg = MRI->createVirtualRegister(MRI->getRegClass(CurrLI->reg)); - - // Create the new live interval - LiveInterval& NewLI = LIs->getOrCreateInterval(NewVReg); - - for (SmallVector<VNInfo*, 4>::iterator OI = VNsToCopy.begin(), OE = - VNsToCopy.end(); OI != OE; ++OI) { - VNInfo* OldVN = *OI; - - // Copy the valno over - VNInfo* NewVN = NewLI.createValueCopy(OldVN, LIs->getVNInfoAllocator()); - NewLI.MergeValueInAsValue(*CurrLI, OldVN, NewVN); - - // Remove the valno from the old interval - CurrLI->removeValNo(OldVN); - } - - // Rewrite defs and uses. This is done in two stages to avoid invalidating - // the reg_iterator. - SmallVector<std::pair<MachineInstr*, unsigned>, 8> OpsToChange; - - for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(CurrLI->reg), - E = MRI->reg_end(); I != E; ++I) { - MachineOperand& MO = I.getOperand(); - SlotIndex InstrIdx = LIs->getInstructionIndex(&*I); - - if ((MO.isUse() && NewLI.liveAt(InstrIdx.getUseIndex())) || - (MO.isDef() && NewLI.liveAt(InstrIdx.getDefIndex()))) - OpsToChange.push_back(std::make_pair(&*I, I.getOperandNo())); - } - - for (SmallVector<std::pair<MachineInstr*, unsigned>, 8>::iterator I = - OpsToChange.begin(), E = OpsToChange.end(); I != E; ++I) { - MachineInstr* Inst = I->first; - unsigned OpIdx = I->second; - MachineOperand& MO = Inst->getOperand(OpIdx); - MO.setReg(NewVReg); - } - - // Grow the VirtRegMap, since we've created a new vreg. - VRM->grow(); - - // The renumbered vreg shares a stack slot with the old register. - if (IntervalSSMap.count(CurrLI->reg)) - IntervalSSMap[NewVReg] = IntervalSSMap[CurrLI->reg]; - - ++NumRenumbers; -} - -bool PreAllocSplitting::Rematerialize(unsigned VReg, VNInfo* ValNo, - MachineInstr* DefMI, - MachineBasicBlock::iterator RestorePt, - SmallPtrSet<MachineInstr*, 4>& RefsInMBB) { - MachineBasicBlock& MBB = *RestorePt->getParent(); - - MachineBasicBlock::iterator KillPt = BarrierMBB->end(); - if (!DefMI || DefMI->getParent() == BarrierMBB) - KillPt = findSpillPoint(BarrierMBB, Barrier, NULL, RefsInMBB); - else - KillPt = llvm::next(MachineBasicBlock::iterator(DefMI)); - - if (KillPt == DefMI->getParent()->end()) - return false; - - TII->reMaterialize(MBB, RestorePt, VReg, 0, DefMI, *TRI); - SlotIndex RematIdx = LIs->InsertMachineInstrInMaps(prior(RestorePt)); - - ReconstructLiveInterval(CurrLI); - RematIdx = RematIdx.getDefIndex(); - RenumberValno(CurrLI->findDefinedVNInfoForRegInt(RematIdx)); - - ++NumSplits; - ++NumRemats; - return true; -} - -MachineInstr* PreAllocSplitting::FoldSpill(unsigned vreg, - const TargetRegisterClass* RC, - MachineInstr* DefMI, - MachineInstr* Barrier, - MachineBasicBlock* MBB, - int& SS, - SmallPtrSet<MachineInstr*, 4>& RefsInMBB) { - // Go top down if RefsInMBB is empty. - if (RefsInMBB.empty()) - return 0; - - MachineBasicBlock::iterator FoldPt = Barrier; - while (&*FoldPt != DefMI && FoldPt != MBB->begin() && - !RefsInMBB.count(FoldPt)) - --FoldPt; - - int OpIdx = FoldPt->findRegisterDefOperandIdx(vreg); - if (OpIdx == -1) - return 0; - - SmallVector<unsigned, 1> Ops; - Ops.push_back(OpIdx); - - if (!TII->canFoldMemoryOperand(FoldPt, Ops)) - return 0; - - DenseMap<unsigned, int>::iterator I = IntervalSSMap.find(vreg); - if (I != IntervalSSMap.end()) { - SS = I->second; - } else { - SS = MFI->CreateSpillStackObject(RC->getSize(), RC->getAlignment()); - } - - MachineInstr* FMI = TII->foldMemoryOperand(FoldPt, Ops, SS); - - if (FMI) { - LIs->ReplaceMachineInstrInMaps(FoldPt, FMI); - FoldPt->eraseFromParent(); - ++NumFolds; - - IntervalSSMap[vreg] = SS; - CurrSLI = &LSs->getOrCreateInterval(SS, RC); - if (CurrSLI->hasAtLeastOneValue()) - CurrSValNo = CurrSLI->getValNumInfo(0); - else - CurrSValNo = CurrSLI->getNextValue(SlotIndex(), 0, - LSs->getVNInfoAllocator()); - } - - return FMI; -} - -MachineInstr* PreAllocSplitting::FoldRestore(unsigned vreg, - const TargetRegisterClass* RC, - MachineInstr* Barrier, - MachineBasicBlock* MBB, - int SS, - SmallPtrSet<MachineInstr*, 4>& RefsInMBB) { - if ((int)RestoreFoldLimit != -1 && RestoreFoldLimit == (int)NumRestoreFolds) - return 0; - - // Go top down if RefsInMBB is empty. - if (RefsInMBB.empty()) - return 0; - - // Can't fold a restore between a call stack setup and teardown. - MachineBasicBlock::iterator FoldPt = Barrier; - - // Advance from barrier to call frame teardown. - while (FoldPt != MBB->getFirstTerminator() && - FoldPt->getOpcode() != TRI->getCallFrameDestroyOpcode()) { - if (RefsInMBB.count(FoldPt)) - return 0; - - ++FoldPt; - } - - if (FoldPt == MBB->getFirstTerminator()) - return 0; - else - ++FoldPt; - - // Now find the restore point. - while (FoldPt != MBB->getFirstTerminator() && !RefsInMBB.count(FoldPt)) { - if (FoldPt->getOpcode() == TRI->getCallFrameSetupOpcode()) { - while (FoldPt != MBB->getFirstTerminator() && - FoldPt->getOpcode() != TRI->getCallFrameDestroyOpcode()) { - if (RefsInMBB.count(FoldPt)) - return 0; - - ++FoldPt; - } - - if (FoldPt == MBB->getFirstTerminator()) - return 0; - } - - ++FoldPt; - } - - if (FoldPt == MBB->getFirstTerminator()) - return 0; - - int OpIdx = FoldPt->findRegisterUseOperandIdx(vreg, true); - if (OpIdx == -1) - return 0; - - SmallVector<unsigned, 1> Ops; - Ops.push_back(OpIdx); - - if (!TII->canFoldMemoryOperand(FoldPt, Ops)) - return 0; - - MachineInstr* FMI = TII->foldMemoryOperand(FoldPt, Ops, SS); - - if (FMI) { - LIs->ReplaceMachineInstrInMaps(FoldPt, FMI); - FoldPt->eraseFromParent(); - ++NumRestoreFolds; - } - - return FMI; -} - -/// SplitRegLiveInterval - Split (spill and restore) the given live interval -/// so it would not cross the barrier that's being processed. Shrink wrap -/// (minimize) the live interval to the last uses. -bool PreAllocSplitting::SplitRegLiveInterval(LiveInterval *LI) { - DEBUG(dbgs() << "Pre-alloc splitting " << LI->reg << " for " << *Barrier - << " result: "); - - CurrLI = LI; - - // Find live range where current interval cross the barrier. - LiveInterval::iterator LR = - CurrLI->FindLiveRangeContaining(BarrierIdx.getUseIndex()); - VNInfo *ValNo = LR->valno; - - assert(!ValNo->isUnused() && "Val# is defined by a dead def?"); - - MachineInstr *DefMI = LIs->getInstructionFromIndex(ValNo->def); - - // If this would create a new join point, do not split. - if (DefMI && createsNewJoin(LR, DefMI->getParent(), Barrier->getParent())) { - DEBUG(dbgs() << "FAILED (would create a new join point).\n"); - return false; - } - - // Find all references in the barrier mbb. - SmallPtrSet<MachineInstr*, 4> RefsInMBB; - for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(CurrLI->reg), - E = MRI->reg_end(); I != E; ++I) { - MachineInstr *RefMI = &*I; - if (RefMI->getParent() == BarrierMBB) - RefsInMBB.insert(RefMI); - } - - // Find a point to restore the value after the barrier. - MachineBasicBlock::iterator RestorePt = - findRestorePoint(BarrierMBB, Barrier, LR->end, RefsInMBB); - if (RestorePt == BarrierMBB->end()) { - DEBUG(dbgs() << "FAILED (could not find a suitable restore point).\n"); - return false; - } - - if (DefMI && LIs->isReMaterializable(*LI, ValNo, DefMI)) - if (Rematerialize(LI->reg, ValNo, DefMI, RestorePt, RefsInMBB)) { - DEBUG(dbgs() << "success (remat).\n"); - return true; - } - - // Add a spill either before the barrier or after the definition. - MachineBasicBlock *DefMBB = DefMI ? DefMI->getParent() : NULL; - const TargetRegisterClass *RC = MRI->getRegClass(CurrLI->reg); - SlotIndex SpillIndex; - MachineInstr *SpillMI = NULL; - int SS = -1; - if (!DefMI) { - // If we don't know where the def is we must split just before the barrier. - if ((SpillMI = FoldSpill(LI->reg, RC, 0, Barrier, - BarrierMBB, SS, RefsInMBB))) { - SpillIndex = LIs->getInstructionIndex(SpillMI); - } else { - MachineBasicBlock::iterator SpillPt = - findSpillPoint(BarrierMBB, Barrier, NULL, RefsInMBB); - if (SpillPt == BarrierMBB->begin()) { - DEBUG(dbgs() << "FAILED (could not find a suitable spill point).\n"); - return false; // No gap to insert spill. - } - // Add spill. - - SS = CreateSpillStackSlot(CurrLI->reg, RC); - TII->storeRegToStackSlot(*BarrierMBB, SpillPt, CurrLI->reg, true, SS, RC, - TRI); - SpillMI = prior(SpillPt); - SpillIndex = LIs->InsertMachineInstrInMaps(SpillMI); - } - } else if (!IsAvailableInStack(DefMBB, CurrLI->reg, ValNo->def, - LIs->getZeroIndex(), SpillIndex, SS)) { - // If it's already split, just restore the value. There is no need to spill - // the def again. - if (!DefMI) { - DEBUG(dbgs() << "FAILED (def is dead).\n"); - return false; // Def is dead. Do nothing. - } - - if ((SpillMI = FoldSpill(LI->reg, RC, DefMI, Barrier, - BarrierMBB, SS, RefsInMBB))) { - SpillIndex = LIs->getInstructionIndex(SpillMI); - } else { - // Check if it's possible to insert a spill after the def MI. - MachineBasicBlock::iterator SpillPt; - if (DefMBB == BarrierMBB) { - // Add spill after the def and the last use before the barrier. - SpillPt = findSpillPoint(BarrierMBB, Barrier, DefMI, - RefsInMBB); - if (SpillPt == DefMBB->begin()) { - DEBUG(dbgs() << "FAILED (could not find a suitable spill point).\n"); - return false; // No gap to insert spill. - } - } else { - SpillPt = llvm::next(MachineBasicBlock::iterator(DefMI)); - if (SpillPt == DefMBB->end()) { - DEBUG(dbgs() << "FAILED (could not find a suitable spill point).\n"); - return false; // No gap to insert spill. - } - } - // Add spill. - SS = CreateSpillStackSlot(CurrLI->reg, RC); - TII->storeRegToStackSlot(*DefMBB, SpillPt, CurrLI->reg, false, SS, RC, - TRI); - SpillMI = prior(SpillPt); - SpillIndex = LIs->InsertMachineInstrInMaps(SpillMI); - } - } - - // Remember def instruction index to spill index mapping. - if (DefMI && SpillMI) - Def2SpillMap[ValNo->def] = SpillIndex; - - // Add restore. - bool FoldedRestore = false; - SlotIndex RestoreIndex; - if (MachineInstr* LMI = FoldRestore(CurrLI->reg, RC, Barrier, - BarrierMBB, SS, RefsInMBB)) { - RestorePt = LMI; - RestoreIndex = LIs->getInstructionIndex(RestorePt); - FoldedRestore = true; - } else { - TII->loadRegFromStackSlot(*BarrierMBB, RestorePt, CurrLI->reg, SS, RC, TRI); - MachineInstr *LoadMI = prior(RestorePt); - RestoreIndex = LIs->InsertMachineInstrInMaps(LoadMI); - } - - // Update spill stack slot live interval. - UpdateSpillSlotInterval(ValNo, SpillIndex.getUseIndex().getNextSlot(), - RestoreIndex.getDefIndex()); - - ReconstructLiveInterval(CurrLI); - - if (!FoldedRestore) { - SlotIndex RestoreIdx = LIs->getInstructionIndex(prior(RestorePt)); - RestoreIdx = RestoreIdx.getDefIndex(); - RenumberValno(CurrLI->findDefinedVNInfoForRegInt(RestoreIdx)); - } - - ++NumSplits; - DEBUG(dbgs() << "success.\n"); - return true; -} - -/// SplitRegLiveIntervals - Split all register live intervals that cross the -/// barrier that's being processed. -bool -PreAllocSplitting::SplitRegLiveIntervals(const TargetRegisterClass **RCs, - SmallPtrSet<LiveInterval*, 8>& Split) { - // First find all the virtual registers whose live intervals are intercepted - // by the current barrier. - SmallVector<LiveInterval*, 8> Intervals; - for (const TargetRegisterClass **RC = RCs; *RC; ++RC) { - // FIXME: If it's not safe to move any instruction that defines the barrier - // register class, then it means there are some special dependencies which - // codegen is not modelling. Ignore these barriers for now. - if (!TII->isSafeToMoveRegClassDefs(*RC)) - continue; - const std::vector<unsigned> &VRs = MRI->getRegClassVirtRegs(*RC); - for (unsigned i = 0, e = VRs.size(); i != e; ++i) { - unsigned Reg = VRs[i]; - if (!LIs->hasInterval(Reg)) - continue; - LiveInterval *LI = &LIs->getInterval(Reg); - if (LI->liveAt(BarrierIdx) && !Barrier->readsRegister(Reg)) - // Virtual register live interval is intercepted by the barrier. We - // should split and shrink wrap its interval if possible. - Intervals.push_back(LI); - } - } - - // Process the affected live intervals. - bool Change = false; - while (!Intervals.empty()) { - if (PreSplitLimit != -1 && (int)NumSplits == PreSplitLimit) - break; - LiveInterval *LI = Intervals.back(); - Intervals.pop_back(); - bool result = SplitRegLiveInterval(LI); - if (result) Split.insert(LI); - Change |= result; - } - - return Change; -} - -unsigned PreAllocSplitting::getNumberOfNonSpills( - SmallPtrSet<MachineInstr*, 4>& MIs, - unsigned Reg, int FrameIndex, - bool& FeedsTwoAddr) { - unsigned NonSpills = 0; - for (SmallPtrSet<MachineInstr*, 4>::iterator UI = MIs.begin(), UE = MIs.end(); - UI != UE; ++UI) { - int StoreFrameIndex; - unsigned StoreVReg = TII->isStoreToStackSlot(*UI, StoreFrameIndex); - if (StoreVReg != Reg || StoreFrameIndex != FrameIndex) - ++NonSpills; - - int DefIdx = (*UI)->findRegisterDefOperandIdx(Reg); - if (DefIdx != -1 && (*UI)->isRegTiedToUseOperand(DefIdx)) - FeedsTwoAddr = true; - } - - return NonSpills; -} - -/// removeDeadSpills - After doing splitting, filter through all intervals we've -/// split, and see if any of the spills are unnecessary. If so, remove them. -bool PreAllocSplitting::removeDeadSpills(SmallPtrSet<LiveInterval*, 8>& split) { - bool changed = false; - - // Walk over all of the live intervals that were touched by the splitter, - // and see if we can do any DCE and/or folding. - for (SmallPtrSet<LiveInterval*, 8>::iterator LI = split.begin(), - LE = split.end(); LI != LE; ++LI) { - DenseMap<VNInfo*, SmallPtrSet<MachineInstr*, 4> > VNUseCount; - - // First, collect all the uses of the vreg, and sort them by their - // reaching definition (VNInfo). - for (MachineRegisterInfo::use_iterator UI = MRI->use_begin((*LI)->reg), - UE = MRI->use_end(); UI != UE; ++UI) { - SlotIndex index = LIs->getInstructionIndex(&*UI); - index = index.getUseIndex(); - - const LiveRange* LR = (*LI)->getLiveRangeContaining(index); - VNUseCount[LR->valno].insert(&*UI); - } - - // Now, take the definitions (VNInfo's) one at a time and try to DCE - // and/or fold them away. - for (LiveInterval::vni_iterator VI = (*LI)->vni_begin(), - VE = (*LI)->vni_end(); VI != VE; ++VI) { - - if (DeadSplitLimit != -1 && (int)NumDeadSpills == DeadSplitLimit) - return changed; - - VNInfo* CurrVN = *VI; - - // We don't currently try to handle definitions with PHI kills, because - // it would involve processing more than one VNInfo at once. - if (CurrVN->hasPHIKill()) continue; - - // We also don't try to handle the results of PHI joins, since there's - // no defining instruction to analyze. - MachineInstr* DefMI = LIs->getInstructionFromIndex(CurrVN->def); - if (!DefMI || CurrVN->isUnused()) continue; - - // We're only interested in eliminating cruft introduced by the splitter, - // is of the form load-use or load-use-store. First, check that the - // definition is a load, and remember what stack slot we loaded it from. - int FrameIndex; - if (!TII->isLoadFromStackSlot(DefMI, FrameIndex)) continue; - - // If the definition has no uses at all, just DCE it. - if (VNUseCount[CurrVN].size() == 0) { - LIs->RemoveMachineInstrFromMaps(DefMI); - (*LI)->removeValNo(CurrVN); - DefMI->eraseFromParent(); - VNUseCount.erase(CurrVN); - ++NumDeadSpills; - changed = true; - continue; - } - - // Second, get the number of non-store uses of the definition, as well as - // a flag indicating whether it feeds into a later two-address definition. - bool FeedsTwoAddr = false; - unsigned NonSpillCount = getNumberOfNonSpills(VNUseCount[CurrVN], - (*LI)->reg, FrameIndex, - FeedsTwoAddr); - - // If there's one non-store use and it doesn't feed a two-addr, then - // this is a load-use-store case that we can try to fold. - if (NonSpillCount == 1 && !FeedsTwoAddr) { - // Start by finding the non-store use MachineInstr. - SmallPtrSet<MachineInstr*, 4>::iterator UI = VNUseCount[CurrVN].begin(); - int StoreFrameIndex; - unsigned StoreVReg = TII->isStoreToStackSlot(*UI, StoreFrameIndex); - while (UI != VNUseCount[CurrVN].end() && - (StoreVReg == (*LI)->reg && StoreFrameIndex == FrameIndex)) { - ++UI; - if (UI != VNUseCount[CurrVN].end()) - StoreVReg = TII->isStoreToStackSlot(*UI, StoreFrameIndex); - } - if (UI == VNUseCount[CurrVN].end()) continue; - - MachineInstr* use = *UI; - - // Attempt to fold it away! - int OpIdx = use->findRegisterUseOperandIdx((*LI)->reg, false); - if (OpIdx == -1) continue; - SmallVector<unsigned, 1> Ops; - Ops.push_back(OpIdx); - if (!TII->canFoldMemoryOperand(use, Ops)) continue; - - MachineInstr* NewMI = TII->foldMemoryOperand(use, Ops, FrameIndex); - - if (!NewMI) continue; - - // Update relevant analyses. - LIs->RemoveMachineInstrFromMaps(DefMI); - LIs->ReplaceMachineInstrInMaps(use, NewMI); - (*LI)->removeValNo(CurrVN); - - DefMI->eraseFromParent(); - use->eraseFromParent(); - VNUseCount[CurrVN].erase(use); - - // Remove deleted instructions. Note that we need to remove them from - // the VNInfo->use map as well, just to be safe. - for (SmallPtrSet<MachineInstr*, 4>::iterator II = - VNUseCount[CurrVN].begin(), IE = VNUseCount[CurrVN].end(); - II != IE; ++II) { - for (DenseMap<VNInfo*, SmallPtrSet<MachineInstr*, 4> >::iterator - VNI = VNUseCount.begin(), VNE = VNUseCount.end(); VNI != VNE; - ++VNI) - if (VNI->first != CurrVN) - VNI->second.erase(*II); - LIs->RemoveMachineInstrFromMaps(*II); - (*II)->eraseFromParent(); - } - - VNUseCount.erase(CurrVN); - - for (DenseMap<VNInfo*, SmallPtrSet<MachineInstr*, 4> >::iterator - VI = VNUseCount.begin(), VE = VNUseCount.end(); VI != VE; ++VI) - if (VI->second.erase(use)) - VI->second.insert(NewMI); - - ++NumDeadSpills; - changed = true; - continue; - } - - // If there's more than one non-store instruction, we can't profitably - // fold it, so bail. - if (NonSpillCount) continue; - - // Otherwise, this is a load-store case, so DCE them. - for (SmallPtrSet<MachineInstr*, 4>::iterator UI = - VNUseCount[CurrVN].begin(), UE = VNUseCount[CurrVN].end(); - UI != UE; ++UI) { - LIs->RemoveMachineInstrFromMaps(*UI); - (*UI)->eraseFromParent(); - } - - VNUseCount.erase(CurrVN); - - LIs->RemoveMachineInstrFromMaps(DefMI); - (*LI)->removeValNo(CurrVN); - DefMI->eraseFromParent(); - ++NumDeadSpills; - changed = true; - } - } - - return changed; -} - -bool PreAllocSplitting::createsNewJoin(LiveRange* LR, - MachineBasicBlock* DefMBB, - MachineBasicBlock* BarrierMBB) { - if (DefMBB == BarrierMBB) - return false; - - if (LR->valno->hasPHIKill()) - return false; - - SlotIndex MBBEnd = LIs->getMBBEndIdx(BarrierMBB); - if (LR->end < MBBEnd) - return false; - - MachineLoopInfo& MLI = getAnalysis<MachineLoopInfo>(); - if (MLI.getLoopFor(DefMBB) != MLI.getLoopFor(BarrierMBB)) - return true; - - MachineDominatorTree& MDT = getAnalysis<MachineDominatorTree>(); - SmallPtrSet<MachineBasicBlock*, 4> Visited; - typedef std::pair<MachineBasicBlock*, - MachineBasicBlock::succ_iterator> ItPair; - SmallVector<ItPair, 4> Stack; - Stack.push_back(std::make_pair(BarrierMBB, BarrierMBB->succ_begin())); - - while (!Stack.empty()) { - ItPair P = Stack.back(); - Stack.pop_back(); - - MachineBasicBlock* PredMBB = P.first; - MachineBasicBlock::succ_iterator S = P.second; - - if (S == PredMBB->succ_end()) - continue; - else if (Visited.count(*S)) { - Stack.push_back(std::make_pair(PredMBB, ++S)); - continue; - } else - Stack.push_back(std::make_pair(PredMBB, S+1)); - - MachineBasicBlock* MBB = *S; - Visited.insert(MBB); - - if (MBB == BarrierMBB) - return true; - - MachineDomTreeNode* DefMDTN = MDT.getNode(DefMBB); - MachineDomTreeNode* BarrierMDTN = MDT.getNode(BarrierMBB); - MachineDomTreeNode* MDTN = MDT.getNode(MBB)->getIDom(); - while (MDTN) { - if (MDTN == DefMDTN) - return true; - else if (MDTN == BarrierMDTN) - break; - MDTN = MDTN->getIDom(); - } - - MBBEnd = LIs->getMBBEndIdx(MBB); - if (LR->end > MBBEnd) - Stack.push_back(std::make_pair(MBB, MBB->succ_begin())); - } - - return false; -} - - -bool PreAllocSplitting::runOnMachineFunction(MachineFunction &MF) { - CurrMF = &MF; - TM = &MF.getTarget(); - TRI = TM->getRegisterInfo(); - TII = TM->getInstrInfo(); - MFI = MF.getFrameInfo(); - MRI = &MF.getRegInfo(); - SIs = &getAnalysis<SlotIndexes>(); - LIs = &getAnalysis<LiveIntervals>(); - LSs = &getAnalysis<LiveStacks>(); - VRM = &getAnalysis<VirtRegMap>(); - - bool MadeChange = false; - - // Make sure blocks are numbered in order. - MF.RenumberBlocks(); - - MachineBasicBlock *Entry = MF.begin(); - SmallPtrSet<MachineBasicBlock*,16> Visited; - - SmallPtrSet<LiveInterval*, 8> Split; - - for (df_ext_iterator<MachineBasicBlock*, SmallPtrSet<MachineBasicBlock*,16> > - DFI = df_ext_begin(Entry, Visited), E = df_ext_end(Entry, Visited); - DFI != E; ++DFI) { - BarrierMBB = *DFI; - for (MachineBasicBlock::iterator I = BarrierMBB->begin(), - E = BarrierMBB->end(); I != E; ++I) { - Barrier = &*I; - const TargetRegisterClass **BarrierRCs = - Barrier->getDesc().getRegClassBarriers(); - if (!BarrierRCs) - continue; - BarrierIdx = LIs->getInstructionIndex(Barrier); - MadeChange |= SplitRegLiveIntervals(BarrierRCs, Split); - } - } - - MadeChange |= removeDeadSpills(Split); - - return MadeChange; -} diff --git a/lib/CodeGen/PrologEpilogInserter.cpp b/lib/CodeGen/PrologEpilogInserter.cpp index f1f3c99..a901c5f 100644 --- a/lib/CodeGen/PrologEpilogInserter.cpp +++ b/lib/CodeGen/PrologEpilogInserter.cpp @@ -145,6 +145,7 @@ void PEI::getAnalysisUsage(AnalysisUsage &AU) const { /// pseudo instructions. void PEI::calculateCallsInformation(MachineFunction &Fn) { const TargetRegisterInfo *RegInfo = Fn.getTarget().getRegisterInfo(); + const TargetInstrInfo &TII = *Fn.getTarget().getInstrInfo(); const TargetFrameLowering *TFI = Fn.getTarget().getFrameLowering(); MachineFrameInfo *MFI = Fn.getFrameInfo(); @@ -152,8 +153,8 @@ void PEI::calculateCallsInformation(MachineFunction &Fn) { bool AdjustsStack = MFI->adjustsStack(); // Get the function call frame set-up and tear-down instruction opcode - int FrameSetupOpcode = RegInfo->getCallFrameSetupOpcode(); - int FrameDestroyOpcode = RegInfo->getCallFrameDestroyOpcode(); + int FrameSetupOpcode = TII.getCallFrameSetupOpcode(); + int FrameDestroyOpcode = TII.getCallFrameDestroyOpcode(); // Early exit for targets which have no call frame setup/destroy pseudo // instructions. @@ -705,12 +706,13 @@ void PEI::replaceFrameIndices(MachineFunction &Fn) { const TargetMachine &TM = Fn.getTarget(); assert(TM.getRegisterInfo() && "TM::getRegisterInfo() must be implemented!"); + const TargetInstrInfo &TII = *Fn.getTarget().getInstrInfo(); const TargetRegisterInfo &TRI = *TM.getRegisterInfo(); const TargetFrameLowering *TFI = TM.getFrameLowering(); bool StackGrowsDown = TFI->getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown; - int FrameSetupOpcode = TRI.getCallFrameSetupOpcode(); - int FrameDestroyOpcode = TRI.getCallFrameDestroyOpcode(); + int FrameSetupOpcode = TII.getCallFrameSetupOpcode(); + int FrameDestroyOpcode = TII.getCallFrameDestroyOpcode(); for (MachineFunction::iterator BB = Fn.begin(), E = Fn.end(); BB != E; ++BB) { diff --git a/lib/CodeGen/RegAllocBasic.cpp b/lib/CodeGen/RegAllocBasic.cpp index 1d77b29..bcb38d7 100644 --- a/lib/CodeGen/RegAllocBasic.cpp +++ b/lib/CodeGen/RegAllocBasic.cpp @@ -20,6 +20,7 @@ #include "RenderMachineFunction.h" #include "Spiller.h" #include "VirtRegMap.h" +#include "RegisterCoalescer.h" #include "llvm/ADT/OwningPtr.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" @@ -34,7 +35,6 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegAllocRegistry.h" -#include "llvm/CodeGen/RegisterCoalescer.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Target/TargetRegisterInfo.h" @@ -141,7 +141,7 @@ RABasic::RABasic(): MachineFunctionPass(ID) { initializeLiveIntervalsPass(*PassRegistry::getPassRegistry()); initializeSlotIndexesPass(*PassRegistry::getPassRegistry()); initializeStrongPHIEliminationPass(*PassRegistry::getPassRegistry()); - initializeRegisterCoalescerAnalysisGroup(*PassRegistry::getPassRegistry()); + initializeRegisterCoalescerPass(*PassRegistry::getPassRegistry()); initializeCalculateSpillWeightsPass(*PassRegistry::getPassRegistry()); initializeLiveStacksPass(*PassRegistry::getPassRegistry()); initializeMachineDominatorTreePass(*PassRegistry::getPassRegistry()); diff --git a/lib/CodeGen/RegAllocFast.cpp b/lib/CodeGen/RegAllocFast.cpp index 0f27dfc..ee23194 100644 --- a/lib/CodeGen/RegAllocFast.cpp +++ b/lib/CodeGen/RegAllocFast.cpp @@ -118,7 +118,7 @@ namespace { // SkippedInstrs - Descriptors of instructions whose clobber list was // ignored because all registers were spilled. It is still necessary to // mark all the clobbered registers as used by the function. - SmallPtrSet<const TargetInstrDesc*, 4> SkippedInstrs; + SmallPtrSet<const MCInstrDesc*, 4> SkippedInstrs; // isBulkSpilling - This flag is set when LiveRegMap will be cleared // completely after spilling all live registers. LiveRegMap entries should @@ -423,7 +423,7 @@ void RAFast::definePhysReg(MachineInstr *MI, unsigned PhysReg, // Returns spillImpossible when PhysReg or an alias can't be spilled. unsigned RAFast::calcSpillCost(unsigned PhysReg) const { if (UsedInInstr.test(PhysReg)) { - DEBUG(dbgs() << "PhysReg: " << PhysReg << " is already used in instr.\n"); + DEBUG(dbgs() << PrintReg(PhysReg, TRI) << " is already used in instr.\n"); return spillImpossible; } switch (unsigned VirtReg = PhysRegState[PhysReg]) { @@ -432,15 +432,15 @@ unsigned RAFast::calcSpillCost(unsigned PhysReg) const { case regFree: return 0; case regReserved: - DEBUG(dbgs() << "VirtReg: " << VirtReg << " corresponding to PhysReg: " - << PhysReg << " is reserved already.\n"); + DEBUG(dbgs() << PrintReg(VirtReg, TRI) << " corresponding " + << PrintReg(PhysReg, TRI) << " is reserved already.\n"); return spillImpossible; default: return LiveVirtRegs.lookup(VirtReg).Dirty ? spillDirty : spillClean; } // This is a disabled register, add up cost of aliases. - DEBUG(dbgs() << "\tRegister: " << PhysReg << " is disabled.\n"); + DEBUG(dbgs() << PrintReg(PhysReg, TRI) << " is disabled.\n"); unsigned Cost = 0; for (const unsigned *AS = TRI->getAliasSet(PhysReg); unsigned Alias = *AS; ++AS) { @@ -515,7 +515,7 @@ void RAFast::allocVirtReg(MachineInstr *MI, LiveRegEntry &LRE, unsigned Hint) { unsigned BestReg = 0, BestCost = spillImpossible; for (ArrayRef<unsigned>::iterator I = AO.begin(), E = AO.end(); I != E; ++I) { unsigned Cost = calcSpillCost(*I); - DEBUG(dbgs() << "\tRegister: " << *I << "\n"); + DEBUG(dbgs() << "\tRegister: " << PrintReg(*I, TRI) << "\n"); DEBUG(dbgs() << "\tCost: " << Cost << "\n"); DEBUG(dbgs() << "\tBestCost: " << BestCost << "\n"); // Cost is 0 when all aliases are already disabled. @@ -726,7 +726,8 @@ void RAFast::handleThroughOperands(MachineInstr *MI, if (!MO.isReg() || (MO.isDef() && !MO.isEarlyClobber())) continue; unsigned Reg = MO.getReg(); if (!Reg || !TargetRegisterInfo::isPhysicalRegister(Reg)) continue; - DEBUG(dbgs() << "\tSetting reg " << Reg << " as used in instr\n"); + DEBUG(dbgs() << "\tSetting " << PrintReg(Reg, TRI) + << " as used in instr\n"); UsedInInstr.set(Reg); } @@ -776,7 +777,7 @@ void RAFast::AllocateBasicBlock() { // Otherwise, sequentially allocate each instruction in the MBB. while (MII != MBB->end()) { MachineInstr *MI = MII++; - const TargetInstrDesc &TID = MI->getDesc(); + const MCInstrDesc &MCID = MI->getDesc(); DEBUG({ dbgs() << "\n>> " << *MI << "Regs:"; for (unsigned Reg = 1, E = TRI->getNumRegs(); Reg != E; ++Reg) { @@ -889,7 +890,7 @@ void RAFast::AllocateBasicBlock() { VirtOpEnd = i+1; if (MO.isUse()) { hasTiedOps = hasTiedOps || - TID.getOperandConstraint(i, TOI::TIED_TO) != -1; + MCID.getOperandConstraint(i, MCOI::TIED_TO) != -1; } else { if (MO.isEarlyClobber()) hasEarlyClobbers = true; @@ -919,7 +920,7 @@ void RAFast::AllocateBasicBlock() { // We didn't detect inline asm tied operands above, so just make this extra // pass for all inline asm. if (MI->isInlineAsm() || hasEarlyClobbers || hasPartialRedefs || - (hasTiedOps && (hasPhysDefs || TID.getNumDefs() > 1))) { + (hasTiedOps && (hasPhysDefs || MCID.getNumDefs() > 1))) { handleThroughOperands(MI, VirtDead); // Don't attempt coalescing when we have funny stuff going on. CopyDst = 0; @@ -964,7 +965,7 @@ void RAFast::AllocateBasicBlock() { } unsigned DefOpEnd = MI->getNumOperands(); - if (TID.isCall()) { + if (MCID.isCall()) { // Spill all virtregs before a call. This serves two purposes: 1. If an // exception is thrown, the landing pad is going to expect to find // registers in their spill slots, and 2. we don't have to wade through @@ -975,7 +976,7 @@ void RAFast::AllocateBasicBlock() { // The imp-defs are skipped below, but we still need to mark those // registers as used by the function. - SkippedInstrs.insert(&TID); + SkippedInstrs.insert(&MCID); } // Third scan. @@ -1061,7 +1062,7 @@ bool RAFast::runOnMachineFunction(MachineFunction &Fn) { MRI->closePhysRegsUsed(*TRI); // Add the clobber lists for all the instructions we skipped earlier. - for (SmallPtrSet<const TargetInstrDesc*, 4>::const_iterator + for (SmallPtrSet<const MCInstrDesc*, 4>::const_iterator I = SkippedInstrs.begin(), E = SkippedInstrs.end(); I != E; ++I) if (const unsigned *Defs = (*I)->getImplicitDefs()) while (*Defs) diff --git a/lib/CodeGen/RegAllocGreedy.cpp b/lib/CodeGen/RegAllocGreedy.cpp index 8d06325..912d899 100644 --- a/lib/CodeGen/RegAllocGreedy.cpp +++ b/lib/CodeGen/RegAllocGreedy.cpp @@ -22,6 +22,7 @@ #include "SpillPlacement.h" #include "SplitKit.h" #include "VirtRegMap.h" +#include "RegisterCoalescer.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Function.h" @@ -37,7 +38,6 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegAllocRegistry.h" -#include "llvm/CodeGen/RegisterCoalescer.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" @@ -234,7 +234,7 @@ RAGreedy::RAGreedy(): MachineFunctionPass(ID), LRStage(RS_New) { initializeLiveIntervalsPass(*PassRegistry::getPassRegistry()); initializeSlotIndexesPass(*PassRegistry::getPassRegistry()); initializeStrongPHIEliminationPass(*PassRegistry::getPassRegistry()); - initializeRegisterCoalescerAnalysisGroup(*PassRegistry::getPassRegistry()); + initializeRegisterCoalescerPass(*PassRegistry::getPassRegistry()); initializeCalculateSpillWeightsPass(*PassRegistry::getPassRegistry()); initializeLiveStacksPass(*PassRegistry::getPassRegistry()); initializeMachineDominatorTreePass(*PassRegistry::getPassRegistry()); @@ -763,32 +763,46 @@ void RAGreedy::splitAroundRegion(LiveInterval &VirtReg, // Create the main cross-block interval. const unsigned MainIntv = SE->openIntv(); - // First add all defs that are live out of a block. + // First handle all the blocks with uses. ArrayRef<SplitAnalysis::BlockInfo> UseBlocks = SA->getUseBlocks(); for (unsigned i = 0; i != UseBlocks.size(); ++i) { const SplitAnalysis::BlockInfo &BI = UseBlocks[i]; - bool RegIn = LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 0)]; - bool RegOut = LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 1)]; + bool RegIn = BI.LiveIn && + LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 0)]; + bool RegOut = BI.LiveOut && + LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 1)]; // Create separate intervals for isolated blocks with multiple uses. - if (!RegIn && !RegOut && BI.FirstUse != BI.LastUse) { + // + // |---o---o---| Enter and leave on the stack. + // ____-----____ Create local interval for uses. + // + // | o---o---| Defined in block, leave on stack. + // -----____ Create local interval for uses. + // + // |---o---x | Enter on stack, killed in block. + // ____----- Create local interval for uses. + // + if (!RegIn && !RegOut) { DEBUG(dbgs() << "BB#" << BI.MBB->getNumber() << " isolated.\n"); - SE->splitSingleBlock(BI); - SE->selectIntv(MainIntv); + if (!BI.isOneInstr()) { + SE->splitSingleBlock(BI); + SE->selectIntv(MainIntv); + } continue; } - // Should the register be live out? - if (!BI.LiveOut || !RegOut) - continue; - SlotIndex Start, Stop; tie(Start, Stop) = Indexes->getMBBRange(BI.MBB); Intf.moveToBlock(BI.MBB->getNumber()); - DEBUG(dbgs() << "BB#" << BI.MBB->getNumber() << " -> EB#" - << Bundles->getBundle(BI.MBB->getNumber(), 1) + DEBUG(dbgs() << "EB#" << Bundles->getBundle(BI.MBB->getNumber(), 0) + << (RegIn ? " => " : " -- ") + << "BB#" << BI.MBB->getNumber() + << (RegOut ? " => " : " -- ") + << " EB#" << Bundles->getBundle(BI.MBB->getNumber(), 1) << " [" << Start << ';' << SA->getLastSplitPoint(BI.MBB->getNumber()) << '-' << Stop + << ") uses [" << BI.FirstUse << ';' << BI.LastUse << ") intf [" << Intf.first() << ';' << Intf.last() << ')'); // The interference interval should either be invalid or overlap MBB. @@ -797,150 +811,266 @@ void RAGreedy::splitAroundRegion(LiveInterval &VirtReg, assert((!Intf.hasInterference() || Intf.last() > Start) && "Bad interference"); - // Check interference leaving the block. + // We are now ready to decide where to split in the current block. There + // are many variables guiding the decision: + // + // - RegIn / RegOut: The global splitting algorithm's decisions for our + // ingoing and outgoing bundles. + // + // - BI.BlockIn / BI.BlockOut: Is the live range live-in and/or live-out + // from this block. + // + // - Intf.hasInterference(): Is there interference in this block. + // + // - Intf.first() / Inft.last(): The range of interference. + // + // The live range should be split such that MainIntv is live-in when RegIn + // is set, and live-out when RegOut is set. MainIntv should never overlap + // the interference, and the stack interval should never have more than one + // use per block. + + // No splits can be inserted after LastSplitPoint, overlap instead. + SlotIndex LastSplitPoint = Stop; + if (BI.LiveOut) + LastSplitPoint = SA->getLastSplitPoint(BI.MBB->getNumber()); + + // At this point, we know that either RegIn or RegOut is set. We dealt with + // the all-stack case above. + + // Blocks without interference are relatively easy. if (!Intf.hasInterference()) { - // Block is interference-free. - DEBUG(dbgs() << ", no interference"); - if (!BI.LiveThrough) { - DEBUG(dbgs() << ", not live-through.\n"); - SE->useIntv(SE->enterIntvBefore(BI.FirstUse), Stop); - continue; - } - if (!RegIn) { - // Block is live-through, but entry bundle is on the stack. - // Reload just before the first use. - DEBUG(dbgs() << ", not live-in, enter before first use.\n"); - SE->useIntv(SE->enterIntvBefore(BI.FirstUse), Stop); - continue; + DEBUG(dbgs() << ", no interference.\n"); + SE->selectIntv(MainIntv); + // The easiest case has MainIntv live through. + // + // |---o---o---| Live-in, live-out. + // ============= Use MainIntv everywhere. + // + SlotIndex From = Start, To = Stop; + + // Block entry. Reload before the first use if MainIntv is not live-in. + // + // |---o-- Enter on stack. + // ____=== Reload before first use. + // + // | o-- Defined in block. + // === Use MainIntv from def. + // + if (!RegIn) + From = SE->enterIntvBefore(BI.FirstUse); + + // Block exit. Handle cases where MainIntv is not live-out. + if (!BI.LiveOut) + // + // --x | Killed in block. + // === Use MainIntv up to kill. + // + To = SE->leaveIntvAfter(BI.LastUse); + else if (!RegOut) { + // + // --o---| Live-out on stack. + // ===____ Use MainIntv up to last use, switch to stack. + // + // -----o| Live-out on stack, last use after last split point. + // ====== Extend MainIntv to last use, overlapping. + // \____ Copy to stack interval before last split point. + // + if (BI.LastUse < LastSplitPoint) + To = SE->leaveIntvAfter(BI.LastUse); + else { + // The last use is after the last split point, it is probably an + // indirect branch. + To = SE->leaveIntvBefore(LastSplitPoint); + // Run a double interval from the split to the last use. This makes + // it possible to spill the complement without affecting the indirect + // branch. + SE->overlapIntv(To, BI.LastUse); + } } - DEBUG(dbgs() << ", live-through.\n"); - continue; - } - - // Block has interference. - DEBUG(dbgs() << ", interference to " << Intf.last()); - if (!BI.LiveThrough && Intf.last() <= BI.FirstUse) { - // The interference doesn't reach the outgoing segment. - DEBUG(dbgs() << " doesn't affect def from " << BI.FirstUse << '\n'); - SE->useIntv(BI.FirstUse, Stop); + // Paint in MainIntv liveness for this block. + SE->useIntv(From, To); continue; } - SlotIndex LastSplitPoint = SA->getLastSplitPoint(BI.MBB->getNumber()); - if (Intf.last().getBoundaryIndex() < BI.LastUse) { - // There are interference-free uses at the end of the block. - // Find the first use that can get the live-out register. - SmallVectorImpl<SlotIndex>::const_iterator UI = - std::lower_bound(SA->UseSlots.begin(), SA->UseSlots.end(), - Intf.last().getBoundaryIndex()); - assert(UI != SA->UseSlots.end() && "Couldn't find last use"); - SlotIndex Use = *UI; - assert(Use <= BI.LastUse && "Couldn't find last use"); - // Only attempt a split befroe the last split point. - if (Use.getBaseIndex() <= LastSplitPoint) { - DEBUG(dbgs() << ", free use at " << Use << ".\n"); - SlotIndex SegStart = SE->enterIntvBefore(Use); - assert(SegStart >= Intf.last() && "Couldn't avoid interference"); - assert(SegStart < LastSplitPoint && "Impossible split point"); - SE->useIntv(SegStart, Stop); - continue; - } - } + // We are now looking at a block with interference, and we know that either + // RegIn or RegOut is set. + assert(Intf.hasInterference() && (RegIn || RegOut) && "Bad invariant"); - // Interference is after the last use. - DEBUG(dbgs() << " after last use.\n"); - SlotIndex SegStart = SE->enterIntvAtEnd(*BI.MBB); - assert(SegStart >= Intf.last() && "Couldn't avoid interference"); - } + // If the live range is not live through the block, it is possible that the + // interference doesn't even overlap. Deal with those cases first. Since + // no copy instructions are required, we can tolerate interference starting + // or ending at the same instruction that kills or defines our live range. - // Now all defs leading to live bundles are handled, do everything else. - for (unsigned i = 0; i != UseBlocks.size(); ++i) { - const SplitAnalysis::BlockInfo &BI = UseBlocks[i]; - bool RegIn = LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 0)]; - bool RegOut = LiveBundles[Bundles->getBundle(BI.MBB->getNumber(), 1)]; + // Live-in, killed before interference. + // + // ~~~ Interference after kill. + // |---o---x | Killed in block. + // ========= Use MainIntv everywhere. + // + if (RegIn && !BI.LiveOut && BI.LastUse <= Intf.first()) { + DEBUG(dbgs() << ", live-in, killed before interference.\n"); + SE->selectIntv(MainIntv); + SlotIndex To = SE->leaveIntvAfter(BI.LastUse); + SE->useIntv(Start, To); + continue; + } - // Is the register live-in? - if (!BI.LiveIn || !RegIn) + // Live-out, defined after interference. + // + // ~~~ Interference before def. + // | o---o---| Defined in block. + // ========= Use MainIntv everywhere. + // + if (RegOut && !BI.LiveIn && BI.FirstUse >= Intf.last()) { + DEBUG(dbgs() << ", live-out, defined after interference.\n"); + SE->selectIntv(MainIntv); + SlotIndex From = SE->enterIntvBefore(BI.FirstUse); + SE->useIntv(From, Stop); continue; + } - // We have an incoming register. Check for interference. - SlotIndex Start, Stop; - tie(Start, Stop) = Indexes->getMBBRange(BI.MBB); - Intf.moveToBlock(BI.MBB->getNumber()); - DEBUG(dbgs() << "EB#" << Bundles->getBundle(BI.MBB->getNumber(), 0) - << " -> BB#" << BI.MBB->getNumber() << " [" << Start << ';' - << SA->getLastSplitPoint(BI.MBB->getNumber()) << '-' << Stop - << ')'); + // The interference is now known to overlap the live range, but it may + // still be easy to avoid if all the interference is on one side of the + // uses, and we enter or leave on the stack. - // Check interference entering the block. - if (!Intf.hasInterference()) { - // Block is interference-free. - DEBUG(dbgs() << ", no interference"); - if (!BI.LiveThrough) { - DEBUG(dbgs() << ", killed in block.\n"); - SE->useIntv(Start, SE->leaveIntvAfter(BI.LastUse)); - continue; - } - if (!RegOut) { - SlotIndex LastSplitPoint = SA->getLastSplitPoint(BI.MBB->getNumber()); - // Block is live-through, but exit bundle is on the stack. - // Spill immediately after the last use. - if (BI.LastUse < LastSplitPoint) { - DEBUG(dbgs() << ", uses, stack-out.\n"); - SE->useIntv(Start, SE->leaveIntvAfter(BI.LastUse)); - continue; - } - // The last use is after the last split point, it is probably an - // indirect jump. - DEBUG(dbgs() << ", uses at " << BI.LastUse << " after split point " - << LastSplitPoint << ", stack-out.\n"); - SlotIndex SegEnd = SE->leaveIntvBefore(LastSplitPoint); - SE->useIntv(Start, SegEnd); - // Run a double interval from the split to the last use. - // This makes it possible to spill the complement without affecting the - // indirect branch. - SE->overlapIntv(SegEnd, BI.LastUse); - continue; + // Live-out on stack, interference after last use. + // + // ~~~ Interference after last use. + // |---o---o---| Live-out on stack. + // =========____ Leave MainIntv after last use. + // + // ~ Interference after last use. + // |---o---o--o| Live-out on stack, late last use. + // =========____ Copy to stack after LSP, overlap MainIntv. + // + if (!RegOut && Intf.first() > BI.LastUse.getBoundaryIndex()) { + assert(RegIn && "Stack-in, stack-out should already be handled"); + if (BI.LastUse < LastSplitPoint) { + DEBUG(dbgs() << ", live-in, stack-out, interference after last use.\n"); + SE->selectIntv(MainIntv); + SlotIndex To = SE->leaveIntvAfter(BI.LastUse); + assert(To <= Intf.first() && "Expected to avoid interference"); + SE->useIntv(Start, To); + } else { + DEBUG(dbgs() << ", live-in, stack-out, avoid last split point\n"); + SE->selectIntv(MainIntv); + SlotIndex To = SE->leaveIntvBefore(LastSplitPoint); + assert(To <= Intf.first() && "Expected to avoid interference"); + SE->overlapIntv(To, BI.LastUse); + SE->useIntv(Start, To); } - // Register is live-through. - DEBUG(dbgs() << ", uses, live-through.\n"); - SE->useIntv(Start, Stop); continue; } - // Block has interference. - DEBUG(dbgs() << ", interference from " << Intf.first()); - - if (!BI.LiveThrough && Intf.first() >= BI.LastUse) { - // The interference doesn't reach the outgoing segment. - DEBUG(dbgs() << " doesn't affect kill at " << BI.LastUse << '\n'); - SE->useIntv(Start, BI.LastUse); + // Live-in on stack, interference before first use. + // + // ~~~ Interference before first use. + // |---o---o---| Live-in on stack. + // ____========= Enter MainIntv before first use. + // + if (!RegIn && Intf.last() < BI.FirstUse.getBaseIndex()) { + assert(RegOut && "Stack-in, stack-out should already be handled"); + DEBUG(dbgs() << ", stack-in, interference before first use.\n"); + SE->selectIntv(MainIntv); + SlotIndex From = SE->enterIntvBefore(BI.FirstUse); + assert(From >= Intf.last() && "Expected to avoid interference"); + SE->useIntv(From, Stop); continue; } - if (Intf.first().getBaseIndex() > BI.FirstUse) { - // There are interference-free uses at the beginning of the block. - // Find the last use that can get the register. - SmallVectorImpl<SlotIndex>::const_iterator UI = - std::lower_bound(SA->UseSlots.begin(), SA->UseSlots.end(), - Intf.first().getBaseIndex()); - assert(UI != SA->UseSlots.begin() && "Couldn't find first use"); - SlotIndex Use = (--UI)->getBoundaryIndex(); - DEBUG(dbgs() << ", free use at " << *UI << ".\n"); - SlotIndex SegEnd = SE->leaveIntvAfter(Use); - assert(SegEnd <= Intf.first() && "Couldn't avoid interference"); - SE->useIntv(Start, SegEnd); - continue; + // The interference is overlapping somewhere we wanted to use MainIntv. That + // means we need to create a local interval that can be allocated a + // different register. + DEBUG(dbgs() << ", creating local interval.\n"); + unsigned LocalIntv = SE->openIntv(); + + // We may be creating copies directly between MainIntv and LocalIntv, + // bypassing the stack interval. When we do that, we should never use the + // leaveIntv* methods as they define values in the stack interval. By + // starting from the end of the block and working our way backwards, we can + // get by with only enterIntv* methods. + // + // When selecting split points, we generally try to maximize the stack + // interval as long at it contains no uses, maximize the main interval as + // long as it doesn't overlap interference, and minimize the local interval + // that we don't know how to allocate yet. + + // Handle the block exit, set Pos to the first handled slot. + SlotIndex Pos = BI.LastUse; + if (RegOut) { + assert(Intf.last() < LastSplitPoint && "Cannot be live-out in register"); + // Create a snippet of MainIntv that is live-out. + // + // ~~~ Interference overlapping uses. + // --o---| Live-out in MainIntv. + // ----=== Switch from LocalIntv to MainIntv after interference. + // + SE->selectIntv(MainIntv); + Pos = SE->enterIntvAfter(Intf.last()); + assert(Pos >= Intf.last() && "Expected to avoid interference"); + SE->useIntv(Pos, Stop); + SE->selectIntv(LocalIntv); + } else if (BI.LiveOut) { + if (BI.LastUse < LastSplitPoint) { + // Live-out on the stack. + // + // ~~~ Interference overlapping uses. + // --o---| Live-out on stack. + // ---____ Switch from LocalIntv to stack after last use. + // + Pos = SE->leaveIntvAfter(BI.LastUse); + } else { + // Live-out on the stack, last use after last split point. + // + // ~~~ Interference overlapping uses. + // --o--o| Live-out on stack, late use. + // ------ Copy to stack before LSP, overlap LocalIntv. + // \__ + // + Pos = SE->leaveIntvBefore(LastSplitPoint); + // We need to overlap LocalIntv so it can reach LastUse. + SE->overlapIntv(Pos, BI.LastUse); + } } - // Interference is before the first use. - DEBUG(dbgs() << " before first use.\n"); - SlotIndex SegEnd = SE->leaveIntvAtTop(*BI.MBB); - assert(SegEnd <= Intf.first() && "Couldn't avoid interference"); + // When not live-out, leave Pos at LastUse. We have handled everything from + // Pos to Stop. Find the starting point for LocalIntv. + assert(SE->currentIntv() == LocalIntv && "Expecting local interval"); + + if (RegIn) { + assert(Start < Intf.first() && "Cannot be live-in with interference"); + // Live-in in MainIntv, only use LocalIntv for interference. + // + // ~~~ Interference overlapping uses. + // |---o-- Live-in in MainIntv. + // ====--- Switch to LocalIntv before interference. + // + SlotIndex Switch = SE->enterIntvBefore(Intf.first()); + assert(Switch <= Intf.first() && "Expected to avoid interference"); + SE->useIntv(Switch, Pos); + SE->selectIntv(MainIntv); + SE->useIntv(Start, Switch); + } else { + // Live-in on stack, enter LocalIntv before first use. + // + // ~~~ Interference overlapping uses. + // |---o-- Live-in in MainIntv. + // ____--- Reload to LocalIntv before interference. + // + // Defined in block. + // + // ~~~ Interference overlapping uses. + // | o-- Defined in block. + // --- Begin LocalIntv at first use. + // + SlotIndex Switch = SE->enterIntvBefore(BI.FirstUse); + SE->useIntv(Switch, Pos); + } } // Handle live-through blocks. + SE->selectIntv(MainIntv); for (unsigned i = 0, e = Cand.ActiveBlocks.size(); i != e; ++i) { unsigned Number = Cand.ActiveBlocks[i]; bool RegIn = LiveBundles[Bundles->getBundle(Number, 0)]; diff --git a/lib/CodeGen/RegAllocLinearScan.cpp b/lib/CodeGen/RegAllocLinearScan.cpp index 7a2ea6c..0dd3c59 100644 --- a/lib/CodeGen/RegAllocLinearScan.cpp +++ b/lib/CodeGen/RegAllocLinearScan.cpp @@ -18,6 +18,7 @@ #include "VirtRegRewriter.h" #include "RegisterClassInfo.h" #include "Spiller.h" +#include "RegisterCoalescer.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Function.h" #include "llvm/CodeGen/CalcSpillWeights.h" @@ -28,7 +29,6 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegAllocRegistry.h" -#include "llvm/CodeGen/RegisterCoalescer.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" @@ -58,11 +58,6 @@ NewHeuristic("new-spilling-heuristic", cl::init(false), cl::Hidden); static cl::opt<bool> -PreSplitIntervals("pre-alloc-split", - cl::desc("Pre-register allocation live interval splitting"), - cl::init(false), cl::Hidden); - -static cl::opt<bool> TrivCoalesceEnds("trivial-coalesce-ends", cl::desc("Attempt trivial coalescing of interval ends"), cl::init(false), cl::Hidden); @@ -101,10 +96,9 @@ namespace { initializeLiveDebugVariablesPass(*PassRegistry::getPassRegistry()); initializeLiveIntervalsPass(*PassRegistry::getPassRegistry()); initializeStrongPHIEliminationPass(*PassRegistry::getPassRegistry()); - initializeRegisterCoalescerAnalysisGroup( + initializeRegisterCoalescerPass( *PassRegistry::getPassRegistry()); initializeCalculateSpillWeightsPass(*PassRegistry::getPassRegistry()); - initializePreAllocSplittingPass(*PassRegistry::getPassRegistry()); initializeLiveStacksPass(*PassRegistry::getPassRegistry()); initializeMachineDominatorTreePass(*PassRegistry::getPassRegistry()); initializeMachineLoopInfoPass(*PassRegistry::getPassRegistry()); @@ -217,8 +211,6 @@ namespace { // to coalescing and which analyses coalescing invalidates. AU.addRequiredTransitive<RegisterCoalescer>(); AU.addRequired<CalculateSpillWeights>(); - if (PreSplitIntervals) - AU.addRequiredID(PreAllocSplittingID); AU.addRequiredID(LiveStacksID); AU.addPreservedID(LiveStacksID); AU.addRequired<MachineLoopInfo>(); @@ -401,11 +393,10 @@ INITIALIZE_PASS_BEGIN(RALinScan, "linearscan-regalloc", INITIALIZE_PASS_DEPENDENCY(LiveIntervals) INITIALIZE_PASS_DEPENDENCY(StrongPHIElimination) INITIALIZE_PASS_DEPENDENCY(CalculateSpillWeights) -INITIALIZE_PASS_DEPENDENCY(PreAllocSplitting) INITIALIZE_PASS_DEPENDENCY(LiveStacks) INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) INITIALIZE_PASS_DEPENDENCY(VirtRegMap) -INITIALIZE_AG_DEPENDENCY(RegisterCoalescer) +INITIALIZE_PASS_DEPENDENCY(RegisterCoalescer) INITIALIZE_AG_DEPENDENCY(AliasAnalysis) INITIALIZE_PASS_END(RALinScan, "linearscan-regalloc", "Linear Scan Register Allocator", false, false) diff --git a/lib/CodeGen/RegAllocPBQP.cpp b/lib/CodeGen/RegAllocPBQP.cpp index 49f8fb4..72230d4 100644 --- a/lib/CodeGen/RegAllocPBQP.cpp +++ b/lib/CodeGen/RegAllocPBQP.cpp @@ -35,6 +35,7 @@ #include "Splitter.h" #include "VirtRegMap.h" #include "VirtRegRewriter.h" +#include "RegisterCoalescer.h" #include "llvm/CodeGen/CalcSpillWeights.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveStackAnalysis.h" @@ -46,7 +47,6 @@ #include "llvm/CodeGen/PBQP/Graph.h" #include "llvm/CodeGen/PBQP/Heuristics/Briggs.h" #include "llvm/CodeGen/RegAllocRegistry.h" -#include "llvm/CodeGen/RegisterCoalescer.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" @@ -88,7 +88,7 @@ public: : MachineFunctionPass(ID), builder(b), customPassID(cPassID) { initializeSlotIndexesPass(*PassRegistry::getPassRegistry()); initializeLiveIntervalsPass(*PassRegistry::getPassRegistry()); - initializeRegisterCoalescerAnalysisGroup(*PassRegistry::getPassRegistry()); + initializeRegisterCoalescerPass(*PassRegistry::getPassRegistry()); initializeCalculateSpillWeightsPass(*PassRegistry::getPassRegistry()); initializeLiveStacksPass(*PassRegistry::getPassRegistry()); initializeMachineLoopInfoPass(*PassRegistry::getPassRegistry()); diff --git a/lib/CodeGen/RegisterClassInfo.h b/lib/CodeGen/RegisterClassInfo.h index 6f7d9c9..d21fd67 100644 --- a/lib/CodeGen/RegisterClassInfo.h +++ b/lib/CodeGen/RegisterClassInfo.h @@ -112,7 +112,7 @@ public: /// register, so a register allocator needs to track its liveness and /// availability. bool isAllocatable(unsigned PhysReg) const { - return TRI->get(PhysReg).inAllocatableClass && !isReserved(PhysReg); + return TRI->isInAllocatableClass(PhysReg) && !isReserved(PhysReg); } }; } // end namespace llvm diff --git a/lib/CodeGen/RegisterCoalescer.cpp b/lib/CodeGen/RegisterCoalescer.cpp index 407559a..d5025b9 100644 --- a/lib/CodeGen/RegisterCoalescer.cpp +++ b/lib/CodeGen/RegisterCoalescer.cpp @@ -13,38 +13,92 @@ // //===----------------------------------------------------------------------===// -#include "llvm/CodeGen/RegisterCoalescer.h" +#define DEBUG_TYPE "regcoalescing" +#include "RegisterCoalescer.h" +#include "VirtRegMap.h" +#include "LiveDebugVariables.h" + +#include "llvm/Pass.h" +#include "llvm/Value.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetRegisterInfo.h" -#include "llvm/Pass.h" - +#include "llvm/CodeGen/LiveIntervalAnalysis.h" +#include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineLoopInfo.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetOptions.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/ADT/OwningPtr.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/STLExtras.h" +#include <algorithm> +#include <cmath> using namespace llvm; -// Register the RegisterCoalescer interface, providing a nice name to refer to. -INITIALIZE_ANALYSIS_GROUP(RegisterCoalescer, "Register Coalescer", - SimpleRegisterCoalescing) -char RegisterCoalescer::ID = 0; +STATISTIC(numJoins , "Number of interval joins performed"); +STATISTIC(numCrossRCs , "Number of cross class joins performed"); +STATISTIC(numCommutes , "Number of instruction commuting performed"); +STATISTIC(numExtends , "Number of copies extended"); +STATISTIC(NumReMats , "Number of instructions re-materialized"); +STATISTIC(numPeep , "Number of identity moves eliminated after coalescing"); +STATISTIC(numAborts , "Number of times interval joining aborted"); -// RegisterCoalescer destructor: DO NOT move this to the header file -// for RegisterCoalescer or else clients of the RegisterCoalescer -// class may not depend on the RegisterCoalescer.o file in the current -// .a file, causing alias analysis support to not be included in the -// tool correctly! -// -RegisterCoalescer::~RegisterCoalescer() {} +static cl::opt<bool> +EnableJoining("join-liveintervals", + cl::desc("Coalesce copies (default=true)"), + cl::init(true)); + +static cl::opt<bool> +DisableCrossClassJoin("disable-cross-class-join", + cl::desc("Avoid coalescing cross register class copies"), + cl::init(false), cl::Hidden); -unsigned CoalescerPair::compose(unsigned a, unsigned b) const { +static cl::opt<bool> +EnablePhysicalJoin("join-physregs", + cl::desc("Join physical register copies"), + cl::init(false), cl::Hidden); + +static cl::opt<bool> +VerifyCoalescing("verify-coalescing", + cl::desc("Verify machine instrs before and after register coalescing"), + cl::Hidden); + +INITIALIZE_PASS_BEGIN(RegisterCoalescer, "simple-register-coalescing", + "Simple Register Coalescing", false, false) +INITIALIZE_PASS_DEPENDENCY(LiveIntervals) +INITIALIZE_PASS_DEPENDENCY(LiveDebugVariables) +INITIALIZE_PASS_DEPENDENCY(SlotIndexes) +INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) +INITIALIZE_PASS_DEPENDENCY(StrongPHIElimination) +INITIALIZE_PASS_DEPENDENCY(PHIElimination) +INITIALIZE_PASS_DEPENDENCY(TwoAddressInstructionPass) +INITIALIZE_AG_DEPENDENCY(AliasAnalysis) +INITIALIZE_PASS_END(RegisterCoalescer, "simple-register-coalescing", + "Simple Register Coalescing", false, false) + +char RegisterCoalescer::ID = 0; + +static unsigned compose(const TargetRegisterInfo &tri, unsigned a, unsigned b) { if (!a) return b; if (!b) return a; - return tri_.composeSubRegIndices(a, b); + return tri.composeSubRegIndices(a, b); } -bool CoalescerPair::isMoveInstr(const MachineInstr *MI, - unsigned &Src, unsigned &Dst, - unsigned &SrcSub, unsigned &DstSub) const { +static bool isMoveInstr(const TargetRegisterInfo &tri, const MachineInstr *MI, + unsigned &Src, unsigned &Dst, + unsigned &SrcSub, unsigned &DstSub) { if (MI->isCopy()) { Dst = MI->getOperand(0).getReg(); DstSub = MI->getOperand(0).getSubReg(); @@ -52,7 +106,8 @@ bool CoalescerPair::isMoveInstr(const MachineInstr *MI, SrcSub = MI->getOperand(1).getSubReg(); } else if (MI->isSubregToReg()) { Dst = MI->getOperand(0).getReg(); - DstSub = compose(MI->getOperand(0).getSubReg(), MI->getOperand(3).getImm()); + DstSub = compose(tri, MI->getOperand(0).getSubReg(), + MI->getOperand(3).getImm()); Src = MI->getOperand(2).getReg(); SrcSub = MI->getOperand(2).getSubReg(); } else @@ -66,7 +121,7 @@ bool CoalescerPair::setRegisters(const MachineInstr *MI) { flipped_ = crossClass_ = false; unsigned Src, Dst, SrcSub, DstSub; - if (!isMoveInstr(MI, Src, Dst, SrcSub, DstSub)) + if (!isMoveInstr(tri_, MI, Src, Dst, SrcSub, DstSub)) return false; partial_ = SrcSub || DstSub; @@ -156,7 +211,7 @@ bool CoalescerPair::isCoalescable(const MachineInstr *MI) const { if (!MI) return false; unsigned Src, Dst, SrcSub, DstSub; - if (!isMoveInstr(MI, Src, Dst, SrcSub, DstSub)) + if (!isMoveInstr(tri_, MI, Src, Dst, SrcSub, DstSub)) return false; // Find the virtual register that is srcReg_. @@ -185,13 +240,1550 @@ bool CoalescerPair::isCoalescable(const MachineInstr *MI) const { if (dstReg_ != Dst) return false; // Registers match, do the subregisters line up? - return compose(subIdx_, SrcSub) == DstSub; + return compose(tri_, subIdx_, SrcSub) == DstSub; + } +} + +void RegisterCoalescer::getAnalysisUsage(AnalysisUsage &AU) const { + AU.setPreservesCFG(); + AU.addRequired<AliasAnalysis>(); + AU.addRequired<LiveIntervals>(); + AU.addPreserved<LiveIntervals>(); + AU.addRequired<LiveDebugVariables>(); + AU.addPreserved<LiveDebugVariables>(); + AU.addPreserved<SlotIndexes>(); + AU.addRequired<MachineLoopInfo>(); + AU.addPreserved<MachineLoopInfo>(); + AU.addPreservedID(MachineDominatorsID); + AU.addPreservedID(StrongPHIEliminationID); + AU.addPreservedID(PHIEliminationID); + AU.addPreservedID(TwoAddressInstructionPassID); + MachineFunctionPass::getAnalysisUsage(AU); +} + +void RegisterCoalescer::markAsJoined(MachineInstr *CopyMI) { + /// Joined copies are not deleted immediately, but kept in JoinedCopies. + JoinedCopies.insert(CopyMI); + + /// Mark all register operands of CopyMI as <undef> so they won't affect dead + /// code elimination. + for (MachineInstr::mop_iterator I = CopyMI->operands_begin(), + E = CopyMI->operands_end(); I != E; ++I) + if (I->isReg()) + I->setIsUndef(true); +} + +/// AdjustCopiesBackFrom - We found a non-trivially-coalescable copy with IntA +/// being the source and IntB being the dest, thus this defines a value number +/// in IntB. If the source value number (in IntA) is defined by a copy from B, +/// see if we can merge these two pieces of B into a single value number, +/// eliminating a copy. For example: +/// +/// A3 = B0 +/// ... +/// B1 = A3 <- this copy +/// +/// In this case, B0 can be extended to where the B1 copy lives, allowing the B1 +/// value number to be replaced with B0 (which simplifies the B liveinterval). +/// +/// This returns true if an interval was modified. +/// +bool RegisterCoalescer::AdjustCopiesBackFrom(const CoalescerPair &CP, + MachineInstr *CopyMI) { + // Bail if there is no dst interval - can happen when merging physical subreg + // operations. + if (!li_->hasInterval(CP.getDstReg())) + return false; + + LiveInterval &IntA = + li_->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg()); + LiveInterval &IntB = + li_->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg()); + SlotIndex CopyIdx = li_->getInstructionIndex(CopyMI).getDefIndex(); + + // BValNo is a value number in B that is defined by a copy from A. 'B3' in + // the example above. + LiveInterval::iterator BLR = IntB.FindLiveRangeContaining(CopyIdx); + if (BLR == IntB.end()) return false; + VNInfo *BValNo = BLR->valno; + + // Get the location that B is defined at. Two options: either this value has + // an unknown definition point or it is defined at CopyIdx. If unknown, we + // can't process it. + if (!BValNo->isDefByCopy()) return false; + assert(BValNo->def == CopyIdx && "Copy doesn't define the value?"); + + // AValNo is the value number in A that defines the copy, A3 in the example. + SlotIndex CopyUseIdx = CopyIdx.getUseIndex(); + LiveInterval::iterator ALR = IntA.FindLiveRangeContaining(CopyUseIdx); + // The live range might not exist after fun with physreg coalescing. + if (ALR == IntA.end()) return false; + VNInfo *AValNo = ALR->valno; + // If it's re-defined by an early clobber somewhere in the live range, then + // it's not safe to eliminate the copy. FIXME: This is a temporary workaround. + // See PR3149: + // 172 %ECX<def> = MOV32rr %reg1039<kill> + // 180 INLINEASM <es:subl $5,$1 + // sbbl $3,$0>, 10, %EAX<def>, 14, %ECX<earlyclobber,def>, 9, + // %EAX<kill>, + // 36, <fi#0>, 1, %reg0, 0, 9, %ECX<kill>, 36, <fi#1>, 1, %reg0, 0 + // 188 %EAX<def> = MOV32rr %EAX<kill> + // 196 %ECX<def> = MOV32rr %ECX<kill> + // 204 %ECX<def> = MOV32rr %ECX<kill> + // 212 %EAX<def> = MOV32rr %EAX<kill> + // 220 %EAX<def> = MOV32rr %EAX + // 228 %reg1039<def> = MOV32rr %ECX<kill> + // The early clobber operand ties ECX input to the ECX def. + // + // The live interval of ECX is represented as this: + // %reg20,inf = [46,47:1)[174,230:0) 0@174-(230) 1@46-(47) + // The coalescer has no idea there was a def in the middle of [174,230]. + if (AValNo->hasRedefByEC()) + return false; + + // If AValNo is defined as a copy from IntB, we can potentially process this. + // Get the instruction that defines this value number. + if (!CP.isCoalescable(AValNo->getCopy())) + return false; + + // Get the LiveRange in IntB that this value number starts with. + LiveInterval::iterator ValLR = + IntB.FindLiveRangeContaining(AValNo->def.getPrevSlot()); + if (ValLR == IntB.end()) + return false; + + // Make sure that the end of the live range is inside the same block as + // CopyMI. + MachineInstr *ValLREndInst = + li_->getInstructionFromIndex(ValLR->end.getPrevSlot()); + if (!ValLREndInst || ValLREndInst->getParent() != CopyMI->getParent()) + return false; + + // Okay, we now know that ValLR ends in the same block that the CopyMI + // live-range starts. If there are no intervening live ranges between them in + // IntB, we can merge them. + if (ValLR+1 != BLR) return false; + + // If a live interval is a physical register, conservatively check if any + // of its aliases is overlapping the live interval of the virtual register. + // If so, do not coalesce. + if (TargetRegisterInfo::isPhysicalRegister(IntB.reg)) { + for (const unsigned *AS = tri_->getAliasSet(IntB.reg); *AS; ++AS) + if (li_->hasInterval(*AS) && IntA.overlaps(li_->getInterval(*AS))) { + DEBUG({ + dbgs() << "\t\tInterfere with alias "; + li_->getInterval(*AS).print(dbgs(), tri_); + }); + return false; + } + } + + DEBUG({ + dbgs() << "Extending: "; + IntB.print(dbgs(), tri_); + }); + + SlotIndex FillerStart = ValLR->end, FillerEnd = BLR->start; + // We are about to delete CopyMI, so need to remove it as the 'instruction + // that defines this value #'. Update the valnum with the new defining + // instruction #. + BValNo->def = FillerStart; + BValNo->setCopy(0); + + // Okay, we can merge them. We need to insert a new liverange: + // [ValLR.end, BLR.begin) of either value number, then we merge the + // two value numbers. + IntB.addRange(LiveRange(FillerStart, FillerEnd, BValNo)); + + // If the IntB live range is assigned to a physical register, and if that + // physreg has sub-registers, update their live intervals as well. + if (TargetRegisterInfo::isPhysicalRegister(IntB.reg)) { + for (const unsigned *SR = tri_->getSubRegisters(IntB.reg); *SR; ++SR) { + if (!li_->hasInterval(*SR)) + continue; + LiveInterval &SRLI = li_->getInterval(*SR); + SRLI.addRange(LiveRange(FillerStart, FillerEnd, + SRLI.getNextValue(FillerStart, 0, + li_->getVNInfoAllocator()))); + } + } + + // Okay, merge "B1" into the same value number as "B0". + if (BValNo != ValLR->valno) { + // If B1 is killed by a PHI, then the merged live range must also be killed + // by the same PHI, as B0 and B1 can not overlap. + bool HasPHIKill = BValNo->hasPHIKill(); + IntB.MergeValueNumberInto(BValNo, ValLR->valno); + if (HasPHIKill) + ValLR->valno->setHasPHIKill(true); + } + DEBUG({ + dbgs() << " result = "; + IntB.print(dbgs(), tri_); + dbgs() << "\n"; + }); + + // If the source instruction was killing the source register before the + // merge, unset the isKill marker given the live range has been extended. + int UIdx = ValLREndInst->findRegisterUseOperandIdx(IntB.reg, true); + if (UIdx != -1) { + ValLREndInst->getOperand(UIdx).setIsKill(false); + } + + // If the copy instruction was killing the destination register before the + // merge, find the last use and trim the live range. That will also add the + // isKill marker. + if (ALR->end == CopyIdx) + li_->shrinkToUses(&IntA); + + ++numExtends; + return true; +} + +/// HasOtherReachingDefs - Return true if there are definitions of IntB +/// other than BValNo val# that can reach uses of AValno val# of IntA. +bool RegisterCoalescer::HasOtherReachingDefs(LiveInterval &IntA, + LiveInterval &IntB, + VNInfo *AValNo, + VNInfo *BValNo) { + for (LiveInterval::iterator AI = IntA.begin(), AE = IntA.end(); + AI != AE; ++AI) { + if (AI->valno != AValNo) continue; + LiveInterval::Ranges::iterator BI = + std::upper_bound(IntB.ranges.begin(), IntB.ranges.end(), AI->start); + if (BI != IntB.ranges.begin()) + --BI; + for (; BI != IntB.ranges.end() && AI->end >= BI->start; ++BI) { + if (BI->valno == BValNo) + continue; + if (BI->start <= AI->start && BI->end > AI->start) + return true; + if (BI->start > AI->start && BI->start < AI->end) + return true; + } + } + return false; +} + +/// RemoveCopyByCommutingDef - We found a non-trivially-coalescable copy with +/// IntA being the source and IntB being the dest, thus this defines a value +/// number in IntB. If the source value number (in IntA) is defined by a +/// commutable instruction and its other operand is coalesced to the copy dest +/// register, see if we can transform the copy into a noop by commuting the +/// definition. For example, +/// +/// A3 = op A2 B0<kill> +/// ... +/// B1 = A3 <- this copy +/// ... +/// = op A3 <- more uses +/// +/// ==> +/// +/// B2 = op B0 A2<kill> +/// ... +/// B1 = B2 <- now an identify copy +/// ... +/// = op B2 <- more uses +/// +/// This returns true if an interval was modified. +/// +bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP, + MachineInstr *CopyMI) { + // FIXME: For now, only eliminate the copy by commuting its def when the + // source register is a virtual register. We want to guard against cases + // where the copy is a back edge copy and commuting the def lengthen the + // live interval of the source register to the entire loop. + if (CP.isPhys() && CP.isFlipped()) + return false; + + // Bail if there is no dst interval. + if (!li_->hasInterval(CP.getDstReg())) + return false; + + SlotIndex CopyIdx = li_->getInstructionIndex(CopyMI).getDefIndex(); + + LiveInterval &IntA = + li_->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg()); + LiveInterval &IntB = + li_->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg()); + + // BValNo is a value number in B that is defined by a copy from A. 'B3' in + // the example above. + VNInfo *BValNo = IntB.getVNInfoAt(CopyIdx); + if (!BValNo || !BValNo->isDefByCopy()) + return false; + + assert(BValNo->def == CopyIdx && "Copy doesn't define the value?"); + + // AValNo is the value number in A that defines the copy, A3 in the example. + VNInfo *AValNo = IntA.getVNInfoAt(CopyIdx.getUseIndex()); + assert(AValNo && "COPY source not live"); + + // If other defs can reach uses of this def, then it's not safe to perform + // the optimization. + if (AValNo->isPHIDef() || AValNo->isUnused() || AValNo->hasPHIKill()) + return false; + MachineInstr *DefMI = li_->getInstructionFromIndex(AValNo->def); + if (!DefMI) + return false; + const MCInstrDesc &MCID = DefMI->getDesc(); + if (!MCID.isCommutable()) + return false; + // If DefMI is a two-address instruction then commuting it will change the + // destination register. + int DefIdx = DefMI->findRegisterDefOperandIdx(IntA.reg); + assert(DefIdx != -1); + unsigned UseOpIdx; + if (!DefMI->isRegTiedToUseOperand(DefIdx, &UseOpIdx)) + return false; + unsigned Op1, Op2, NewDstIdx; + if (!tii_->findCommutedOpIndices(DefMI, Op1, Op2)) + return false; + if (Op1 == UseOpIdx) + NewDstIdx = Op2; + else if (Op2 == UseOpIdx) + NewDstIdx = Op1; + else + return false; + + MachineOperand &NewDstMO = DefMI->getOperand(NewDstIdx); + unsigned NewReg = NewDstMO.getReg(); + if (NewReg != IntB.reg || !NewDstMO.isKill()) + return false; + + // Make sure there are no other definitions of IntB that would reach the + // uses which the new definition can reach. + if (HasOtherReachingDefs(IntA, IntB, AValNo, BValNo)) + return false; + + // Abort if the aliases of IntB.reg have values that are not simply the + // clobbers from the superreg. + if (TargetRegisterInfo::isPhysicalRegister(IntB.reg)) + for (const unsigned *AS = tri_->getAliasSet(IntB.reg); *AS; ++AS) + if (li_->hasInterval(*AS) && + HasOtherReachingDefs(IntA, li_->getInterval(*AS), AValNo, 0)) + return false; + + // If some of the uses of IntA.reg is already coalesced away, return false. + // It's not possible to determine whether it's safe to perform the coalescing. + for (MachineRegisterInfo::use_nodbg_iterator UI = + mri_->use_nodbg_begin(IntA.reg), + UE = mri_->use_nodbg_end(); UI != UE; ++UI) { + MachineInstr *UseMI = &*UI; + SlotIndex UseIdx = li_->getInstructionIndex(UseMI); + LiveInterval::iterator ULR = IntA.FindLiveRangeContaining(UseIdx); + if (ULR == IntA.end()) + continue; + if (ULR->valno == AValNo && JoinedCopies.count(UseMI)) + return false; + } + + DEBUG(dbgs() << "\tRemoveCopyByCommutingDef: " << AValNo->def << '\t' + << *DefMI); + + // At this point we have decided that it is legal to do this + // transformation. Start by commuting the instruction. + MachineBasicBlock *MBB = DefMI->getParent(); + MachineInstr *NewMI = tii_->commuteInstruction(DefMI); + if (!NewMI) + return false; + if (TargetRegisterInfo::isVirtualRegister(IntA.reg) && + TargetRegisterInfo::isVirtualRegister(IntB.reg) && + !mri_->constrainRegClass(IntB.reg, mri_->getRegClass(IntA.reg))) + return false; + if (NewMI != DefMI) { + li_->ReplaceMachineInstrInMaps(DefMI, NewMI); + MBB->insert(DefMI, NewMI); + MBB->erase(DefMI); + } + unsigned OpIdx = NewMI->findRegisterUseOperandIdx(IntA.reg, false); + NewMI->getOperand(OpIdx).setIsKill(); + + // If ALR and BLR overlaps and end of BLR extends beyond end of ALR, e.g. + // A = or A, B + // ... + // B = A + // ... + // C = A<kill> + // ... + // = B + + // Update uses of IntA of the specific Val# with IntB. + for (MachineRegisterInfo::use_iterator UI = mri_->use_begin(IntA.reg), + UE = mri_->use_end(); UI != UE;) { + MachineOperand &UseMO = UI.getOperand(); + MachineInstr *UseMI = &*UI; + ++UI; + if (JoinedCopies.count(UseMI)) + continue; + if (UseMI->isDebugValue()) { + // FIXME These don't have an instruction index. Not clear we have enough + // info to decide whether to do this replacement or not. For now do it. + UseMO.setReg(NewReg); + continue; + } + SlotIndex UseIdx = li_->getInstructionIndex(UseMI).getUseIndex(); + LiveInterval::iterator ULR = IntA.FindLiveRangeContaining(UseIdx); + if (ULR == IntA.end() || ULR->valno != AValNo) + continue; + if (TargetRegisterInfo::isPhysicalRegister(NewReg)) + UseMO.substPhysReg(NewReg, *tri_); + else + UseMO.setReg(NewReg); + if (UseMI == CopyMI) + continue; + if (!UseMI->isCopy()) + continue; + if (UseMI->getOperand(0).getReg() != IntB.reg || + UseMI->getOperand(0).getSubReg()) + continue; + + // This copy will become a noop. If it's defining a new val#, merge it into + // BValNo. + SlotIndex DefIdx = UseIdx.getDefIndex(); + VNInfo *DVNI = IntB.getVNInfoAt(DefIdx); + if (!DVNI) + continue; + DEBUG(dbgs() << "\t\tnoop: " << DefIdx << '\t' << *UseMI); + assert(DVNI->def == DefIdx); + BValNo = IntB.MergeValueNumberInto(BValNo, DVNI); + markAsJoined(UseMI); + } + + // Extend BValNo by merging in IntA live ranges of AValNo. Val# definition + // is updated. + VNInfo *ValNo = BValNo; + ValNo->def = AValNo->def; + ValNo->setCopy(0); + for (LiveInterval::iterator AI = IntA.begin(), AE = IntA.end(); + AI != AE; ++AI) { + if (AI->valno != AValNo) continue; + IntB.addRange(LiveRange(AI->start, AI->end, ValNo)); + } + DEBUG(dbgs() << "\t\textended: " << IntB << '\n'); + + IntA.removeValNo(AValNo); + DEBUG(dbgs() << "\t\ttrimmed: " << IntA << '\n'); + ++numCommutes; + return true; +} + +/// ReMaterializeTrivialDef - If the source of a copy is defined by a trivial +/// computation, replace the copy by rematerialize the definition. +bool RegisterCoalescer::ReMaterializeTrivialDef(LiveInterval &SrcInt, + bool preserveSrcInt, + unsigned DstReg, + unsigned DstSubIdx, + MachineInstr *CopyMI) { + SlotIndex CopyIdx = li_->getInstructionIndex(CopyMI).getUseIndex(); + LiveInterval::iterator SrcLR = SrcInt.FindLiveRangeContaining(CopyIdx); + assert(SrcLR != SrcInt.end() && "Live range not found!"); + VNInfo *ValNo = SrcLR->valno; + // If other defs can reach uses of this def, then it's not safe to perform + // the optimization. + if (ValNo->isPHIDef() || ValNo->isUnused() || ValNo->hasPHIKill()) + return false; + MachineInstr *DefMI = li_->getInstructionFromIndex(ValNo->def); + if (!DefMI) + return false; + assert(DefMI && "Defining instruction disappeared"); + const MCInstrDesc &MCID = DefMI->getDesc(); + if (!MCID.isAsCheapAsAMove()) + return false; + if (!tii_->isTriviallyReMaterializable(DefMI, AA)) + return false; + bool SawStore = false; + if (!DefMI->isSafeToMove(tii_, AA, SawStore)) + return false; + if (MCID.getNumDefs() != 1) + return false; + if (!DefMI->isImplicitDef()) { + // Make sure the copy destination register class fits the instruction + // definition register class. The mismatch can happen as a result of earlier + // extract_subreg, insert_subreg, subreg_to_reg coalescing. + const TargetRegisterClass *RC = tii_->getRegClass(MCID, 0, tri_); + if (TargetRegisterInfo::isVirtualRegister(DstReg)) { + if (mri_->getRegClass(DstReg) != RC) + return false; + } else if (!RC->contains(DstReg)) + return false; } + + // If destination register has a sub-register index on it, make sure it + // matches the instruction register class. + if (DstSubIdx) { + const MCInstrDesc &MCID = DefMI->getDesc(); + if (MCID.getNumDefs() != 1) + return false; + const TargetRegisterClass *DstRC = mri_->getRegClass(DstReg); + const TargetRegisterClass *DstSubRC = + DstRC->getSubRegisterRegClass(DstSubIdx); + const TargetRegisterClass *DefRC = tii_->getRegClass(MCID, 0, tri_); + if (DefRC == DstRC) + DstSubIdx = 0; + else if (DefRC != DstSubRC) + return false; + } + + RemoveCopyFlag(DstReg, CopyMI); + + MachineBasicBlock *MBB = CopyMI->getParent(); + MachineBasicBlock::iterator MII = + llvm::next(MachineBasicBlock::iterator(CopyMI)); + tii_->reMaterialize(*MBB, MII, DstReg, DstSubIdx, DefMI, *tri_); + MachineInstr *NewMI = prior(MII); + + // CopyMI may have implicit operands, transfer them over to the newly + // rematerialized instruction. And update implicit def interval valnos. + for (unsigned i = CopyMI->getDesc().getNumOperands(), + e = CopyMI->getNumOperands(); i != e; ++i) { + MachineOperand &MO = CopyMI->getOperand(i); + if (MO.isReg() && MO.isImplicit()) + NewMI->addOperand(MO); + if (MO.isDef()) + RemoveCopyFlag(MO.getReg(), CopyMI); + } + + NewMI->copyImplicitOps(CopyMI); + li_->ReplaceMachineInstrInMaps(CopyMI, NewMI); + CopyMI->eraseFromParent(); + ReMatCopies.insert(CopyMI); + ReMatDefs.insert(DefMI); + DEBUG(dbgs() << "Remat: " << *NewMI); + ++NumReMats; + + // The source interval can become smaller because we removed a use. + if (preserveSrcInt) + li_->shrinkToUses(&SrcInt); + + return true; } -// Because of the way .a files work, we must force the SimpleRC -// implementation to be pulled in if the RegisterCoalescer classes are -// pulled in. Otherwise we run the risk of RegisterCoalescer being -// used, but the default implementation not being linked into the tool -// that uses it. -DEFINING_FILE_FOR(RegisterCoalescer) +/// UpdateRegDefsUses - Replace all defs and uses of SrcReg to DstReg and +/// update the subregister number if it is not zero. If DstReg is a +/// physical register and the existing subregister number of the def / use +/// being updated is not zero, make sure to set it to the correct physical +/// subregister. +void +RegisterCoalescer::UpdateRegDefsUses(const CoalescerPair &CP) { + bool DstIsPhys = CP.isPhys(); + unsigned SrcReg = CP.getSrcReg(); + unsigned DstReg = CP.getDstReg(); + unsigned SubIdx = CP.getSubIdx(); + + // Update LiveDebugVariables. + ldv_->renameRegister(SrcReg, DstReg, SubIdx); + + for (MachineRegisterInfo::reg_iterator I = mri_->reg_begin(SrcReg); + MachineInstr *UseMI = I.skipInstruction();) { + // A PhysReg copy that won't be coalesced can perhaps be rematerialized + // instead. + if (DstIsPhys) { + if (UseMI->isCopy() && + !UseMI->getOperand(1).getSubReg() && + !UseMI->getOperand(0).getSubReg() && + UseMI->getOperand(1).getReg() == SrcReg && + UseMI->getOperand(0).getReg() != SrcReg && + UseMI->getOperand(0).getReg() != DstReg && + !JoinedCopies.count(UseMI) && + ReMaterializeTrivialDef(li_->getInterval(SrcReg), false, + UseMI->getOperand(0).getReg(), 0, UseMI)) + continue; + } + + SmallVector<unsigned,8> Ops; + bool Reads, Writes; + tie(Reads, Writes) = UseMI->readsWritesVirtualRegister(SrcReg, &Ops); + bool Kills = false, Deads = false; + + // Replace SrcReg with DstReg in all UseMI operands. + for (unsigned i = 0, e = Ops.size(); i != e; ++i) { + MachineOperand &MO = UseMI->getOperand(Ops[i]); + Kills |= MO.isKill(); + Deads |= MO.isDead(); + + if (DstIsPhys) + MO.substPhysReg(DstReg, *tri_); + else + MO.substVirtReg(DstReg, SubIdx, *tri_); + } + + // This instruction is a copy that will be removed. + if (JoinedCopies.count(UseMI)) + continue; + + if (SubIdx) { + // If UseMI was a simple SrcReg def, make sure we didn't turn it into a + // read-modify-write of DstReg. + if (Deads) + UseMI->addRegisterDead(DstReg, tri_); + else if (!Reads && Writes) + UseMI->addRegisterDefined(DstReg, tri_); + + // Kill flags apply to the whole physical register. + if (DstIsPhys && Kills) + UseMI->addRegisterKilled(DstReg, tri_); + } + + DEBUG({ + dbgs() << "\t\tupdated: "; + if (!UseMI->isDebugValue()) + dbgs() << li_->getInstructionIndex(UseMI) << "\t"; + dbgs() << *UseMI; + }); + } +} + +/// removeIntervalIfEmpty - Check if the live interval of a physical register +/// is empty, if so remove it and also remove the empty intervals of its +/// sub-registers. Return true if live interval is removed. +static bool removeIntervalIfEmpty(LiveInterval &li, LiveIntervals *li_, + const TargetRegisterInfo *tri_) { + if (li.empty()) { + if (TargetRegisterInfo::isPhysicalRegister(li.reg)) + for (const unsigned* SR = tri_->getSubRegisters(li.reg); *SR; ++SR) { + if (!li_->hasInterval(*SR)) + continue; + LiveInterval &sli = li_->getInterval(*SR); + if (sli.empty()) + li_->removeInterval(*SR); + } + li_->removeInterval(li.reg); + return true; + } + return false; +} + +/// RemoveDeadDef - If a def of a live interval is now determined dead, remove +/// the val# it defines. If the live interval becomes empty, remove it as well. +bool RegisterCoalescer::RemoveDeadDef(LiveInterval &li, + MachineInstr *DefMI) { + SlotIndex DefIdx = li_->getInstructionIndex(DefMI).getDefIndex(); + LiveInterval::iterator MLR = li.FindLiveRangeContaining(DefIdx); + if (DefIdx != MLR->valno->def) + return false; + li.removeValNo(MLR->valno); + return removeIntervalIfEmpty(li, li_, tri_); +} + +void RegisterCoalescer::RemoveCopyFlag(unsigned DstReg, + const MachineInstr *CopyMI) { + SlotIndex DefIdx = li_->getInstructionIndex(CopyMI).getDefIndex(); + if (li_->hasInterval(DstReg)) { + LiveInterval &LI = li_->getInterval(DstReg); + if (const LiveRange *LR = LI.getLiveRangeContaining(DefIdx)) + if (LR->valno->def == DefIdx) + LR->valno->setCopy(0); + } + if (!TargetRegisterInfo::isPhysicalRegister(DstReg)) + return; + for (const unsigned* AS = tri_->getAliasSet(DstReg); *AS; ++AS) { + if (!li_->hasInterval(*AS)) + continue; + LiveInterval &LI = li_->getInterval(*AS); + if (const LiveRange *LR = LI.getLiveRangeContaining(DefIdx)) + if (LR->valno->def == DefIdx) + LR->valno->setCopy(0); + } +} + +/// shouldJoinPhys - Return true if a copy involving a physreg should be joined. +/// We need to be careful about coalescing a source physical register with a +/// virtual register. Once the coalescing is done, it cannot be broken and these +/// are not spillable! If the destination interval uses are far away, think +/// twice about coalescing them! +bool RegisterCoalescer::shouldJoinPhys(CoalescerPair &CP) { + bool Allocatable = li_->isAllocatable(CP.getDstReg()); + LiveInterval &JoinVInt = li_->getInterval(CP.getSrcReg()); + + /// Always join simple intervals that are defined by a single copy from a + /// reserved register. This doesn't increase register pressure, so it is + /// always beneficial. + if (!Allocatable && CP.isFlipped() && JoinVInt.containsOneValue()) + return true; + + if (!EnablePhysicalJoin) { + DEBUG(dbgs() << "\tPhysreg joins disabled.\n"); + return false; + } + + // Only coalesce to allocatable physreg, we don't want to risk modifying + // reserved registers. + if (!Allocatable) { + DEBUG(dbgs() << "\tRegister is an unallocatable physreg.\n"); + return false; // Not coalescable. + } + + // Don't join with physregs that have a ridiculous number of live + // ranges. The data structure performance is really bad when that + // happens. + if (li_->hasInterval(CP.getDstReg()) && + li_->getInterval(CP.getDstReg()).ranges.size() > 1000) { + ++numAborts; + DEBUG(dbgs() + << "\tPhysical register live interval too complicated, abort!\n"); + return false; + } + + // FIXME: Why are we skipping this test for partial copies? + // CodeGen/X86/phys_subreg_coalesce-3.ll needs it. + if (!CP.isPartial()) { + const TargetRegisterClass *RC = mri_->getRegClass(CP.getSrcReg()); + unsigned Threshold = RegClassInfo.getNumAllocatableRegs(RC) * 2; + unsigned Length = li_->getApproximateInstructionCount(JoinVInt); + if (Length > Threshold) { + ++numAborts; + DEBUG(dbgs() << "\tMay tie down a physical register, abort!\n"); + return false; + } + } + return true; +} + +/// isWinToJoinCrossClass - Return true if it's profitable to coalesce +/// two virtual registers from different register classes. +bool +RegisterCoalescer::isWinToJoinCrossClass(unsigned SrcReg, + unsigned DstReg, + const TargetRegisterClass *SrcRC, + const TargetRegisterClass *DstRC, + const TargetRegisterClass *NewRC) { + unsigned NewRCCount = RegClassInfo.getNumAllocatableRegs(NewRC); + // This heuristics is good enough in practice, but it's obviously not *right*. + // 4 is a magic number that works well enough for x86, ARM, etc. It filter + // out all but the most restrictive register classes. + if (NewRCCount > 4 || + // Early exit if the function is fairly small, coalesce aggressively if + // that's the case. For really special register classes with 3 or + // fewer registers, be a bit more careful. + (li_->getFuncInstructionCount() / NewRCCount) < 8) + return true; + LiveInterval &SrcInt = li_->getInterval(SrcReg); + LiveInterval &DstInt = li_->getInterval(DstReg); + unsigned SrcSize = li_->getApproximateInstructionCount(SrcInt); + unsigned DstSize = li_->getApproximateInstructionCount(DstInt); + + // Coalesce aggressively if the intervals are small compared to the number of + // registers in the new class. The number 4 is fairly arbitrary, chosen to be + // less aggressive than the 8 used for the whole function size. + const unsigned ThresSize = 4 * NewRCCount; + if (SrcSize <= ThresSize && DstSize <= ThresSize) + return true; + + // Estimate *register use density*. If it doubles or more, abort. + unsigned SrcUses = std::distance(mri_->use_nodbg_begin(SrcReg), + mri_->use_nodbg_end()); + unsigned DstUses = std::distance(mri_->use_nodbg_begin(DstReg), + mri_->use_nodbg_end()); + unsigned NewUses = SrcUses + DstUses; + unsigned NewSize = SrcSize + DstSize; + if (SrcRC != NewRC && SrcSize > ThresSize) { + unsigned SrcRCCount = RegClassInfo.getNumAllocatableRegs(SrcRC); + if (NewUses*SrcSize*SrcRCCount > 2*SrcUses*NewSize*NewRCCount) + return false; + } + if (DstRC != NewRC && DstSize > ThresSize) { + unsigned DstRCCount = RegClassInfo.getNumAllocatableRegs(DstRC); + if (NewUses*DstSize*DstRCCount > 2*DstUses*NewSize*NewRCCount) + return false; + } + return true; +} + + +/// JoinCopy - Attempt to join intervals corresponding to SrcReg/DstReg, +/// which are the src/dst of the copy instruction CopyMI. This returns true +/// if the copy was successfully coalesced away. If it is not currently +/// possible to coalesce this interval, but it may be possible if other +/// things get coalesced, then it returns true by reference in 'Again'. +bool RegisterCoalescer::JoinCopy(MachineInstr *CopyMI, bool &Again) { + + Again = false; + if (JoinedCopies.count(CopyMI) || ReMatCopies.count(CopyMI)) + return false; // Already done. + + DEBUG(dbgs() << li_->getInstructionIndex(CopyMI) << '\t' << *CopyMI); + + CoalescerPair CP(*tii_, *tri_); + if (!CP.setRegisters(CopyMI)) { + DEBUG(dbgs() << "\tNot coalescable.\n"); + return false; + } + + // If they are already joined we continue. + if (CP.getSrcReg() == CP.getDstReg()) { + markAsJoined(CopyMI); + DEBUG(dbgs() << "\tCopy already coalesced.\n"); + return false; // Not coalescable. + } + + DEBUG(dbgs() << "\tConsidering merging " << PrintReg(CP.getSrcReg(), tri_) + << " with " << PrintReg(CP.getDstReg(), tri_, CP.getSubIdx()) + << "\n"); + + // Enforce policies. + if (CP.isPhys()) { + if (!shouldJoinPhys(CP)) { + // Before giving up coalescing, if definition of source is defined by + // trivial computation, try rematerializing it. + if (!CP.isFlipped() && + ReMaterializeTrivialDef(li_->getInterval(CP.getSrcReg()), true, + CP.getDstReg(), 0, CopyMI)) + return true; + return false; + } + } else { + // Avoid constraining virtual register regclass too much. + if (CP.isCrossClass()) { + DEBUG(dbgs() << "\tCross-class to " << CP.getNewRC()->getName() << ".\n"); + if (DisableCrossClassJoin) { + DEBUG(dbgs() << "\tCross-class joins disabled.\n"); + return false; + } + if (!isWinToJoinCrossClass(CP.getSrcReg(), CP.getDstReg(), + mri_->getRegClass(CP.getSrcReg()), + mri_->getRegClass(CP.getDstReg()), + CP.getNewRC())) { + DEBUG(dbgs() << "\tAvoid coalescing to constrained register class.\n"); + Again = true; // May be possible to coalesce later. + return false; + } + } + + // When possible, let DstReg be the larger interval. + if (!CP.getSubIdx() && li_->getInterval(CP.getSrcReg()).ranges.size() > + li_->getInterval(CP.getDstReg()).ranges.size()) + CP.flip(); + } + + // Okay, attempt to join these two intervals. On failure, this returns false. + // Otherwise, if one of the intervals being joined is a physreg, this method + // always canonicalizes DstInt to be it. The output "SrcInt" will not have + // been modified, so we can use this information below to update aliases. + if (!JoinIntervals(CP)) { + // Coalescing failed. + + // If definition of source is defined by trivial computation, try + // rematerializing it. + if (!CP.isFlipped() && + ReMaterializeTrivialDef(li_->getInterval(CP.getSrcReg()), true, + CP.getDstReg(), 0, CopyMI)) + return true; + + // If we can eliminate the copy without merging the live ranges, do so now. + if (!CP.isPartial()) { + if (AdjustCopiesBackFrom(CP, CopyMI) || + RemoveCopyByCommutingDef(CP, CopyMI)) { + markAsJoined(CopyMI); + DEBUG(dbgs() << "\tTrivial!\n"); + return true; + } + } + + // Otherwise, we are unable to join the intervals. + DEBUG(dbgs() << "\tInterference!\n"); + Again = true; // May be possible to coalesce later. + return false; + } + + // Coalescing to a virtual register that is of a sub-register class of the + // other. Make sure the resulting register is set to the right register class. + if (CP.isCrossClass()) { + ++numCrossRCs; + mri_->setRegClass(CP.getDstReg(), CP.getNewRC()); + } + + // Remember to delete the copy instruction. + markAsJoined(CopyMI); + + UpdateRegDefsUses(CP); + + // If we have extended the live range of a physical register, make sure we + // update live-in lists as well. + if (CP.isPhys()) { + SmallVector<MachineBasicBlock*, 16> BlockSeq; + // JoinIntervals invalidates the VNInfos in SrcInt, but we only need the + // ranges for this, and they are preserved. + LiveInterval &SrcInt = li_->getInterval(CP.getSrcReg()); + for (LiveInterval::const_iterator I = SrcInt.begin(), E = SrcInt.end(); + I != E; ++I ) { + li_->findLiveInMBBs(I->start, I->end, BlockSeq); + for (unsigned idx = 0, size = BlockSeq.size(); idx != size; ++idx) { + MachineBasicBlock &block = *BlockSeq[idx]; + if (!block.isLiveIn(CP.getDstReg())) + block.addLiveIn(CP.getDstReg()); + } + BlockSeq.clear(); + } + } + + // SrcReg is guarateed to be the register whose live interval that is + // being merged. + li_->removeInterval(CP.getSrcReg()); + + // Update regalloc hint. + tri_->UpdateRegAllocHint(CP.getSrcReg(), CP.getDstReg(), *mf_); + + DEBUG({ + LiveInterval &DstInt = li_->getInterval(CP.getDstReg()); + dbgs() << "\tJoined. Result = "; + DstInt.print(dbgs(), tri_); + dbgs() << "\n"; + }); + + ++numJoins; + return true; +} + +/// ComputeUltimateVN - Assuming we are going to join two live intervals, +/// compute what the resultant value numbers for each value in the input two +/// ranges will be. This is complicated by copies between the two which can +/// and will commonly cause multiple value numbers to be merged into one. +/// +/// VN is the value number that we're trying to resolve. InstDefiningValue +/// keeps track of the new InstDefiningValue assignment for the result +/// LiveInterval. ThisFromOther/OtherFromThis are sets that keep track of +/// whether a value in this or other is a copy from the opposite set. +/// ThisValNoAssignments/OtherValNoAssignments keep track of value #'s that have +/// already been assigned. +/// +/// ThisFromOther[x] - If x is defined as a copy from the other interval, this +/// contains the value number the copy is from. +/// +static unsigned ComputeUltimateVN(VNInfo *VNI, + SmallVector<VNInfo*, 16> &NewVNInfo, + DenseMap<VNInfo*, VNInfo*> &ThisFromOther, + DenseMap<VNInfo*, VNInfo*> &OtherFromThis, + SmallVector<int, 16> &ThisValNoAssignments, + SmallVector<int, 16> &OtherValNoAssignments) { + unsigned VN = VNI->id; + + // If the VN has already been computed, just return it. + if (ThisValNoAssignments[VN] >= 0) + return ThisValNoAssignments[VN]; + assert(ThisValNoAssignments[VN] != -2 && "Cyclic value numbers"); + + // If this val is not a copy from the other val, then it must be a new value + // number in the destination. + DenseMap<VNInfo*, VNInfo*>::iterator I = ThisFromOther.find(VNI); + if (I == ThisFromOther.end()) { + NewVNInfo.push_back(VNI); + return ThisValNoAssignments[VN] = NewVNInfo.size()-1; + } + VNInfo *OtherValNo = I->second; + + // Otherwise, this *is* a copy from the RHS. If the other side has already + // been computed, return it. + if (OtherValNoAssignments[OtherValNo->id] >= 0) + return ThisValNoAssignments[VN] = OtherValNoAssignments[OtherValNo->id]; + + // Mark this value number as currently being computed, then ask what the + // ultimate value # of the other value is. + ThisValNoAssignments[VN] = -2; + unsigned UltimateVN = + ComputeUltimateVN(OtherValNo, NewVNInfo, OtherFromThis, ThisFromOther, + OtherValNoAssignments, ThisValNoAssignments); + return ThisValNoAssignments[VN] = UltimateVN; +} + + +// Find out if we have something like +// A = X +// B = X +// if so, we can pretend this is actually +// A = X +// B = A +// which allows us to coalesce A and B. +// VNI is the definition of B. LR is the life range of A that includes +// the slot just before B. If we return true, we add "B = X" to DupCopies. +static bool RegistersDefinedFromSameValue(LiveIntervals &li, + const TargetRegisterInfo &tri, + CoalescerPair &CP, + VNInfo *VNI, + LiveRange *LR, + SmallVector<MachineInstr*, 8> &DupCopies) { + return false; // To see if this fixes the i386 dragonegg buildbot miscompile. + // FIXME: This is very conservative. For example, we don't handle + // physical registers. + + MachineInstr *MI = VNI->getCopy(); + + if (!MI->isFullCopy() || CP.isPartial() || CP.isPhys()) + return false; + + unsigned Dst = MI->getOperand(0).getReg(); + unsigned Src = MI->getOperand(1).getReg(); + + // FIXME: If "B = X" kills X, we have to move the kill back to its + // previous use. For now we just avoid the optimization in that case. + LiveInterval &SrcInt = li.getInterval(Src); + if (SrcInt.killedAt(VNI->def)) + return false; + + if (!TargetRegisterInfo::isVirtualRegister(Src) || + !TargetRegisterInfo::isVirtualRegister(Dst)) + return false; + + unsigned A = CP.getDstReg(); + unsigned B = CP.getSrcReg(); + + if (B == Dst) + std::swap(A, B); + assert(Dst == A); + + VNInfo *Other = LR->valno; + if (!Other->isDefByCopy()) + return false; + const MachineInstr *OtherMI = Other->getCopy(); + + if (!OtherMI->isFullCopy()) + return false; + + unsigned OtherDst = OtherMI->getOperand(0).getReg(); + unsigned OtherSrc = OtherMI->getOperand(1).getReg(); + + if (!TargetRegisterInfo::isVirtualRegister(OtherSrc) || + !TargetRegisterInfo::isVirtualRegister(OtherDst)) + return false; + + assert(OtherDst == B); + + if (Src != OtherSrc) + return false; + + DupCopies.push_back(MI); + + return true; +} + +/// JoinIntervals - Attempt to join these two intervals. On failure, this +/// returns false. +bool RegisterCoalescer::JoinIntervals(CoalescerPair &CP) { + LiveInterval &RHS = li_->getInterval(CP.getSrcReg()); + DEBUG({ dbgs() << "\t\tRHS = "; RHS.print(dbgs(), tri_); dbgs() << "\n"; }); + + // If a live interval is a physical register, check for interference with any + // aliases. The interference check implemented here is a bit more conservative + // than the full interfeence check below. We allow overlapping live ranges + // only when one is a copy of the other. + if (CP.isPhys()) { + for (const unsigned *AS = tri_->getAliasSet(CP.getDstReg()); *AS; ++AS){ + if (!li_->hasInterval(*AS)) + continue; + const LiveInterval &LHS = li_->getInterval(*AS); + LiveInterval::const_iterator LI = LHS.begin(); + for (LiveInterval::const_iterator RI = RHS.begin(), RE = RHS.end(); + RI != RE; ++RI) { + LI = std::lower_bound(LI, LHS.end(), RI->start); + // Does LHS have an overlapping live range starting before RI? + if ((LI != LHS.begin() && LI[-1].end > RI->start) && + (RI->start != RI->valno->def || + !CP.isCoalescable(li_->getInstructionFromIndex(RI->start)))) { + DEBUG({ + dbgs() << "\t\tInterference from alias: "; + LHS.print(dbgs(), tri_); + dbgs() << "\n\t\tOverlap at " << RI->start << " and no copy.\n"; + }); + return false; + } + + // Check that LHS ranges beginning in this range are copies. + for (; LI != LHS.end() && LI->start < RI->end; ++LI) { + if (LI->start != LI->valno->def || + !CP.isCoalescable(li_->getInstructionFromIndex(LI->start))) { + DEBUG({ + dbgs() << "\t\tInterference from alias: "; + LHS.print(dbgs(), tri_); + dbgs() << "\n\t\tDef at " << LI->start << " is not a copy.\n"; + }); + return false; + } + } + } + } + } + + // Compute the final value assignment, assuming that the live ranges can be + // coalesced. + SmallVector<int, 16> LHSValNoAssignments; + SmallVector<int, 16> RHSValNoAssignments; + DenseMap<VNInfo*, VNInfo*> LHSValsDefinedFromRHS; + DenseMap<VNInfo*, VNInfo*> RHSValsDefinedFromLHS; + SmallVector<VNInfo*, 16> NewVNInfo; + + SmallVector<MachineInstr*, 8> DupCopies; + + LiveInterval &LHS = li_->getOrCreateInterval(CP.getDstReg()); + DEBUG({ dbgs() << "\t\tLHS = "; LHS.print(dbgs(), tri_); dbgs() << "\n"; }); + + // Loop over the value numbers of the LHS, seeing if any are defined from + // the RHS. + for (LiveInterval::vni_iterator i = LHS.vni_begin(), e = LHS.vni_end(); + i != e; ++i) { + VNInfo *VNI = *i; + if (VNI->isUnused() || !VNI->isDefByCopy()) // Src not defined by a copy? + continue; + + // Never join with a register that has EarlyClobber redefs. + if (VNI->hasRedefByEC()) + return false; + + // Figure out the value # from the RHS. + LiveRange *lr = RHS.getLiveRangeContaining(VNI->def.getPrevSlot()); + // The copy could be to an aliased physreg. + if (!lr) continue; + + // DstReg is known to be a register in the LHS interval. If the src is + // from the RHS interval, we can use its value #. + MachineInstr *MI = VNI->getCopy(); + if (!CP.isCoalescable(MI) && + !RegistersDefinedFromSameValue(*li_, *tri_, CP, VNI, lr, DupCopies)) + continue; + + LHSValsDefinedFromRHS[VNI] = lr->valno; + } + + // Loop over the value numbers of the RHS, seeing if any are defined from + // the LHS. + for (LiveInterval::vni_iterator i = RHS.vni_begin(), e = RHS.vni_end(); + i != e; ++i) { + VNInfo *VNI = *i; + if (VNI->isUnused() || !VNI->isDefByCopy()) // Src not defined by a copy? + continue; + + // Never join with a register that has EarlyClobber redefs. + if (VNI->hasRedefByEC()) + return false; + + // Figure out the value # from the LHS. + LiveRange *lr = LHS.getLiveRangeContaining(VNI->def.getPrevSlot()); + // The copy could be to an aliased physreg. + if (!lr) continue; + + // DstReg is known to be a register in the RHS interval. If the src is + // from the LHS interval, we can use its value #. + MachineInstr *MI = VNI->getCopy(); + if (!CP.isCoalescable(MI) && + !RegistersDefinedFromSameValue(*li_, *tri_, CP, VNI, lr, DupCopies)) + continue; + + RHSValsDefinedFromLHS[VNI] = lr->valno; + } + + LHSValNoAssignments.resize(LHS.getNumValNums(), -1); + RHSValNoAssignments.resize(RHS.getNumValNums(), -1); + NewVNInfo.reserve(LHS.getNumValNums() + RHS.getNumValNums()); + + for (LiveInterval::vni_iterator i = LHS.vni_begin(), e = LHS.vni_end(); + i != e; ++i) { + VNInfo *VNI = *i; + unsigned VN = VNI->id; + if (LHSValNoAssignments[VN] >= 0 || VNI->isUnused()) + continue; + ComputeUltimateVN(VNI, NewVNInfo, + LHSValsDefinedFromRHS, RHSValsDefinedFromLHS, + LHSValNoAssignments, RHSValNoAssignments); + } + for (LiveInterval::vni_iterator i = RHS.vni_begin(), e = RHS.vni_end(); + i != e; ++i) { + VNInfo *VNI = *i; + unsigned VN = VNI->id; + if (RHSValNoAssignments[VN] >= 0 || VNI->isUnused()) + continue; + // If this value number isn't a copy from the LHS, it's a new number. + if (RHSValsDefinedFromLHS.find(VNI) == RHSValsDefinedFromLHS.end()) { + NewVNInfo.push_back(VNI); + RHSValNoAssignments[VN] = NewVNInfo.size()-1; + continue; + } + + ComputeUltimateVN(VNI, NewVNInfo, + RHSValsDefinedFromLHS, LHSValsDefinedFromRHS, + RHSValNoAssignments, LHSValNoAssignments); + } + + // Armed with the mappings of LHS/RHS values to ultimate values, walk the + // interval lists to see if these intervals are coalescable. + LiveInterval::const_iterator I = LHS.begin(); + LiveInterval::const_iterator IE = LHS.end(); + LiveInterval::const_iterator J = RHS.begin(); + LiveInterval::const_iterator JE = RHS.end(); + + // Skip ahead until the first place of potential sharing. + if (I != IE && J != JE) { + if (I->start < J->start) { + I = std::upper_bound(I, IE, J->start); + if (I != LHS.begin()) --I; + } else if (J->start < I->start) { + J = std::upper_bound(J, JE, I->start); + if (J != RHS.begin()) --J; + } + } + + while (I != IE && J != JE) { + // Determine if these two live ranges overlap. + bool Overlaps; + if (I->start < J->start) { + Overlaps = I->end > J->start; + } else { + Overlaps = J->end > I->start; + } + + // If so, check value # info to determine if they are really different. + if (Overlaps) { + // If the live range overlap will map to the same value number in the + // result liverange, we can still coalesce them. If not, we can't. + if (LHSValNoAssignments[I->valno->id] != + RHSValNoAssignments[J->valno->id]) + return false; + // If it's re-defined by an early clobber somewhere in the live range, + // then conservatively abort coalescing. + if (NewVNInfo[LHSValNoAssignments[I->valno->id]]->hasRedefByEC()) + return false; + } + + if (I->end < J->end) + ++I; + else + ++J; + } + + // Update kill info. Some live ranges are extended due to copy coalescing. + for (DenseMap<VNInfo*, VNInfo*>::iterator I = LHSValsDefinedFromRHS.begin(), + E = LHSValsDefinedFromRHS.end(); I != E; ++I) { + VNInfo *VNI = I->first; + unsigned LHSValID = LHSValNoAssignments[VNI->id]; + if (VNI->hasPHIKill()) + NewVNInfo[LHSValID]->setHasPHIKill(true); + } + + // Update kill info. Some live ranges are extended due to copy coalescing. + for (DenseMap<VNInfo*, VNInfo*>::iterator I = RHSValsDefinedFromLHS.begin(), + E = RHSValsDefinedFromLHS.end(); I != E; ++I) { + VNInfo *VNI = I->first; + unsigned RHSValID = RHSValNoAssignments[VNI->id]; + if (VNI->hasPHIKill()) + NewVNInfo[RHSValID]->setHasPHIKill(true); + } + + if (LHSValNoAssignments.empty()) + LHSValNoAssignments.push_back(-1); + if (RHSValNoAssignments.empty()) + RHSValNoAssignments.push_back(-1); + + for (SmallVector<MachineInstr*, 8>::iterator I = DupCopies.begin(), + E = DupCopies.end(); I != E; ++I) { + MachineInstr *MI = *I; + + // We have pretended that the assignment to B in + // A = X + // B = X + // was actually a copy from A. Now that we decided to coalesce A and B, + // transform the code into + // A = X + // X = X + // and mark the X as coalesced to keep the illusion. + unsigned Src = MI->getOperand(1).getReg(); + MI->getOperand(0).substVirtReg(Src, 0, *tri_); + + markAsJoined(MI); + } + + // If we get here, we know that we can coalesce the live ranges. Ask the + // intervals to coalesce themselves now. + LHS.join(RHS, &LHSValNoAssignments[0], &RHSValNoAssignments[0], NewVNInfo, + mri_); + return true; +} + +namespace { + // DepthMBBCompare - Comparison predicate that sort first based on the loop + // depth of the basic block (the unsigned), and then on the MBB number. + struct DepthMBBCompare { + typedef std::pair<unsigned, MachineBasicBlock*> DepthMBBPair; + bool operator()(const DepthMBBPair &LHS, const DepthMBBPair &RHS) const { + // Deeper loops first + if (LHS.first != RHS.first) + return LHS.first > RHS.first; + + // Prefer blocks that are more connected in the CFG. This takes care of + // the most difficult copies first while intervals are short. + unsigned cl = LHS.second->pred_size() + LHS.second->succ_size(); + unsigned cr = RHS.second->pred_size() + RHS.second->succ_size(); + if (cl != cr) + return cl > cr; + + // As a last resort, sort by block number. + return LHS.second->getNumber() < RHS.second->getNumber(); + } + }; +} + +void RegisterCoalescer::CopyCoalesceInMBB(MachineBasicBlock *MBB, + std::vector<MachineInstr*> &TryAgain) { + DEBUG(dbgs() << MBB->getName() << ":\n"); + + SmallVector<MachineInstr*, 8> VirtCopies; + SmallVector<MachineInstr*, 8> PhysCopies; + SmallVector<MachineInstr*, 8> ImpDefCopies; + for (MachineBasicBlock::iterator MII = MBB->begin(), E = MBB->end(); + MII != E;) { + MachineInstr *Inst = MII++; + + // If this isn't a copy nor a extract_subreg, we can't join intervals. + unsigned SrcReg, DstReg; + if (Inst->isCopy()) { + DstReg = Inst->getOperand(0).getReg(); + SrcReg = Inst->getOperand(1).getReg(); + } else if (Inst->isSubregToReg()) { + DstReg = Inst->getOperand(0).getReg(); + SrcReg = Inst->getOperand(2).getReg(); + } else + continue; + + bool SrcIsPhys = TargetRegisterInfo::isPhysicalRegister(SrcReg); + bool DstIsPhys = TargetRegisterInfo::isPhysicalRegister(DstReg); + if (li_->hasInterval(SrcReg) && li_->getInterval(SrcReg).empty()) + ImpDefCopies.push_back(Inst); + else if (SrcIsPhys || DstIsPhys) + PhysCopies.push_back(Inst); + else + VirtCopies.push_back(Inst); + } + + // Try coalescing implicit copies and insert_subreg <undef> first, + // followed by copies to / from physical registers, then finally copies + // from virtual registers to virtual registers. + for (unsigned i = 0, e = ImpDefCopies.size(); i != e; ++i) { + MachineInstr *TheCopy = ImpDefCopies[i]; + bool Again = false; + if (!JoinCopy(TheCopy, Again)) + if (Again) + TryAgain.push_back(TheCopy); + } + for (unsigned i = 0, e = PhysCopies.size(); i != e; ++i) { + MachineInstr *TheCopy = PhysCopies[i]; + bool Again = false; + if (!JoinCopy(TheCopy, Again)) + if (Again) + TryAgain.push_back(TheCopy); + } + for (unsigned i = 0, e = VirtCopies.size(); i != e; ++i) { + MachineInstr *TheCopy = VirtCopies[i]; + bool Again = false; + if (!JoinCopy(TheCopy, Again)) + if (Again) + TryAgain.push_back(TheCopy); + } +} + +void RegisterCoalescer::joinIntervals() { + DEBUG(dbgs() << "********** JOINING INTERVALS ***********\n"); + + std::vector<MachineInstr*> TryAgainList; + if (loopInfo->empty()) { + // If there are no loops in the function, join intervals in function order. + for (MachineFunction::iterator I = mf_->begin(), E = mf_->end(); + I != E; ++I) + CopyCoalesceInMBB(I, TryAgainList); + } else { + // Otherwise, join intervals in inner loops before other intervals. + // Unfortunately we can't just iterate over loop hierarchy here because + // there may be more MBB's than BB's. Collect MBB's for sorting. + + // Join intervals in the function prolog first. We want to join physical + // registers with virtual registers before the intervals got too long. + std::vector<std::pair<unsigned, MachineBasicBlock*> > MBBs; + for (MachineFunction::iterator I = mf_->begin(), E = mf_->end();I != E;++I){ + MachineBasicBlock *MBB = I; + MBBs.push_back(std::make_pair(loopInfo->getLoopDepth(MBB), I)); + } + + // Sort by loop depth. + std::sort(MBBs.begin(), MBBs.end(), DepthMBBCompare()); + + // Finally, join intervals in loop nest order. + for (unsigned i = 0, e = MBBs.size(); i != e; ++i) + CopyCoalesceInMBB(MBBs[i].second, TryAgainList); + } + + // Joining intervals can allow other intervals to be joined. Iteratively join + // until we make no progress. + bool ProgressMade = true; + while (ProgressMade) { + ProgressMade = false; + + for (unsigned i = 0, e = TryAgainList.size(); i != e; ++i) { + MachineInstr *&TheCopy = TryAgainList[i]; + if (!TheCopy) + continue; + + bool Again = false; + bool Success = JoinCopy(TheCopy, Again); + if (Success || !Again) { + TheCopy= 0; // Mark this one as done. + ProgressMade = true; + } + } + } +} + +void RegisterCoalescer::releaseMemory() { + JoinedCopies.clear(); + ReMatCopies.clear(); + ReMatDefs.clear(); +} + +bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) { + mf_ = &fn; + mri_ = &fn.getRegInfo(); + tm_ = &fn.getTarget(); + tri_ = tm_->getRegisterInfo(); + tii_ = tm_->getInstrInfo(); + li_ = &getAnalysis<LiveIntervals>(); + ldv_ = &getAnalysis<LiveDebugVariables>(); + AA = &getAnalysis<AliasAnalysis>(); + loopInfo = &getAnalysis<MachineLoopInfo>(); + + DEBUG(dbgs() << "********** SIMPLE REGISTER COALESCING **********\n" + << "********** Function: " + << ((Value*)mf_->getFunction())->getName() << '\n'); + + if (VerifyCoalescing) + mf_->verify(this, "Before register coalescing"); + + RegClassInfo.runOnMachineFunction(fn); + + // Join (coalesce) intervals if requested. + if (EnableJoining) { + joinIntervals(); + DEBUG({ + dbgs() << "********** INTERVALS POST JOINING **********\n"; + for (LiveIntervals::iterator I = li_->begin(), E = li_->end(); + I != E; ++I){ + I->second->print(dbgs(), tri_); + dbgs() << "\n"; + } + }); + } + + // Perform a final pass over the instructions and compute spill weights + // and remove identity moves. + SmallVector<unsigned, 4> DeadDefs; + for (MachineFunction::iterator mbbi = mf_->begin(), mbbe = mf_->end(); + mbbi != mbbe; ++mbbi) { + MachineBasicBlock* mbb = mbbi; + for (MachineBasicBlock::iterator mii = mbb->begin(), mie = mbb->end(); + mii != mie; ) { + MachineInstr *MI = mii; + if (JoinedCopies.count(MI)) { + // Delete all coalesced copies. + bool DoDelete = true; + assert(MI->isCopyLike() && "Unrecognized copy instruction"); + unsigned SrcReg = MI->getOperand(MI->isSubregToReg() ? 2 : 1).getReg(); + if (TargetRegisterInfo::isPhysicalRegister(SrcReg) && + MI->getNumOperands() > 2) + // Do not delete extract_subreg, insert_subreg of physical + // registers unless the definition is dead. e.g. + // %DO<def> = INSERT_SUBREG %D0<undef>, %S0<kill>, 1 + // or else the scavenger may complain. LowerSubregs will + // delete them later. + DoDelete = false; + + if (MI->allDefsAreDead()) { + if (TargetRegisterInfo::isVirtualRegister(SrcReg) && + li_->hasInterval(SrcReg)) + li_->shrinkToUses(&li_->getInterval(SrcReg)); + DoDelete = true; + } + if (!DoDelete) { + // We need the instruction to adjust liveness, so make it a KILL. + if (MI->isSubregToReg()) { + MI->RemoveOperand(3); + MI->RemoveOperand(1); + } + MI->setDesc(tii_->get(TargetOpcode::KILL)); + mii = llvm::next(mii); + } else { + li_->RemoveMachineInstrFromMaps(MI); + mii = mbbi->erase(mii); + ++numPeep; + } + continue; + } + + // Now check if this is a remat'ed def instruction which is now dead. + if (ReMatDefs.count(MI)) { + bool isDead = true; + for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { + const MachineOperand &MO = MI->getOperand(i); + if (!MO.isReg()) + continue; + unsigned Reg = MO.getReg(); + if (!Reg) + continue; + if (TargetRegisterInfo::isVirtualRegister(Reg)) + DeadDefs.push_back(Reg); + if (MO.isDead()) + continue; + if (TargetRegisterInfo::isPhysicalRegister(Reg) || + !mri_->use_nodbg_empty(Reg)) { + isDead = false; + break; + } + } + if (isDead) { + while (!DeadDefs.empty()) { + unsigned DeadDef = DeadDefs.back(); + DeadDefs.pop_back(); + RemoveDeadDef(li_->getInterval(DeadDef), MI); + } + li_->RemoveMachineInstrFromMaps(mii); + mii = mbbi->erase(mii); + continue; + } else + DeadDefs.clear(); + } + + ++mii; + + // Check for now unnecessary kill flags. + if (li_->isNotInMIMap(MI)) continue; + SlotIndex DefIdx = li_->getInstructionIndex(MI).getDefIndex(); + for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { + MachineOperand &MO = MI->getOperand(i); + if (!MO.isReg() || !MO.isKill()) continue; + unsigned reg = MO.getReg(); + if (!reg || !li_->hasInterval(reg)) continue; + if (!li_->getInterval(reg).killedAt(DefIdx)) { + MO.setIsKill(false); + continue; + } + // When leaving a kill flag on a physreg, check if any subregs should + // remain alive. + if (!TargetRegisterInfo::isPhysicalRegister(reg)) + continue; + for (const unsigned *SR = tri_->getSubRegisters(reg); + unsigned S = *SR; ++SR) + if (li_->hasInterval(S) && li_->getInterval(S).liveAt(DefIdx)) + MI->addRegisterDefined(S, tri_); + } + } + } + + DEBUG(dump()); + DEBUG(ldv_->dump()); + if (VerifyCoalescing) + mf_->verify(this, "After register coalescing"); + return true; +} + +/// print - Implement the dump method. +void RegisterCoalescer::print(raw_ostream &O, const Module* m) const { + li_->print(O, m); +} + +RegisterCoalescer *llvm::createRegisterCoalescer() { + return new RegisterCoalescer(); +} diff --git a/lib/CodeGen/SimpleRegisterCoalescing.h b/lib/CodeGen/RegisterCoalescer.h index 92f6c64..4131d91 100644 --- a/lib/CodeGen/SimpleRegisterCoalescing.h +++ b/lib/CodeGen/RegisterCoalescer.h @@ -1,4 +1,4 @@ -//===-- SimpleRegisterCoalescing.h - Register Coalescing --------*- C++ -*-===// +//===-- RegisterCoalescer.h - Register Coalescing Interface ------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -7,37 +7,38 @@ // //===----------------------------------------------------------------------===// // -// This file implements a simple register copy coalescing phase. +// This file contains the abstract interface for register coalescers, +// allowing them to interact with and query register allocators. // //===----------------------------------------------------------------------===// -#ifndef LLVM_CODEGEN_SIMPLE_REGISTER_COALESCING_H -#define LLVM_CODEGEN_SIMPLE_REGISTER_COALESCING_H - -#include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/LiveIntervalAnalysis.h" -#include "llvm/CodeGen/RegisterCoalescer.h" #include "RegisterClassInfo.h" +#include "llvm/Support/IncludeFile.h" +#include "llvm/CodeGen/LiveInterval.h" +#include "llvm/ADT/SmallPtrSet.h" + +#ifndef LLVM_CODEGEN_REGISTER_COALESCER_H +#define LLVM_CODEGEN_REGISTER_COALESCER_H namespace llvm { - class SimpleRegisterCoalescing; - class LiveDebugVariables; + + class MachineFunction; + class RegallocQuery; + class AnalysisUsage; + class MachineInstr; class TargetRegisterInfo; + class TargetRegisterClass; class TargetInstrInfo; + class LiveDebugVariables; class VirtRegMap; class MachineLoopInfo; - /// CopyRec - Representation for copy instructions in coalescer queue. - /// - struct CopyRec { - MachineInstr *MI; - unsigned LoopDepth; - CopyRec(MachineInstr *mi, unsigned depth) - : MI(mi), LoopDepth(depth) {} - }; + class CoalescerPair; - class SimpleRegisterCoalescing : public MachineFunctionPass, - public RegisterCoalescer { + /// An abstract interface for register coalescers. Coalescers must + /// implement this interface to be part of the coalescer analysis + /// group. + class RegisterCoalescer : public MachineFunctionPass { MachineFunction* mf_; MachineRegisterInfo* mri_; const TargetMachine* tm_; @@ -61,41 +62,20 @@ namespace llvm { /// been remat'ed. SmallPtrSet<MachineInstr*, 8> ReMatDefs; - public: - static char ID; // Pass identifcation, replacement for typeid - SimpleRegisterCoalescing() : MachineFunctionPass(ID) { - initializeSimpleRegisterCoalescingPass(*PassRegistry::getPassRegistry()); - } - - virtual void getAnalysisUsage(AnalysisUsage &AU) const; - virtual void releaseMemory(); - - /// runOnMachineFunction - pass entry point - virtual bool runOnMachineFunction(MachineFunction&); - - bool coalesceFunction(MachineFunction &mf, RegallocQuery &) { - // This runs as an independent pass, so don't do anything. - return false; - } - - /// print - Implement the dump method. - virtual void print(raw_ostream &O, const Module* = 0) const; - - private: /// joinIntervals - join compatible live intervals void joinIntervals(); /// CopyCoalesceInMBB - Coalesce copies in the specified MBB, putting /// copies that cannot yet be coalesced into the "TryAgain" list. void CopyCoalesceInMBB(MachineBasicBlock *MBB, - std::vector<CopyRec> &TryAgain); + std::vector<MachineInstr*> &TryAgain); /// JoinCopy - Attempt to join intervals corresponding to SrcReg/DstReg, /// which are the src/dst of the copy instruction CopyMI. This returns true /// if the copy was successfully coalesced away. If it is not currently /// possible to coalesce this interval, but it may be possible if other /// things get coalesced, then it returns true by reference in 'Again'. - bool JoinCopy(CopyRec &TheCopy, bool &Again); + bool JoinCopy(MachineInstr *TheCopy, bool &Again); /// JoinIntervals - Attempt to join these two intervals. On failure, this /// returns false. The output "SrcInt" will not have been modified, so we can @@ -155,8 +135,109 @@ namespace llvm { /// markAsJoined - Remember that CopyMI has already been joined. void markAsJoined(MachineInstr *CopyMI); + + public: + static char ID; // Class identification, replacement for typeinfo + RegisterCoalescer() : MachineFunctionPass(ID) { + initializeRegisterCoalescerPass(*PassRegistry::getPassRegistry()); + } + + /// Register allocators must call this from their own + /// getAnalysisUsage to cover the case where the coalescer is not + /// a Pass in the proper sense and isn't managed by PassManager. + /// PassManager needs to know which analyses to make available and + /// which to invalidate when running the register allocator or any + /// pass that might call coalescing. The long-term solution is to + /// allow hierarchies of PassManagers. + virtual void getAnalysisUsage(AnalysisUsage &AU) const; + + virtual void releaseMemory(); + + /// runOnMachineFunction - pass entry point + virtual bool runOnMachineFunction(MachineFunction&); + + /// print - Implement the dump method. + virtual void print(raw_ostream &O, const Module* = 0) const; }; + /// CoalescerPair - A helper class for register coalescers. When deciding if + /// two registers can be coalesced, CoalescerPair can determine if a copy + /// instruction would become an identity copy after coalescing. + class CoalescerPair { + const TargetInstrInfo &tii_; + const TargetRegisterInfo &tri_; + + /// dstReg_ - The register that will be left after coalescing. It can be a + /// virtual or physical register. + unsigned dstReg_; + + /// srcReg_ - the virtual register that will be coalesced into dstReg. + unsigned srcReg_; + + /// subReg_ - The subregister index of srcReg in dstReg_. It is possible the + /// coalesce srcReg_ into a subreg of the larger dstReg_ when dstReg_ is a + /// virtual register. + unsigned subIdx_; + + /// partial_ - True when the original copy was a partial subregister copy. + bool partial_; + + /// crossClass_ - True when both regs are virtual, and newRC is constrained. + bool crossClass_; + + /// flipped_ - True when DstReg and SrcReg are reversed from the oriignal copy + /// instruction. + bool flipped_; + + /// newRC_ - The register class of the coalesced register, or NULL if dstReg_ + /// is a physreg. + const TargetRegisterClass *newRC_; + + public: + CoalescerPair(const TargetInstrInfo &tii, const TargetRegisterInfo &tri) + : tii_(tii), tri_(tri), dstReg_(0), srcReg_(0), subIdx_(0), + partial_(false), crossClass_(false), flipped_(false), newRC_(0) {} + + /// setRegisters - set registers to match the copy instruction MI. Return + /// false if MI is not a coalescable copy instruction. + bool setRegisters(const MachineInstr*); + + /// flip - Swap srcReg_ and dstReg_. Return false if swapping is impossible + /// because dstReg_ is a physical register, or subIdx_ is set. + bool flip(); + + /// isCoalescable - Return true if MI is a copy instruction that will become + /// an identity copy after coalescing. + bool isCoalescable(const MachineInstr*) const; + + /// isPhys - Return true if DstReg is a physical register. + bool isPhys() const { return !newRC_; } + + /// isPartial - Return true if the original copy instruction did not copy the + /// full register, but was a subreg operation. + bool isPartial() const { return partial_; } + + /// isCrossClass - Return true if DstReg is virtual and NewRC is a smaller register class than DstReg's. + bool isCrossClass() const { return crossClass_; } + + /// isFlipped - Return true when getSrcReg is the register being defined by + /// the original copy instruction. + bool isFlipped() const { return flipped_; } + + /// getDstReg - Return the register (virtual or physical) that will remain + /// after coalescing. + unsigned getDstReg() const { return dstReg_; } + + /// getSrcReg - Return the virtual register that will be coalesced away. + unsigned getSrcReg() const { return srcReg_; } + + /// getSubIdx - Return the subregister index in DstReg that SrcReg will be + /// coalesced into, or 0. + unsigned getSubIdx() const { return subIdx_; } + + /// getNewRC - Return the register class of the coalesced register. + const TargetRegisterClass *getNewRC() const { return newRC_; } + }; } // End llvm namespace #endif diff --git a/lib/CodeGen/ScheduleDAG.cpp b/lib/CodeGen/ScheduleDAG.cpp index f328493..21375b2 100644 --- a/lib/CodeGen/ScheduleDAG.cpp +++ b/lib/CodeGen/ScheduleDAG.cpp @@ -45,7 +45,7 @@ ScheduleDAG::ScheduleDAG(MachineFunction &mf) ScheduleDAG::~ScheduleDAG() {} /// getInstrDesc helper to handle SDNodes. -const TargetInstrDesc *ScheduleDAG::getNodeDesc(const SDNode *Node) const { +const MCInstrDesc *ScheduleDAG::getNodeDesc(const SDNode *Node) const { if (!Node || !Node->isMachineOpcode()) return NULL; return &TII->get(Node->getMachineOpcode()); } diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp index 2363df4..9cceb4e 100644 --- a/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -21,6 +21,7 @@ #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/MC/MCInstrItineraries.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetRegisterInfo.h" @@ -236,13 +237,13 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) { continue; } - const TargetInstrDesc &TID = MI->getDesc(); - assert(!TID.isTerminator() && !MI->isLabel() && + const MCInstrDesc &MCID = MI->getDesc(); + assert(!MCID.isTerminator() && !MI->isLabel() && "Cannot schedule terminators or labels!"); // Create the SUnit for this MI. SUnit *SU = NewSUnit(MI); - SU->isCall = TID.isCall(); - SU->isCommutable = TID.isCommutable(); + SU->isCall = MCID.isCall(); + SU->isCommutable = MCID.isCommutable(); // Assign the Latency field of SU using target-provided information. if (UnitLatencies) @@ -309,13 +310,13 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) { if (SpecialAddressLatency != 0 && !UnitLatencies && UseSU != &ExitSU) { MachineInstr *UseMI = UseSU->getInstr(); - const TargetInstrDesc &UseTID = UseMI->getDesc(); + const MCInstrDesc &UseMCID = UseMI->getDesc(); int RegUseIndex = UseMI->findRegisterUseOperandIdx(Reg); assert(RegUseIndex >= 0 && "UseMI doesn's use register!"); if (RegUseIndex >= 0 && - (UseTID.mayLoad() || UseTID.mayStore()) && - (unsigned)RegUseIndex < UseTID.getNumOperands() && - UseTID.OpInfo[RegUseIndex].isLookupPtrRegClass()) + (UseMCID.mayLoad() || UseMCID.mayStore()) && + (unsigned)RegUseIndex < UseMCID.getNumOperands() && + UseMCID.OpInfo[RegUseIndex].isLookupPtrRegClass()) LDataLatency += SpecialAddressLatency; } // Adjust the dependence latency using operand def/use @@ -352,17 +353,17 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) { unsigned Count = I->second.second; const MachineInstr *UseMI = UseMO->getParent(); unsigned UseMOIdx = UseMO - &UseMI->getOperand(0); - const TargetInstrDesc &UseTID = UseMI->getDesc(); + const MCInstrDesc &UseMCID = UseMI->getDesc(); // TODO: If we knew the total depth of the region here, we could // handle the case where the whole loop is inside the region but // is large enough that the isScheduleHigh trick isn't needed. - if (UseMOIdx < UseTID.getNumOperands()) { + if (UseMOIdx < UseMCID.getNumOperands()) { // Currently, we only support scheduling regions consisting of // single basic blocks. Check to see if the instruction is in // the same region by checking to see if it has the same parent. if (UseMI->getParent() != MI->getParent()) { unsigned Latency = SU->Latency; - if (UseTID.OpInfo[UseMOIdx].isLookupPtrRegClass()) + if (UseMCID.OpInfo[UseMOIdx].isLookupPtrRegClass()) Latency += SpecialAddressLatency; // This is a wild guess as to the portion of the latency which // will be overlapped by work done outside the current @@ -374,7 +375,7 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) { /*isMustAlias=*/false, /*isArtificial=*/true)); } else if (SpecialAddressLatency > 0 && - UseTID.OpInfo[UseMOIdx].isLookupPtrRegClass()) { + UseMCID.OpInfo[UseMOIdx].isLookupPtrRegClass()) { // The entire loop body is within the current scheduling region // and the latency of this operation is assumed to be greater // than the latency of the loop. @@ -417,9 +418,9 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) { // produce more precise dependence information. #define STORE_LOAD_LATENCY 1 unsigned TrueMemOrderLatency = 0; - if (TID.isCall() || MI->hasUnmodeledSideEffects() || + if (MCID.isCall() || MI->hasUnmodeledSideEffects() || (MI->hasVolatileMemoryRef() && - (!TID.mayLoad() || !MI->isInvariantLoad(AA)))) { + (!MCID.mayLoad() || !MI->isInvariantLoad(AA)))) { // Be conservative with these and add dependencies on all memory // references, even those that are known to not alias. for (std::map<const Value *, SUnit *>::iterator I = @@ -458,7 +459,7 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) { PendingLoads.clear(); AliasMemDefs.clear(); AliasMemUses.clear(); - } else if (TID.mayStore()) { + } else if (MCID.mayStore()) { bool MayAlias = true; TrueMemOrderLatency = STORE_LOAD_LATENCY; if (const Value *V = getUnderlyingObjectForInstr(MI, MFI, MayAlias)) { @@ -514,7 +515,7 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) { /*Reg=*/0, /*isNormalMemory=*/false, /*isMustAlias=*/false, /*isArtificial=*/true)); - } else if (TID.mayLoad()) { + } else if (MCID.mayLoad()) { bool MayAlias = true; TrueMemOrderLatency = 0; if (MI->isInvariantLoad(AA)) { diff --git a/lib/CodeGen/ScoreboardHazardRecognizer.cpp b/lib/CodeGen/ScoreboardHazardRecognizer.cpp index e6d7ded..0e005d3 100644 --- a/lib/CodeGen/ScoreboardHazardRecognizer.cpp +++ b/lib/CodeGen/ScoreboardHazardRecognizer.cpp @@ -16,11 +16,11 @@ #define DEBUG_TYPE ::llvm::ScoreboardHazardRecognizer::DebugType #include "llvm/CodeGen/ScoreboardHazardRecognizer.h" #include "llvm/CodeGen/ScheduleDAG.h" +#include "llvm/MC/MCInstrItineraries.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetInstrItineraries.h" using namespace llvm; @@ -115,12 +115,12 @@ ScoreboardHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { // Use the itinerary for the underlying instruction to check for // free FU's in the scoreboard at the appropriate future cycles. - const TargetInstrDesc *TID = DAG->getInstrDesc(SU); - if (TID == NULL) { + const MCInstrDesc *MCID = DAG->getInstrDesc(SU); + if (MCID == NULL) { // Don't check hazards for non-machineinstr Nodes. return NoHazard; } - unsigned idx = TID->getSchedClass(); + unsigned idx = MCID->getSchedClass(); for (const InstrStage *IS = ItinData->beginStage(idx), *E = ItinData->endStage(idx); IS != E; ++IS) { // We must find one of the stage's units free for every cycle the @@ -173,16 +173,16 @@ void ScoreboardHazardRecognizer::EmitInstruction(SUnit *SU) { // Use the itinerary for the underlying instruction to reserve FU's // in the scoreboard at the appropriate future cycles. - const TargetInstrDesc *TID = DAG->getInstrDesc(SU); - assert(TID && "The scheduler must filter non-machineinstrs"); - if (DAG->TII->isZeroCost(TID->Opcode)) + const MCInstrDesc *MCID = DAG->getInstrDesc(SU); + assert(MCID && "The scheduler must filter non-machineinstrs"); + if (DAG->TII->isZeroCost(MCID->Opcode)) return; ++IssueCount; unsigned cycle = 0; - unsigned idx = TID->getSchedClass(); + unsigned idx = MCID->getSchedClass(); for (const InstrStage *IS = ItinData->beginStage(idx), *E = ItinData->endStage(idx); IS != E; ++IS) { // We must reserve one of the stage's units for every cycle the diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index e3d3906..90e0cc7 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -1310,16 +1310,6 @@ SDValue combineShlAddConstant(DebugLoc DL, SDValue N0, SDValue N1, return SDValue(); } -/// isCarryMaterialization - Returns true if V is an ADDE node that is known to -/// return 0 or 1 depending on the carry flag. -static bool isCarryMaterialization(SDValue V) { - if (V.getOpcode() != ISD::ADDE) - return false; - - ConstantSDNode *C = dyn_cast<ConstantSDNode>(V.getOperand(0)); - return C && C->isNullValue() && V.getOperand(0) == V.getOperand(1); -} - SDValue DAGCombiner::visitADD(SDNode *N) { SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); @@ -1483,18 +1473,6 @@ SDValue DAGCombiner::visitADD(SDNode *N) { return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt); } - // add (adde 0, 0, glue), X -> adde X, 0, glue - if (N0->hasOneUse() && isCarryMaterialization(N0)) - return DAG.getNode(ISD::ADDE, N->getDebugLoc(), - DAG.getVTList(VT, MVT::Glue), N1, N0.getOperand(0), - N0.getOperand(2)); - - // add X, (adde 0, 0, glue) -> adde X, 0, glue - if (N1->hasOneUse() && isCarryMaterialization(N1)) - return DAG.getNode(ISD::ADDE, N->getDebugLoc(), - DAG.getVTList(VT, MVT::Glue), N0, N1.getOperand(0), - N1.getOperand(2)); - return SDValue(); } @@ -1538,16 +1516,6 @@ SDValue DAGCombiner::visitADDC(SDNode *N) { N->getDebugLoc(), MVT::Glue)); } - // addc (adde 0, 0, glue), X -> adde X, 0, glue - if (N0->hasOneUse() && isCarryMaterialization(N0)) - return DAG.getNode(ISD::ADDE, N->getDebugLoc(), N->getVTList(), N1, - DAG.getConstant(0, VT), N0.getOperand(2)); - - // addc X, (adde 0, 0, glue) -> adde X, 0, glue - if (N1->hasOneUse() && isCarryMaterialization(N1)) - return DAG.getNode(ISD::ADDE, N->getDebugLoc(), N->getVTList(), N0, - DAG.getConstant(0, VT), N1.getOperand(2)); - return SDValue(); } diff --git a/lib/CodeGen/SelectionDAG/FastISel.cpp b/lib/CodeGen/SelectionDAG/FastISel.cpp index 797f174..ea7fead 100644 --- a/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -547,7 +547,7 @@ bool FastISel::SelectCall(const User *I) { case Intrinsic::dbg_value: { // This form of DBG_VALUE is target-independent. const DbgValueInst *DI = cast<DbgValueInst>(Call); - const TargetInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE); + const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE); const Value *V = DI->getValue(); if (!V) { // Currently the optimizer can produce this; insert an undef to @@ -556,9 +556,14 @@ bool FastISel::SelectCall(const User *I) { .addReg(0U).addImm(DI->getOffset()) .addMetadata(DI->getVariable()); } else if (const ConstantInt *CI = dyn_cast<ConstantInt>(V)) { - BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II) - .addImm(CI->getZExtValue()).addImm(DI->getOffset()) - .addMetadata(DI->getVariable()); + if (CI->getBitWidth() > 64) + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II) + .addCImm(CI).addImm(DI->getOffset()) + .addMetadata(DI->getVariable()); + else + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II) + .addImm(CI->getZExtValue()).addImm(DI->getOffset()) + .addMetadata(DI->getVariable()); } else if (const ConstantFP *CF = dyn_cast<ConstantFP>(V)) { BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II) .addFPImm(CF).addImm(DI->getOffset()) @@ -1085,7 +1090,7 @@ unsigned FastISel::createResultReg(const TargetRegisterClass* RC) { unsigned FastISel::FastEmitInst_(unsigned MachineInstOpcode, const TargetRegisterClass* RC) { unsigned ResultReg = createResultReg(RC); - const TargetInstrDesc &II = TII.get(MachineInstOpcode); + const MCInstrDesc &II = TII.get(MachineInstOpcode); BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, ResultReg); return ResultReg; @@ -1095,7 +1100,7 @@ unsigned FastISel::FastEmitInst_r(unsigned MachineInstOpcode, const TargetRegisterClass *RC, unsigned Op0, bool Op0IsKill) { unsigned ResultReg = createResultReg(RC); - const TargetInstrDesc &II = TII.get(MachineInstOpcode); + const MCInstrDesc &II = TII.get(MachineInstOpcode); if (II.getNumDefs() >= 1) BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, ResultReg) @@ -1115,7 +1120,7 @@ unsigned FastISel::FastEmitInst_rr(unsigned MachineInstOpcode, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { unsigned ResultReg = createResultReg(RC); - const TargetInstrDesc &II = TII.get(MachineInstOpcode); + const MCInstrDesc &II = TII.get(MachineInstOpcode); if (II.getNumDefs() >= 1) BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, ResultReg) @@ -1137,7 +1142,7 @@ unsigned FastISel::FastEmitInst_rrr(unsigned MachineInstOpcode, unsigned Op1, bool Op1IsKill, unsigned Op2, bool Op2IsKill) { unsigned ResultReg = createResultReg(RC); - const TargetInstrDesc &II = TII.get(MachineInstOpcode); + const MCInstrDesc &II = TII.get(MachineInstOpcode); if (II.getNumDefs() >= 1) BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, ResultReg) @@ -1160,7 +1165,7 @@ unsigned FastISel::FastEmitInst_ri(unsigned MachineInstOpcode, unsigned Op0, bool Op0IsKill, uint64_t Imm) { unsigned ResultReg = createResultReg(RC); - const TargetInstrDesc &II = TII.get(MachineInstOpcode); + const MCInstrDesc &II = TII.get(MachineInstOpcode); if (II.getNumDefs() >= 1) BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, ResultReg) @@ -1181,7 +1186,7 @@ unsigned FastISel::FastEmitInst_rii(unsigned MachineInstOpcode, unsigned Op0, bool Op0IsKill, uint64_t Imm1, uint64_t Imm2) { unsigned ResultReg = createResultReg(RC); - const TargetInstrDesc &II = TII.get(MachineInstOpcode); + const MCInstrDesc &II = TII.get(MachineInstOpcode); if (II.getNumDefs() >= 1) BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, ResultReg) @@ -1204,7 +1209,7 @@ unsigned FastISel::FastEmitInst_rf(unsigned MachineInstOpcode, unsigned Op0, bool Op0IsKill, const ConstantFP *FPImm) { unsigned ResultReg = createResultReg(RC); - const TargetInstrDesc &II = TII.get(MachineInstOpcode); + const MCInstrDesc &II = TII.get(MachineInstOpcode); if (II.getNumDefs() >= 1) BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, ResultReg) @@ -1226,7 +1231,7 @@ unsigned FastISel::FastEmitInst_rri(unsigned MachineInstOpcode, unsigned Op1, bool Op1IsKill, uint64_t Imm) { unsigned ResultReg = createResultReg(RC); - const TargetInstrDesc &II = TII.get(MachineInstOpcode); + const MCInstrDesc &II = TII.get(MachineInstOpcode); if (II.getNumDefs() >= 1) BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, ResultReg) @@ -1248,7 +1253,7 @@ unsigned FastISel::FastEmitInst_i(unsigned MachineInstOpcode, const TargetRegisterClass *RC, uint64_t Imm) { unsigned ResultReg = createResultReg(RC); - const TargetInstrDesc &II = TII.get(MachineInstOpcode); + const MCInstrDesc &II = TII.get(MachineInstOpcode); if (II.getNumDefs() >= 1) BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, ResultReg).addImm(Imm); @@ -1264,7 +1269,7 @@ unsigned FastISel::FastEmitInst_ii(unsigned MachineInstOpcode, const TargetRegisterClass *RC, uint64_t Imm1, uint64_t Imm2) { unsigned ResultReg = createResultReg(RC); - const TargetInstrDesc &II = TII.get(MachineInstOpcode); + const MCInstrDesc &II = TII.get(MachineInstOpcode); if (II.getNumDefs() >= 1) BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, II, ResultReg) diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index 2a65d65..f0f4743 100644 --- a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -106,10 +106,10 @@ EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, bool IsCloned, continue; Match = false; if (User->isMachineOpcode()) { - const TargetInstrDesc &II = TII->get(User->getMachineOpcode()); + const MCInstrDesc &II = TII->get(User->getMachineOpcode()); const TargetRegisterClass *RC = 0; if (i+II.getNumDefs() < II.getNumOperands()) - RC = II.OpInfo[i+II.getNumDefs()].getRegClass(TRI); + RC = TII->getRegClass(II, i+II.getNumDefs(), TRI); if (!UseRC) UseRC = RC; else if (RC) { @@ -178,7 +178,7 @@ unsigned InstrEmitter::getDstOfOnlyCopyToRegUse(SDNode *Node, } void InstrEmitter::CreateVirtualRegisters(SDNode *Node, MachineInstr *MI, - const TargetInstrDesc &II, + const MCInstrDesc &II, bool IsClone, bool IsCloned, DenseMap<SDValue, unsigned> &VRBaseMap) { assert(Node->getMachineOpcode() != TargetOpcode::IMPLICIT_DEF && @@ -189,7 +189,7 @@ void InstrEmitter::CreateVirtualRegisters(SDNode *Node, MachineInstr *MI, // is a vreg in the same register class, use the CopyToReg'd destination // register instead of creating a new vreg. unsigned VRBase = 0; - const TargetRegisterClass *RC = II.OpInfo[i].getRegClass(TRI); + const TargetRegisterClass *RC = TII->getRegClass(II, i, TRI); if (II.OpInfo[i].isOptionalDef()) { // Optional def must be a physical register. unsigned NumResults = CountResults(Node); @@ -242,7 +242,7 @@ unsigned InstrEmitter::getVR(SDValue Op, Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF) { // Add an IMPLICIT_DEF instruction before every use. unsigned VReg = getDstOfOnlyCopyToRegUse(Op.getNode(), Op.getResNo()); - // IMPLICIT_DEF can produce any type of result so its TargetInstrDesc + // IMPLICIT_DEF can produce any type of result so its MCInstrDesc // does not include operand register class info. if (!VReg) { const TargetRegisterClass *RC = TLI->getRegClassFor(Op.getValueType()); @@ -265,7 +265,7 @@ unsigned InstrEmitter::getVR(SDValue Op, void InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op, unsigned IIOpNum, - const TargetInstrDesc *II, + const MCInstrDesc *II, DenseMap<SDValue, unsigned> &VRBaseMap, bool IsDebug, bool IsClone, bool IsCloned) { assert(Op.getValueType() != MVT::Other && @@ -275,9 +275,9 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op, unsigned VReg = getVR(Op, VRBaseMap); assert(TargetRegisterInfo::isVirtualRegister(VReg) && "Not a vreg?"); - const TargetInstrDesc &TID = MI->getDesc(); - bool isOptDef = IIOpNum < TID.getNumOperands() && - TID.OpInfo[IIOpNum].isOptionalDef(); + const MCInstrDesc &MCID = MI->getDesc(); + bool isOptDef = IIOpNum < MCID.getNumOperands() && + MCID.OpInfo[IIOpNum].isOptionalDef(); // If the instruction requires a register in a different class, create // a new virtual register and copy the value into it. @@ -285,8 +285,8 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op, const TargetRegisterClass *SrcRC = MRI->getRegClass(VReg); const TargetRegisterClass *DstRC = 0; if (IIOpNum < II->getNumOperands()) - DstRC = II->OpInfo[IIOpNum].getRegClass(TRI); - assert((DstRC || (TID.isVariadic() && IIOpNum >= TID.getNumOperands())) && + DstRC = TII->getRegClass(*II, IIOpNum, TRI); + assert((DstRC || (MCID.isVariadic() && IIOpNum >= MCID.getNumOperands())) && "Don't have operand info for this instruction!"); if (DstRC && !SrcRC->hasSuperClassEq(DstRC)) { unsigned NewVReg = MRI->createVirtualRegister(DstRC); @@ -312,7 +312,7 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op, while (Idx > 0 && MI->getOperand(Idx-1).isReg() && MI->getOperand(Idx-1).isImplicit()) --Idx; - bool isTied = MI->getDesc().getOperandConstraint(Idx, TOI::TIED_TO) != -1; + bool isTied = MI->getDesc().getOperandConstraint(Idx, MCOI::TIED_TO) != -1; if (isTied) isKill = false; } @@ -330,7 +330,7 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op, /// assertions only. void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op, unsigned IIOpNum, - const TargetInstrDesc *II, + const MCInstrDesc *II, DenseMap<SDValue, unsigned> &VRBaseMap, bool IsDebug, bool IsClone, bool IsCloned) { if (Op.isMachineOpcode()) { @@ -556,7 +556,7 @@ void InstrEmitter::EmitRegSequence(SDNode *Node, unsigned NumOps = Node->getNumOperands(); assert((NumOps & 1) == 1 && "REG_SEQUENCE must have an odd number of operands!"); - const TargetInstrDesc &II = TII->get(TargetOpcode::REG_SEQUENCE); + const MCInstrDesc &II = TII->get(TargetOpcode::REG_SEQUENCE); for (unsigned i = 1; i != NumOps; ++i) { SDValue Op = Node->getOperand(i); if ((i & 1) == 0) { @@ -597,7 +597,7 @@ InstrEmitter::EmitDbgValue(SDDbgValue *SD, return TII->emitFrameIndexDebugValue(*MF, FrameIx, Offset, MDPtr, DL); } // Otherwise, we're going to create an instruction here. - const TargetInstrDesc &II = TII->get(TargetOpcode::DBG_VALUE); + const MCInstrDesc &II = TII->get(TargetOpcode::DBG_VALUE); MachineInstrBuilder MIB = BuildMI(*MF, DL, II); if (SD->getKind() == SDDbgValue::SDNODE) { SDNode *Node = SD->getSDNode(); @@ -616,12 +616,8 @@ InstrEmitter::EmitDbgValue(SDDbgValue *SD, } else if (SD->getKind() == SDDbgValue::CONST) { const Value *V = SD->getConst(); if (const ConstantInt *CI = dyn_cast<ConstantInt>(V)) { - // FIXME: SDDbgValue constants aren't updated with legalization, so it's - // possible to have i128 constants in them at this point. Dwarf writer - // does not handle i128 constants at the moment so, as a crude workaround, - // just drop the debug info if this happens. - if (!CI->getValue().isSignedIntN(64)) - MIB.addReg(0U); + if (CI->getBitWidth() > 64) + MIB.addCImm(CI); else MIB.addImm(CI->getSExtValue()); } else if (const ConstantFP *CF = dyn_cast<ConstantFP>(V)) { @@ -672,7 +668,7 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, // We want a unique VR for each IMPLICIT_DEF use. return; - const TargetInstrDesc &II = TII->get(Opc); + const MCInstrDesc &II = TII->get(Opc); unsigned NumResults = CountResults(Node); unsigned NodeOperands = CountOperands(Node); bool HasPhysRegOuts = NumResults > II.getNumDefs() && II.getImplicitDefs()!=0; @@ -701,9 +697,9 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, UsedRegs.push_back(cast<RegisterSDNode>(F->getOperand(1))->getReg()); else { // Collect declared implicit uses. - const TargetInstrDesc &TID = TII->get(F->getMachineOpcode()); - UsedRegs.append(TID.getImplicitUses(), - TID.getImplicitUses() + TID.getNumImplicitUses()); + const MCInstrDesc &MCID = TII->get(F->getMachineOpcode()); + UsedRegs.append(MCID.getImplicitUses(), + MCID.getImplicitUses() + MCID.getNumImplicitUses()); // In addition to declared implicit uses, we must also check for // direct RegisterSDNode operands. for (unsigned i = 0, e = F->getNumOperands(); i != e; ++i) @@ -855,6 +851,7 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, } break; case InlineAsm::Kind_RegDefEarlyClobber: + case InlineAsm::Kind_Clobber: for (; NumVals; --NumVals, ++i) { unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg(); MI->addOperand(MachineOperand::CreateReg(Reg, /*isDef=*/ true, diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.h b/lib/CodeGen/SelectionDAG/InstrEmitter.h index 02c044c..19fc044 100644 --- a/lib/CodeGen/SelectionDAG/InstrEmitter.h +++ b/lib/CodeGen/SelectionDAG/InstrEmitter.h @@ -22,7 +22,7 @@ namespace llvm { -class TargetInstrDesc; +class MCInstrDesc; class SDDbgValue; class InstrEmitter { @@ -49,7 +49,7 @@ class InstrEmitter { unsigned ResNo) const; void CreateVirtualRegisters(SDNode *Node, MachineInstr *MI, - const TargetInstrDesc &II, + const MCInstrDesc &II, bool IsClone, bool IsCloned, DenseMap<SDValue, unsigned> &VRBaseMap); @@ -63,7 +63,7 @@ class InstrEmitter { /// not in the required register class. void AddRegisterOperand(MachineInstr *MI, SDValue Op, unsigned IIOpNum, - const TargetInstrDesc *II, + const MCInstrDesc *II, DenseMap<SDValue, unsigned> &VRBaseMap, bool IsDebug, bool IsClone, bool IsCloned); @@ -73,7 +73,7 @@ class InstrEmitter { /// assertions only. void AddOperand(MachineInstr *MI, SDValue Op, unsigned IIOpNum, - const TargetInstrDesc *II, + const MCInstrDesc *II, DenseMap<SDValue, unsigned> &VRBaseMap, bool IsDebug, bool IsClone, bool IsCloned); diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp index 7b560d1..b275c63 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp @@ -249,14 +249,14 @@ SUnit *ScheduleDAGFast::CopyAndMoveSuccessors(SUnit *SU) { assert(N->getNodeId() == -1 && "Node already inserted!"); N->setNodeId(NewSU->NodeNum); - const TargetInstrDesc &TID = TII->get(N->getMachineOpcode()); - for (unsigned i = 0; i != TID.getNumOperands(); ++i) { - if (TID.getOperandConstraint(i, TOI::TIED_TO) != -1) { + const MCInstrDesc &MCID = TII->get(N->getMachineOpcode()); + for (unsigned i = 0; i != MCID.getNumOperands(); ++i) { + if (MCID.getOperandConstraint(i, MCOI::TIED_TO) != -1) { NewSU->isTwoAddress = true; break; } } - if (TID.isCommutable()) + if (MCID.isCommutable()) NewSU->isCommutable = true; // LoadNode may already exist. This can happen when there is another @@ -422,10 +422,10 @@ void ScheduleDAGFast::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg, /// FIXME: Move to SelectionDAG? static EVT getPhysicalRegisterVT(SDNode *N, unsigned Reg, const TargetInstrInfo *TII) { - const TargetInstrDesc &TID = TII->get(N->getMachineOpcode()); - assert(TID.ImplicitDefs && "Physical reg def must be in implicit def list!"); - unsigned NumRes = TID.getNumDefs(); - for (const unsigned *ImpDef = TID.getImplicitDefs(); *ImpDef; ++ImpDef) { + const MCInstrDesc &MCID = TII->get(N->getMachineOpcode()); + assert(MCID.ImplicitDefs && "Physical reg def must be in implicit def list!"); + unsigned NumRes = MCID.getNumDefs(); + for (const unsigned *ImpDef = MCID.getImplicitDefs(); *ImpDef; ++ImpDef) { if (Reg == *ImpDef) break; ++NumRes; @@ -490,7 +490,8 @@ bool ScheduleDAGFast::DelayForLiveRegsBottomUp(SUnit *SU, ++i; // Skip the ID value. if (InlineAsm::isRegDefKind(Flags) || - InlineAsm::isRegDefEarlyClobberKind(Flags)) { + InlineAsm::isRegDefEarlyClobberKind(Flags) || + InlineAsm::isClobberKind(Flags)) { // Check for def of register or earlyclobber register. for (; NumVals; --NumVals, ++i) { unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg(); @@ -504,10 +505,10 @@ bool ScheduleDAGFast::DelayForLiveRegsBottomUp(SUnit *SU, } if (!Node->isMachineOpcode()) continue; - const TargetInstrDesc &TID = TII->get(Node->getMachineOpcode()); - if (!TID.ImplicitDefs) + const MCInstrDesc &MCID = TII->get(Node->getMachineOpcode()); + if (!MCID.ImplicitDefs) continue; - for (const unsigned *Reg = TID.ImplicitDefs; *Reg; ++Reg) { + for (const unsigned *Reg = MCID.ImplicitDefs; *Reg; ++Reg) { CheckForLiveRegDef(SU, *Reg, LiveRegDefs, RegAdded, LRegs, TRI); } } diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp index a827187..12b1838 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp @@ -302,8 +302,8 @@ static void GetCostForDef(const ScheduleDAGSDNodes::RegDefIter &RegDefPos, } unsigned Idx = RegDefPos.GetIdx(); - const TargetInstrDesc Desc = TII->get(Opcode); - const TargetRegisterClass *RC = Desc.getRegClass(Idx, TRI); + const MCInstrDesc Desc = TII->get(Opcode); + const TargetRegisterClass *RC = TII->getRegClass(Desc, Idx, TRI); RegClass = RC->getID(); // FIXME: Cost arbitrarily set to 1 because there doesn't seem to be a // better way to determine it. @@ -837,14 +837,14 @@ SUnit *ScheduleDAGRRList::CopyAndMoveSuccessors(SUnit *SU) { assert(N->getNodeId() == -1 && "Node already inserted!"); N->setNodeId(NewSU->NodeNum); - const TargetInstrDesc &TID = TII->get(N->getMachineOpcode()); - for (unsigned i = 0; i != TID.getNumOperands(); ++i) { - if (TID.getOperandConstraint(i, TOI::TIED_TO) != -1) { + const MCInstrDesc &MCID = TII->get(N->getMachineOpcode()); + for (unsigned i = 0; i != MCID.getNumOperands(); ++i) { + if (MCID.getOperandConstraint(i, MCOI::TIED_TO) != -1) { NewSU->isTwoAddress = true; break; } } - if (TID.isCommutable()) + if (MCID.isCommutable()) NewSU->isCommutable = true; InitNumRegDefsLeft(NewSU); @@ -1024,10 +1024,10 @@ void ScheduleDAGRRList::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg, /// FIXME: Move to SelectionDAG? static EVT getPhysicalRegisterVT(SDNode *N, unsigned Reg, const TargetInstrInfo *TII) { - const TargetInstrDesc &TID = TII->get(N->getMachineOpcode()); - assert(TID.ImplicitDefs && "Physical reg def must be in implicit def list!"); - unsigned NumRes = TID.getNumDefs(); - for (const unsigned *ImpDef = TID.getImplicitDefs(); *ImpDef; ++ImpDef) { + const MCInstrDesc &MCID = TII->get(N->getMachineOpcode()); + assert(MCID.ImplicitDefs && "Physical reg def must be in implicit def list!"); + unsigned NumRes = MCID.getNumDefs(); + for (const unsigned *ImpDef = MCID.getImplicitDefs(); *ImpDef; ++ImpDef) { if (Reg == *ImpDef) break; ++NumRes; @@ -1092,7 +1092,8 @@ DelayForLiveRegsBottomUp(SUnit *SU, SmallVector<unsigned, 4> &LRegs) { ++i; // Skip the ID value. if (InlineAsm::isRegDefKind(Flags) || - InlineAsm::isRegDefEarlyClobberKind(Flags)) { + InlineAsm::isRegDefEarlyClobberKind(Flags) || + InlineAsm::isClobberKind(Flags)) { // Check for def of register or earlyclobber register. for (; NumVals; --NumVals, ++i) { unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg(); @@ -1107,10 +1108,10 @@ DelayForLiveRegsBottomUp(SUnit *SU, SmallVector<unsigned, 4> &LRegs) { if (!Node->isMachineOpcode()) continue; - const TargetInstrDesc &TID = TII->get(Node->getMachineOpcode()); - if (!TID.ImplicitDefs) + const MCInstrDesc &MCID = TII->get(Node->getMachineOpcode()); + if (!MCID.ImplicitDefs) continue; - for (const unsigned *Reg = TID.ImplicitDefs; *Reg; ++Reg) + for (const unsigned *Reg = MCID.ImplicitDefs; *Reg; ++Reg) CheckForLiveRegDef(SU, *Reg, LiveRegDefs, RegAdded, LRegs, TRI); } @@ -2028,13 +2029,9 @@ void RegReductionPQBase::UnscheduledNode(SUnit *SU) { unsigned POpc = PN->getMachineOpcode(); if (POpc == TargetOpcode::IMPLICIT_DEF) continue; - if (POpc == TargetOpcode::EXTRACT_SUBREG) { - EVT VT = PN->getOperand(0).getValueType(); - unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); - RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); - continue; - } else if (POpc == TargetOpcode::INSERT_SUBREG || - POpc == TargetOpcode::SUBREG_TO_REG) { + if (POpc == TargetOpcode::EXTRACT_SUBREG || + POpc == TargetOpcode::INSERT_SUBREG || + POpc == TargetOpcode::SUBREG_TO_REG) { EVT VT = PN->getValueType(0); unsigned RCId = TLI->getRepRegClassFor(VT)->getID(); RegPressure[RCId] += TLI->getRepRegClassCostFor(VT); @@ -2609,11 +2606,11 @@ void RegReductionPQBase::initNodes(std::vector<SUnit> &sunits) { bool RegReductionPQBase::canClobber(const SUnit *SU, const SUnit *Op) { if (SU->isTwoAddress) { unsigned Opc = SU->getNode()->getMachineOpcode(); - const TargetInstrDesc &TID = TII->get(Opc); - unsigned NumRes = TID.getNumDefs(); - unsigned NumOps = TID.getNumOperands() - NumRes; + const MCInstrDesc &MCID = TII->get(Opc); + unsigned NumRes = MCID.getNumDefs(); + unsigned NumOps = MCID.getNumOperands() - NumRes; for (unsigned i = 0; i != NumOps; ++i) { - if (TID.getOperandConstraint(i+NumRes, TOI::TIED_TO) != -1) { + if (MCID.getOperandConstraint(i+NumRes, MCOI::TIED_TO) != -1) { SDNode *DU = SU->getNode()->getOperand(i).getNode(); if (DU->getNodeId() != -1 && Op->OrigNode == &(*SUnits)[DU->getNodeId()]) @@ -2793,11 +2790,11 @@ void RegReductionPQBase::AddPseudoTwoAddrDeps() { bool isLiveOut = hasOnlyLiveOutUses(SU); unsigned Opc = Node->getMachineOpcode(); - const TargetInstrDesc &TID = TII->get(Opc); - unsigned NumRes = TID.getNumDefs(); - unsigned NumOps = TID.getNumOperands() - NumRes; + const MCInstrDesc &MCID = TII->get(Opc); + unsigned NumRes = MCID.getNumDefs(); + unsigned NumOps = MCID.getNumOperands() - NumRes; for (unsigned j = 0; j != NumOps; ++j) { - if (TID.getOperandConstraint(j+NumRes, TOI::TIED_TO) == -1) + if (MCID.getOperandConstraint(j+NumRes, MCOI::TIED_TO) == -1) continue; SDNode *DU = SU->getNode()->getOperand(j).getNode(); if (DU->getNodeId() == -1) diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp index dbc623b..63ca326 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -17,6 +17,7 @@ #include "ScheduleDAGSDNodes.h" #include "InstrEmitter.h" #include "llvm/CodeGen/SelectionDAG.h" +#include "llvm/MC/MCInstrItineraries.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetLowering.h" @@ -111,7 +112,7 @@ static void CheckForPhysRegDependency(SDNode *Def, SDNode *User, unsigned Op, unsigned ResNo = User->getOperand(2).getResNo(); if (Def->isMachineOpcode()) { - const TargetInstrDesc &II = TII->get(Def->getMachineOpcode()); + const MCInstrDesc &II = TII->get(Def->getMachineOpcode()); if (ResNo >= II.getNumDefs() && II.ImplicitDefs[ResNo - II.getNumDefs()] == Reg) { PhysReg = Reg; @@ -255,8 +256,8 @@ void ScheduleDAGSDNodes::ClusterNodes() { continue; unsigned Opc = Node->getMachineOpcode(); - const TargetInstrDesc &TID = TII->get(Opc); - if (TID.mayLoad()) + const MCInstrDesc &MCID = TII->get(Opc); + if (MCID.mayLoad()) // Cluster loads from "near" addresses into combined SUnits. ClusterNeighboringLoads(Node); } @@ -390,14 +391,14 @@ void ScheduleDAGSDNodes::AddSchedEdges() { if (MainNode->isMachineOpcode()) { unsigned Opc = MainNode->getMachineOpcode(); - const TargetInstrDesc &TID = TII->get(Opc); - for (unsigned i = 0; i != TID.getNumOperands(); ++i) { - if (TID.getOperandConstraint(i, TOI::TIED_TO) != -1) { + const MCInstrDesc &MCID = TII->get(Opc); + for (unsigned i = 0; i != MCID.getNumOperands(); ++i) { + if (MCID.getOperandConstraint(i, MCOI::TIED_TO) != -1) { SU->isTwoAddress = true; break; } } - if (TID.isCommutable()) + if (MCID.isCommutable()) SU->isCommutable = true; } @@ -520,14 +521,7 @@ void ScheduleDAGSDNodes::RegDefIter::Advance() { for (;DefIdx < NodeNumDefs; ++DefIdx) { if (!Node->hasAnyUseOfValue(DefIdx)) continue; - if (Node->isMachineOpcode() && - Node->getMachineOpcode() == TargetOpcode::EXTRACT_SUBREG) { - // Propagate the incoming (full-register) type. I doubt it's needed. - ValueType = Node->getOperand(0).getValueType(); - } - else { - ValueType = Node->getValueType(DefIdx); - } + ValueType = Node->getValueType(DefIdx); ++DefIdx; return; // Found a normal regdef. } diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h index 3ad2bd6..9c27b2e 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h @@ -140,7 +140,7 @@ namespace llvm { } unsigned GetIdx() const { - return DefIdx; + return DefIdx-1; } void Advance(); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 19bfa33..ea59ca1 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -5428,55 +5428,6 @@ typedef SmallVector<SDISelAsmOperandInfo,16> SDISelAsmOperandInfoVector; } // end anonymous namespace -/// isAllocatableRegister - If the specified register is safe to allocate, -/// i.e. it isn't a stack pointer or some other special register, return the -/// register class for the register. Otherwise, return null. -static const TargetRegisterClass * -isAllocatableRegister(unsigned Reg, MachineFunction &MF, - const TargetLowering &TLI, - const TargetRegisterInfo *TRI) { - EVT FoundVT = MVT::Other; - const TargetRegisterClass *FoundRC = 0; - for (TargetRegisterInfo::regclass_iterator RCI = TRI->regclass_begin(), - E = TRI->regclass_end(); RCI != E; ++RCI) { - EVT ThisVT = MVT::Other; - - const TargetRegisterClass *RC = *RCI; - if (!RC->isAllocatable()) - continue; - // If none of the value types for this register class are valid, we - // can't use it. For example, 64-bit reg classes on 32-bit targets. - for (TargetRegisterClass::vt_iterator I = RC->vt_begin(), E = RC->vt_end(); - I != E; ++I) { - if (TLI.isTypeLegal(*I)) { - // If we have already found this register in a different register class, - // choose the one with the largest VT specified. For example, on - // PowerPC, we favor f64 register classes over f32. - if (FoundVT == MVT::Other || FoundVT.bitsLT(*I)) { - ThisVT = *I; - break; - } - } - } - - if (ThisVT == MVT::Other) continue; - - // NOTE: This isn't ideal. In particular, this might allocate the - // frame pointer in functions that need it (due to them not being taken - // out of allocation, because a variable sized allocation hasn't been seen - // yet). This is a slight code pessimization, but should still work. - ArrayRef<unsigned> RawOrder = RC->getRawAllocationOrder(MF); - if (std::find(RawOrder.begin(), RawOrder.end(), Reg) != RawOrder.end()) { - // We found a matching register class. Keep looking at others in case - // we find one with larger registers that this physreg is also in. - FoundRC = RC; - FoundVT = ThisVT; - break; - } - } - return FoundRC; -} - /// GetRegistersForValue - Assign registers (virtual or physical) for the /// specified operand. We prefer to assign virtual registers, to allow the /// register allocator to handle the assignment process. However, if the asm @@ -5611,58 +5562,6 @@ static void GetRegistersForValue(SelectionDAG &DAG, return; } - // This is a reference to a register class that doesn't directly correspond - // to an LLVM register class. Allocate NumRegs consecutive, available, - // registers from the class. - std::vector<unsigned> RegClassRegs - = TLI.getRegClassForInlineAsmConstraint(OpInfo.ConstraintCode, - OpInfo.ConstraintVT); - - const TargetRegisterInfo *TRI = DAG.getTarget().getRegisterInfo(); - BitVector Reserved = TRI->getReservedRegs(MF); - unsigned NumAllocated = 0; - for (unsigned i = 0, e = RegClassRegs.size(); i != e; ++i) { - unsigned Reg = RegClassRegs[i]; - // Filter out the reserved registers, but note that reserved registers are - // not fully determined at this point. We may still decide we need a frame - // pointer. - if (Reserved.test(Reg)) - continue; - // See if this register is available. - if ((isOutReg && OutputRegs.count(Reg)) || // Already used. - (isInReg && InputRegs.count(Reg))) { // Already used. - // Make sure we find consecutive registers. - NumAllocated = 0; - continue; - } - - // Check to see if this register is allocatable (i.e. don't give out the - // stack pointer). - const TargetRegisterClass *RC = isAllocatableRegister(Reg, MF, TLI, TRI); - if (!RC) { // Couldn't allocate this register. - // Reset NumAllocated to make sure we return consecutive registers. - NumAllocated = 0; - continue; - } - - // Okay, this register is good, we can use it. - ++NumAllocated; - - // If we allocated enough consecutive registers, succeed. - if (NumAllocated == NumRegs) { - unsigned RegStart = (i-NumAllocated)+1; - unsigned RegEnd = i+1; - // Mark all of the allocated registers used. - for (unsigned i = RegStart; i != RegEnd; ++i) - Regs.push_back(RegClassRegs[i]); - - OpInfo.AssignedRegs = RegsForValue(Regs, *RC->vt_begin(), - OpInfo.ConstraintVT); - OpInfo.MarkAllocatedRegs(isOutReg, isInReg, OutputRegs, InputRegs, *TRI); - return; - } - } - // Otherwise, we couldn't allocate enough registers for this. } @@ -6051,8 +5950,7 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { // Add the clobbered value to the operand list, so that the register // allocator is aware that the physreg got clobbered. if (!OpInfo.AssignedRegs.Regs.empty()) - OpInfo.AssignedRegs.AddInlineAsmOperands( - InlineAsm::Kind_RegDefEarlyClobber, + OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber, false, 0, DAG, AsmNodeOperands); break; diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index dc8044b..87bb296 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -354,9 +354,9 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { const MachineBasicBlock *MBB = I; for (MachineBasicBlock::const_iterator II = MBB->begin(), IE = MBB->end(); II != IE; ++II) { - const TargetInstrDesc &TID = TM.getInstrInfo()->get(II->getOpcode()); + const MCInstrDesc &MCID = TM.getInstrInfo()->get(II->getOpcode()); - if ((TID.isCall() && !TID.isReturn()) || + if ((MCID.isCall() && !MCID.isReturn()) || II->isStackAligningInlineAsm()) { MFI->setHasCalls(true); goto done; @@ -681,7 +681,7 @@ void SelectionDAGISel::PrepareEHLandingPad() { // landing pad can thus be detected via the MachineModuleInfo. MCSymbol *Label = MF->getMMI().addLandingPad(FuncInfo->MBB); - const TargetInstrDesc &II = TM.getInstrInfo()->get(TargetOpcode::EH_LABEL); + const MCInstrDesc &II = TM.getInstrInfo()->get(TargetOpcode::EH_LABEL); BuildMI(*FuncInfo->MBB, FuncInfo->InsertPt, SDB->getCurDebugLoc(), II) .addSym(Label); @@ -2611,9 +2611,9 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, if (EmitNodeInfo & OPFL_MemRefs) { // Only attach load or store memory operands if the generated // instruction may load or store. - const TargetInstrDesc &TID = TM.getInstrInfo()->get(TargetOpc); - bool mayLoad = TID.mayLoad(); - bool mayStore = TID.mayStore(); + const MCInstrDesc &MCID = TM.getInstrInfo()->get(TargetOpc); + bool mayLoad = MCID.mayLoad(); + bool mayStore = MCID.mayStore(); unsigned NumMemRefs = 0; for (SmallVector<MachineMemOperand*, 2>::const_iterator I = diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 474dd7a..758296e 100644 --- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -2737,13 +2737,6 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op, } } -std::vector<unsigned> TargetLowering:: -getRegClassForInlineAsmConstraint(const std::string &Constraint, - EVT VT) const { - return std::vector<unsigned>(); -} - - std::pair<unsigned, const TargetRegisterClass*> TargetLowering:: getRegForInlineAsmConstraint(const std::string &Constraint, EVT VT) const { diff --git a/lib/CodeGen/SimpleRegisterCoalescing.cpp b/lib/CodeGen/SimpleRegisterCoalescing.cpp deleted file mode 100644 index 221bec5..0000000 --- a/lib/CodeGen/SimpleRegisterCoalescing.cpp +++ /dev/null @@ -1,1539 +0,0 @@ -//===-- SimpleRegisterCoalescing.cpp - Register Coalescing ----------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file implements a simple register coalescing pass that attempts to -// aggressively coalesce every register copy that it can. -// -//===----------------------------------------------------------------------===// - -#define DEBUG_TYPE "regcoalescing" -#include "SimpleRegisterCoalescing.h" -#include "VirtRegMap.h" -#include "LiveDebugVariables.h" -#include "llvm/CodeGen/LiveIntervalAnalysis.h" -#include "llvm/Value.h" -#include "llvm/Analysis/AliasAnalysis.h" -#include "llvm/CodeGen/MachineFrameInfo.h" -#include "llvm/CodeGen/MachineInstr.h" -#include "llvm/CodeGen/MachineLoopInfo.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/Passes.h" -#include "llvm/CodeGen/RegisterCoalescer.h" -#include "llvm/Target/TargetInstrInfo.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetOptions.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/ADT/OwningPtr.h" -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/STLExtras.h" -#include <algorithm> -#include <cmath> -using namespace llvm; - -STATISTIC(numJoins , "Number of interval joins performed"); -STATISTIC(numCrossRCs , "Number of cross class joins performed"); -STATISTIC(numCommutes , "Number of instruction commuting performed"); -STATISTIC(numExtends , "Number of copies extended"); -STATISTIC(NumReMats , "Number of instructions re-materialized"); -STATISTIC(numPeep , "Number of identity moves eliminated after coalescing"); -STATISTIC(numAborts , "Number of times interval joining aborted"); - -char SimpleRegisterCoalescing::ID = 0; -static cl::opt<bool> -EnableJoining("join-liveintervals", - cl::desc("Coalesce copies (default=true)"), - cl::init(true)); - -static cl::opt<bool> -DisableCrossClassJoin("disable-cross-class-join", - cl::desc("Avoid coalescing cross register class copies"), - cl::init(false), cl::Hidden); - -static cl::opt<bool> -EnablePhysicalJoin("join-physregs", - cl::desc("Join physical register copies"), - cl::init(false), cl::Hidden); - -static cl::opt<bool> -VerifyCoalescing("verify-coalescing", - cl::desc("Verify machine instrs before and after register coalescing"), - cl::Hidden); - -INITIALIZE_AG_PASS_BEGIN(SimpleRegisterCoalescing, RegisterCoalescer, - "simple-register-coalescing", "Simple Register Coalescing", - false, false, true) -INITIALIZE_PASS_DEPENDENCY(LiveIntervals) -INITIALIZE_PASS_DEPENDENCY(LiveDebugVariables) -INITIALIZE_PASS_DEPENDENCY(SlotIndexes) -INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) -INITIALIZE_PASS_DEPENDENCY(StrongPHIElimination) -INITIALIZE_PASS_DEPENDENCY(PHIElimination) -INITIALIZE_PASS_DEPENDENCY(TwoAddressInstructionPass) -INITIALIZE_AG_DEPENDENCY(AliasAnalysis) -INITIALIZE_AG_PASS_END(SimpleRegisterCoalescing, RegisterCoalescer, - "simple-register-coalescing", "Simple Register Coalescing", - false, false, true) - -char &llvm::SimpleRegisterCoalescingID = SimpleRegisterCoalescing::ID; - -void SimpleRegisterCoalescing::getAnalysisUsage(AnalysisUsage &AU) const { - AU.setPreservesCFG(); - AU.addRequired<AliasAnalysis>(); - AU.addRequired<LiveIntervals>(); - AU.addPreserved<LiveIntervals>(); - AU.addRequired<LiveDebugVariables>(); - AU.addPreserved<LiveDebugVariables>(); - AU.addPreserved<SlotIndexes>(); - AU.addRequired<MachineLoopInfo>(); - AU.addPreserved<MachineLoopInfo>(); - AU.addPreservedID(MachineDominatorsID); - AU.addPreservedID(StrongPHIEliminationID); - AU.addPreservedID(PHIEliminationID); - AU.addPreservedID(TwoAddressInstructionPassID); - MachineFunctionPass::getAnalysisUsage(AU); -} - -void SimpleRegisterCoalescing::markAsJoined(MachineInstr *CopyMI) { - /// Joined copies are not deleted immediately, but kept in JoinedCopies. - JoinedCopies.insert(CopyMI); - - /// Mark all register operands of CopyMI as <undef> so they won't affect dead - /// code elimination. - for (MachineInstr::mop_iterator I = CopyMI->operands_begin(), - E = CopyMI->operands_end(); I != E; ++I) - if (I->isReg()) - I->setIsUndef(true); -} - -/// AdjustCopiesBackFrom - We found a non-trivially-coalescable copy with IntA -/// being the source and IntB being the dest, thus this defines a value number -/// in IntB. If the source value number (in IntA) is defined by a copy from B, -/// see if we can merge these two pieces of B into a single value number, -/// eliminating a copy. For example: -/// -/// A3 = B0 -/// ... -/// B1 = A3 <- this copy -/// -/// In this case, B0 can be extended to where the B1 copy lives, allowing the B1 -/// value number to be replaced with B0 (which simplifies the B liveinterval). -/// -/// This returns true if an interval was modified. -/// -bool SimpleRegisterCoalescing::AdjustCopiesBackFrom(const CoalescerPair &CP, - MachineInstr *CopyMI) { - // Bail if there is no dst interval - can happen when merging physical subreg - // operations. - if (!li_->hasInterval(CP.getDstReg())) - return false; - - LiveInterval &IntA = - li_->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg()); - LiveInterval &IntB = - li_->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg()); - SlotIndex CopyIdx = li_->getInstructionIndex(CopyMI).getDefIndex(); - - // BValNo is a value number in B that is defined by a copy from A. 'B3' in - // the example above. - LiveInterval::iterator BLR = IntB.FindLiveRangeContaining(CopyIdx); - if (BLR == IntB.end()) return false; - VNInfo *BValNo = BLR->valno; - - // Get the location that B is defined at. Two options: either this value has - // an unknown definition point or it is defined at CopyIdx. If unknown, we - // can't process it. - if (!BValNo->isDefByCopy()) return false; - assert(BValNo->def == CopyIdx && "Copy doesn't define the value?"); - - // AValNo is the value number in A that defines the copy, A3 in the example. - SlotIndex CopyUseIdx = CopyIdx.getUseIndex(); - LiveInterval::iterator ALR = IntA.FindLiveRangeContaining(CopyUseIdx); - // The live range might not exist after fun with physreg coalescing. - if (ALR == IntA.end()) return false; - VNInfo *AValNo = ALR->valno; - // If it's re-defined by an early clobber somewhere in the live range, then - // it's not safe to eliminate the copy. FIXME: This is a temporary workaround. - // See PR3149: - // 172 %ECX<def> = MOV32rr %reg1039<kill> - // 180 INLINEASM <es:subl $5,$1 - // sbbl $3,$0>, 10, %EAX<def>, 14, %ECX<earlyclobber,def>, 9, - // %EAX<kill>, - // 36, <fi#0>, 1, %reg0, 0, 9, %ECX<kill>, 36, <fi#1>, 1, %reg0, 0 - // 188 %EAX<def> = MOV32rr %EAX<kill> - // 196 %ECX<def> = MOV32rr %ECX<kill> - // 204 %ECX<def> = MOV32rr %ECX<kill> - // 212 %EAX<def> = MOV32rr %EAX<kill> - // 220 %EAX<def> = MOV32rr %EAX - // 228 %reg1039<def> = MOV32rr %ECX<kill> - // The early clobber operand ties ECX input to the ECX def. - // - // The live interval of ECX is represented as this: - // %reg20,inf = [46,47:1)[174,230:0) 0@174-(230) 1@46-(47) - // The coalescer has no idea there was a def in the middle of [174,230]. - if (AValNo->hasRedefByEC()) - return false; - - // If AValNo is defined as a copy from IntB, we can potentially process this. - // Get the instruction that defines this value number. - if (!CP.isCoalescable(AValNo->getCopy())) - return false; - - // Get the LiveRange in IntB that this value number starts with. - LiveInterval::iterator ValLR = - IntB.FindLiveRangeContaining(AValNo->def.getPrevSlot()); - if (ValLR == IntB.end()) - return false; - - // Make sure that the end of the live range is inside the same block as - // CopyMI. - MachineInstr *ValLREndInst = - li_->getInstructionFromIndex(ValLR->end.getPrevSlot()); - if (!ValLREndInst || ValLREndInst->getParent() != CopyMI->getParent()) - return false; - - // Okay, we now know that ValLR ends in the same block that the CopyMI - // live-range starts. If there are no intervening live ranges between them in - // IntB, we can merge them. - if (ValLR+1 != BLR) return false; - - // If a live interval is a physical register, conservatively check if any - // of its aliases is overlapping the live interval of the virtual register. - // If so, do not coalesce. - if (TargetRegisterInfo::isPhysicalRegister(IntB.reg)) { - for (const unsigned *AS = tri_->getAliasSet(IntB.reg); *AS; ++AS) - if (li_->hasInterval(*AS) && IntA.overlaps(li_->getInterval(*AS))) { - DEBUG({ - dbgs() << "\t\tInterfere with alias "; - li_->getInterval(*AS).print(dbgs(), tri_); - }); - return false; - } - } - - DEBUG({ - dbgs() << "Extending: "; - IntB.print(dbgs(), tri_); - }); - - SlotIndex FillerStart = ValLR->end, FillerEnd = BLR->start; - // We are about to delete CopyMI, so need to remove it as the 'instruction - // that defines this value #'. Update the valnum with the new defining - // instruction #. - BValNo->def = FillerStart; - BValNo->setCopy(0); - - // Okay, we can merge them. We need to insert a new liverange: - // [ValLR.end, BLR.begin) of either value number, then we merge the - // two value numbers. - IntB.addRange(LiveRange(FillerStart, FillerEnd, BValNo)); - - // If the IntB live range is assigned to a physical register, and if that - // physreg has sub-registers, update their live intervals as well. - if (TargetRegisterInfo::isPhysicalRegister(IntB.reg)) { - for (const unsigned *SR = tri_->getSubRegisters(IntB.reg); *SR; ++SR) { - if (!li_->hasInterval(*SR)) - continue; - LiveInterval &SRLI = li_->getInterval(*SR); - SRLI.addRange(LiveRange(FillerStart, FillerEnd, - SRLI.getNextValue(FillerStart, 0, - li_->getVNInfoAllocator()))); - } - } - - // Okay, merge "B1" into the same value number as "B0". - if (BValNo != ValLR->valno) { - // If B1 is killed by a PHI, then the merged live range must also be killed - // by the same PHI, as B0 and B1 can not overlap. - bool HasPHIKill = BValNo->hasPHIKill(); - IntB.MergeValueNumberInto(BValNo, ValLR->valno); - if (HasPHIKill) - ValLR->valno->setHasPHIKill(true); - } - DEBUG({ - dbgs() << " result = "; - IntB.print(dbgs(), tri_); - dbgs() << "\n"; - }); - - // If the source instruction was killing the source register before the - // merge, unset the isKill marker given the live range has been extended. - int UIdx = ValLREndInst->findRegisterUseOperandIdx(IntB.reg, true); - if (UIdx != -1) { - ValLREndInst->getOperand(UIdx).setIsKill(false); - } - - // If the copy instruction was killing the destination register before the - // merge, find the last use and trim the live range. That will also add the - // isKill marker. - if (ALR->end == CopyIdx) - li_->shrinkToUses(&IntA); - - ++numExtends; - return true; -} - -/// HasOtherReachingDefs - Return true if there are definitions of IntB -/// other than BValNo val# that can reach uses of AValno val# of IntA. -bool SimpleRegisterCoalescing::HasOtherReachingDefs(LiveInterval &IntA, - LiveInterval &IntB, - VNInfo *AValNo, - VNInfo *BValNo) { - for (LiveInterval::iterator AI = IntA.begin(), AE = IntA.end(); - AI != AE; ++AI) { - if (AI->valno != AValNo) continue; - LiveInterval::Ranges::iterator BI = - std::upper_bound(IntB.ranges.begin(), IntB.ranges.end(), AI->start); - if (BI != IntB.ranges.begin()) - --BI; - for (; BI != IntB.ranges.end() && AI->end >= BI->start; ++BI) { - if (BI->valno == BValNo) - continue; - if (BI->start <= AI->start && BI->end > AI->start) - return true; - if (BI->start > AI->start && BI->start < AI->end) - return true; - } - } - return false; -} - -/// RemoveCopyByCommutingDef - We found a non-trivially-coalescable copy with -/// IntA being the source and IntB being the dest, thus this defines a value -/// number in IntB. If the source value number (in IntA) is defined by a -/// commutable instruction and its other operand is coalesced to the copy dest -/// register, see if we can transform the copy into a noop by commuting the -/// definition. For example, -/// -/// A3 = op A2 B0<kill> -/// ... -/// B1 = A3 <- this copy -/// ... -/// = op A3 <- more uses -/// -/// ==> -/// -/// B2 = op B0 A2<kill> -/// ... -/// B1 = B2 <- now an identify copy -/// ... -/// = op B2 <- more uses -/// -/// This returns true if an interval was modified. -/// -bool SimpleRegisterCoalescing::RemoveCopyByCommutingDef(const CoalescerPair &CP, - MachineInstr *CopyMI) { - // FIXME: For now, only eliminate the copy by commuting its def when the - // source register is a virtual register. We want to guard against cases - // where the copy is a back edge copy and commuting the def lengthen the - // live interval of the source register to the entire loop. - if (CP.isPhys() && CP.isFlipped()) - return false; - - // Bail if there is no dst interval. - if (!li_->hasInterval(CP.getDstReg())) - return false; - - SlotIndex CopyIdx = li_->getInstructionIndex(CopyMI).getDefIndex(); - - LiveInterval &IntA = - li_->getInterval(CP.isFlipped() ? CP.getDstReg() : CP.getSrcReg()); - LiveInterval &IntB = - li_->getInterval(CP.isFlipped() ? CP.getSrcReg() : CP.getDstReg()); - - // BValNo is a value number in B that is defined by a copy from A. 'B3' in - // the example above. - VNInfo *BValNo = IntB.getVNInfoAt(CopyIdx); - if (!BValNo || !BValNo->isDefByCopy()) - return false; - - assert(BValNo->def == CopyIdx && "Copy doesn't define the value?"); - - // AValNo is the value number in A that defines the copy, A3 in the example. - VNInfo *AValNo = IntA.getVNInfoAt(CopyIdx.getUseIndex()); - assert(AValNo && "COPY source not live"); - - // If other defs can reach uses of this def, then it's not safe to perform - // the optimization. - if (AValNo->isPHIDef() || AValNo->isUnused() || AValNo->hasPHIKill()) - return false; - MachineInstr *DefMI = li_->getInstructionFromIndex(AValNo->def); - if (!DefMI) - return false; - const TargetInstrDesc &TID = DefMI->getDesc(); - if (!TID.isCommutable()) - return false; - // If DefMI is a two-address instruction then commuting it will change the - // destination register. - int DefIdx = DefMI->findRegisterDefOperandIdx(IntA.reg); - assert(DefIdx != -1); - unsigned UseOpIdx; - if (!DefMI->isRegTiedToUseOperand(DefIdx, &UseOpIdx)) - return false; - unsigned Op1, Op2, NewDstIdx; - if (!tii_->findCommutedOpIndices(DefMI, Op1, Op2)) - return false; - if (Op1 == UseOpIdx) - NewDstIdx = Op2; - else if (Op2 == UseOpIdx) - NewDstIdx = Op1; - else - return false; - - MachineOperand &NewDstMO = DefMI->getOperand(NewDstIdx); - unsigned NewReg = NewDstMO.getReg(); - if (NewReg != IntB.reg || !NewDstMO.isKill()) - return false; - - // Make sure there are no other definitions of IntB that would reach the - // uses which the new definition can reach. - if (HasOtherReachingDefs(IntA, IntB, AValNo, BValNo)) - return false; - - // Abort if the aliases of IntB.reg have values that are not simply the - // clobbers from the superreg. - if (TargetRegisterInfo::isPhysicalRegister(IntB.reg)) - for (const unsigned *AS = tri_->getAliasSet(IntB.reg); *AS; ++AS) - if (li_->hasInterval(*AS) && - HasOtherReachingDefs(IntA, li_->getInterval(*AS), AValNo, 0)) - return false; - - // If some of the uses of IntA.reg is already coalesced away, return false. - // It's not possible to determine whether it's safe to perform the coalescing. - for (MachineRegisterInfo::use_nodbg_iterator UI = - mri_->use_nodbg_begin(IntA.reg), - UE = mri_->use_nodbg_end(); UI != UE; ++UI) { - MachineInstr *UseMI = &*UI; - SlotIndex UseIdx = li_->getInstructionIndex(UseMI); - LiveInterval::iterator ULR = IntA.FindLiveRangeContaining(UseIdx); - if (ULR == IntA.end()) - continue; - if (ULR->valno == AValNo && JoinedCopies.count(UseMI)) - return false; - } - - DEBUG(dbgs() << "\tRemoveCopyByCommutingDef: " << AValNo->def << '\t' - << *DefMI); - - // At this point we have decided that it is legal to do this - // transformation. Start by commuting the instruction. - MachineBasicBlock *MBB = DefMI->getParent(); - MachineInstr *NewMI = tii_->commuteInstruction(DefMI); - if (!NewMI) - return false; - if (TargetRegisterInfo::isVirtualRegister(IntA.reg) && - TargetRegisterInfo::isVirtualRegister(IntB.reg) && - !mri_->constrainRegClass(IntB.reg, mri_->getRegClass(IntA.reg))) - return false; - if (NewMI != DefMI) { - li_->ReplaceMachineInstrInMaps(DefMI, NewMI); - MBB->insert(DefMI, NewMI); - MBB->erase(DefMI); - } - unsigned OpIdx = NewMI->findRegisterUseOperandIdx(IntA.reg, false); - NewMI->getOperand(OpIdx).setIsKill(); - - // If ALR and BLR overlaps and end of BLR extends beyond end of ALR, e.g. - // A = or A, B - // ... - // B = A - // ... - // C = A<kill> - // ... - // = B - - // Update uses of IntA of the specific Val# with IntB. - for (MachineRegisterInfo::use_iterator UI = mri_->use_begin(IntA.reg), - UE = mri_->use_end(); UI != UE;) { - MachineOperand &UseMO = UI.getOperand(); - MachineInstr *UseMI = &*UI; - ++UI; - if (JoinedCopies.count(UseMI)) - continue; - if (UseMI->isDebugValue()) { - // FIXME These don't have an instruction index. Not clear we have enough - // info to decide whether to do this replacement or not. For now do it. - UseMO.setReg(NewReg); - continue; - } - SlotIndex UseIdx = li_->getInstructionIndex(UseMI).getUseIndex(); - LiveInterval::iterator ULR = IntA.FindLiveRangeContaining(UseIdx); - if (ULR == IntA.end() || ULR->valno != AValNo) - continue; - if (TargetRegisterInfo::isPhysicalRegister(NewReg)) - UseMO.substPhysReg(NewReg, *tri_); - else - UseMO.setReg(NewReg); - if (UseMI == CopyMI) - continue; - if (!UseMI->isCopy()) - continue; - if (UseMI->getOperand(0).getReg() != IntB.reg || - UseMI->getOperand(0).getSubReg()) - continue; - - // This copy will become a noop. If it's defining a new val#, merge it into - // BValNo. - SlotIndex DefIdx = UseIdx.getDefIndex(); - VNInfo *DVNI = IntB.getVNInfoAt(DefIdx); - if (!DVNI) - continue; - DEBUG(dbgs() << "\t\tnoop: " << DefIdx << '\t' << *UseMI); - assert(DVNI->def == DefIdx); - BValNo = IntB.MergeValueNumberInto(BValNo, DVNI); - markAsJoined(UseMI); - } - - // Extend BValNo by merging in IntA live ranges of AValNo. Val# definition - // is updated. - VNInfo *ValNo = BValNo; - ValNo->def = AValNo->def; - ValNo->setCopy(0); - for (LiveInterval::iterator AI = IntA.begin(), AE = IntA.end(); - AI != AE; ++AI) { - if (AI->valno != AValNo) continue; - IntB.addRange(LiveRange(AI->start, AI->end, ValNo)); - } - DEBUG(dbgs() << "\t\textended: " << IntB << '\n'); - - IntA.removeValNo(AValNo); - DEBUG(dbgs() << "\t\ttrimmed: " << IntA << '\n'); - ++numCommutes; - return true; -} - -/// ReMaterializeTrivialDef - If the source of a copy is defined by a trivial -/// computation, replace the copy by rematerialize the definition. -bool SimpleRegisterCoalescing::ReMaterializeTrivialDef(LiveInterval &SrcInt, - bool preserveSrcInt, - unsigned DstReg, - unsigned DstSubIdx, - MachineInstr *CopyMI) { - SlotIndex CopyIdx = li_->getInstructionIndex(CopyMI).getUseIndex(); - LiveInterval::iterator SrcLR = SrcInt.FindLiveRangeContaining(CopyIdx); - assert(SrcLR != SrcInt.end() && "Live range not found!"); - VNInfo *ValNo = SrcLR->valno; - // If other defs can reach uses of this def, then it's not safe to perform - // the optimization. - if (ValNo->isPHIDef() || ValNo->isUnused() || ValNo->hasPHIKill()) - return false; - MachineInstr *DefMI = li_->getInstructionFromIndex(ValNo->def); - if (!DefMI) - return false; - assert(DefMI && "Defining instruction disappeared"); - const TargetInstrDesc &TID = DefMI->getDesc(); - if (!TID.isAsCheapAsAMove()) - return false; - if (!tii_->isTriviallyReMaterializable(DefMI, AA)) - return false; - bool SawStore = false; - if (!DefMI->isSafeToMove(tii_, AA, SawStore)) - return false; - if (TID.getNumDefs() != 1) - return false; - if (!DefMI->isImplicitDef()) { - // Make sure the copy destination register class fits the instruction - // definition register class. The mismatch can happen as a result of earlier - // extract_subreg, insert_subreg, subreg_to_reg coalescing. - const TargetRegisterClass *RC = TID.OpInfo[0].getRegClass(tri_); - if (TargetRegisterInfo::isVirtualRegister(DstReg)) { - if (mri_->getRegClass(DstReg) != RC) - return false; - } else if (!RC->contains(DstReg)) - return false; - } - - // If destination register has a sub-register index on it, make sure it - // matches the instruction register class. - if (DstSubIdx) { - const TargetInstrDesc &TID = DefMI->getDesc(); - if (TID.getNumDefs() != 1) - return false; - const TargetRegisterClass *DstRC = mri_->getRegClass(DstReg); - const TargetRegisterClass *DstSubRC = - DstRC->getSubRegisterRegClass(DstSubIdx); - const TargetRegisterClass *DefRC = TID.OpInfo[0].getRegClass(tri_); - if (DefRC == DstRC) - DstSubIdx = 0; - else if (DefRC != DstSubRC) - return false; - } - - RemoveCopyFlag(DstReg, CopyMI); - - MachineBasicBlock *MBB = CopyMI->getParent(); - MachineBasicBlock::iterator MII = - llvm::next(MachineBasicBlock::iterator(CopyMI)); - tii_->reMaterialize(*MBB, MII, DstReg, DstSubIdx, DefMI, *tri_); - MachineInstr *NewMI = prior(MII); - - // CopyMI may have implicit operands, transfer them over to the newly - // rematerialized instruction. And update implicit def interval valnos. - for (unsigned i = CopyMI->getDesc().getNumOperands(), - e = CopyMI->getNumOperands(); i != e; ++i) { - MachineOperand &MO = CopyMI->getOperand(i); - if (MO.isReg() && MO.isImplicit()) - NewMI->addOperand(MO); - if (MO.isDef()) - RemoveCopyFlag(MO.getReg(), CopyMI); - } - - NewMI->copyImplicitOps(CopyMI); - li_->ReplaceMachineInstrInMaps(CopyMI, NewMI); - CopyMI->eraseFromParent(); - ReMatCopies.insert(CopyMI); - ReMatDefs.insert(DefMI); - DEBUG(dbgs() << "Remat: " << *NewMI); - ++NumReMats; - - // The source interval can become smaller because we removed a use. - if (preserveSrcInt) - li_->shrinkToUses(&SrcInt); - - return true; -} - -/// UpdateRegDefsUses - Replace all defs and uses of SrcReg to DstReg and -/// update the subregister number if it is not zero. If DstReg is a -/// physical register and the existing subregister number of the def / use -/// being updated is not zero, make sure to set it to the correct physical -/// subregister. -void -SimpleRegisterCoalescing::UpdateRegDefsUses(const CoalescerPair &CP) { - bool DstIsPhys = CP.isPhys(); - unsigned SrcReg = CP.getSrcReg(); - unsigned DstReg = CP.getDstReg(); - unsigned SubIdx = CP.getSubIdx(); - - // Update LiveDebugVariables. - ldv_->renameRegister(SrcReg, DstReg, SubIdx); - - for (MachineRegisterInfo::reg_iterator I = mri_->reg_begin(SrcReg); - MachineInstr *UseMI = I.skipInstruction();) { - // A PhysReg copy that won't be coalesced can perhaps be rematerialized - // instead. - if (DstIsPhys) { - if (UseMI->isCopy() && - !UseMI->getOperand(1).getSubReg() && - !UseMI->getOperand(0).getSubReg() && - UseMI->getOperand(1).getReg() == SrcReg && - UseMI->getOperand(0).getReg() != SrcReg && - UseMI->getOperand(0).getReg() != DstReg && - !JoinedCopies.count(UseMI) && - ReMaterializeTrivialDef(li_->getInterval(SrcReg), false, - UseMI->getOperand(0).getReg(), 0, UseMI)) - continue; - } - - SmallVector<unsigned,8> Ops; - bool Reads, Writes; - tie(Reads, Writes) = UseMI->readsWritesVirtualRegister(SrcReg, &Ops); - bool Kills = false, Deads = false; - - // Replace SrcReg with DstReg in all UseMI operands. - for (unsigned i = 0, e = Ops.size(); i != e; ++i) { - MachineOperand &MO = UseMI->getOperand(Ops[i]); - Kills |= MO.isKill(); - Deads |= MO.isDead(); - - if (DstIsPhys) - MO.substPhysReg(DstReg, *tri_); - else - MO.substVirtReg(DstReg, SubIdx, *tri_); - } - - // This instruction is a copy that will be removed. - if (JoinedCopies.count(UseMI)) - continue; - - if (SubIdx) { - // If UseMI was a simple SrcReg def, make sure we didn't turn it into a - // read-modify-write of DstReg. - if (Deads) - UseMI->addRegisterDead(DstReg, tri_); - else if (!Reads && Writes) - UseMI->addRegisterDefined(DstReg, tri_); - - // Kill flags apply to the whole physical register. - if (DstIsPhys && Kills) - UseMI->addRegisterKilled(DstReg, tri_); - } - - DEBUG({ - dbgs() << "\t\tupdated: "; - if (!UseMI->isDebugValue()) - dbgs() << li_->getInstructionIndex(UseMI) << "\t"; - dbgs() << *UseMI; - }); - } -} - -/// removeIntervalIfEmpty - Check if the live interval of a physical register -/// is empty, if so remove it and also remove the empty intervals of its -/// sub-registers. Return true if live interval is removed. -static bool removeIntervalIfEmpty(LiveInterval &li, LiveIntervals *li_, - const TargetRegisterInfo *tri_) { - if (li.empty()) { - if (TargetRegisterInfo::isPhysicalRegister(li.reg)) - for (const unsigned* SR = tri_->getSubRegisters(li.reg); *SR; ++SR) { - if (!li_->hasInterval(*SR)) - continue; - LiveInterval &sli = li_->getInterval(*SR); - if (sli.empty()) - li_->removeInterval(*SR); - } - li_->removeInterval(li.reg); - return true; - } - return false; -} - -/// RemoveDeadDef - If a def of a live interval is now determined dead, remove -/// the val# it defines. If the live interval becomes empty, remove it as well. -bool SimpleRegisterCoalescing::RemoveDeadDef(LiveInterval &li, - MachineInstr *DefMI) { - SlotIndex DefIdx = li_->getInstructionIndex(DefMI).getDefIndex(); - LiveInterval::iterator MLR = li.FindLiveRangeContaining(DefIdx); - if (DefIdx != MLR->valno->def) - return false; - li.removeValNo(MLR->valno); - return removeIntervalIfEmpty(li, li_, tri_); -} - -void SimpleRegisterCoalescing::RemoveCopyFlag(unsigned DstReg, - const MachineInstr *CopyMI) { - SlotIndex DefIdx = li_->getInstructionIndex(CopyMI).getDefIndex(); - if (li_->hasInterval(DstReg)) { - LiveInterval &LI = li_->getInterval(DstReg); - if (const LiveRange *LR = LI.getLiveRangeContaining(DefIdx)) - if (LR->valno->def == DefIdx) - LR->valno->setCopy(0); - } - if (!TargetRegisterInfo::isPhysicalRegister(DstReg)) - return; - for (const unsigned* AS = tri_->getAliasSet(DstReg); *AS; ++AS) { - if (!li_->hasInterval(*AS)) - continue; - LiveInterval &LI = li_->getInterval(*AS); - if (const LiveRange *LR = LI.getLiveRangeContaining(DefIdx)) - if (LR->valno->def == DefIdx) - LR->valno->setCopy(0); - } -} - -/// shouldJoinPhys - Return true if a copy involving a physreg should be joined. -/// We need to be careful about coalescing a source physical register with a -/// virtual register. Once the coalescing is done, it cannot be broken and these -/// are not spillable! If the destination interval uses are far away, think -/// twice about coalescing them! -bool SimpleRegisterCoalescing::shouldJoinPhys(CoalescerPair &CP) { - bool Allocatable = li_->isAllocatable(CP.getDstReg()); - LiveInterval &JoinVInt = li_->getInterval(CP.getSrcReg()); - - /// Always join simple intervals that are defined by a single copy from a - /// reserved register. This doesn't increase register pressure, so it is - /// always beneficial. - if (!Allocatable && CP.isFlipped() && JoinVInt.containsOneValue()) - return true; - - if (!EnablePhysicalJoin) { - DEBUG(dbgs() << "\tPhysreg joins disabled.\n"); - return false; - } - - // Only coalesce to allocatable physreg, we don't want to risk modifying - // reserved registers. - if (!Allocatable) { - DEBUG(dbgs() << "\tRegister is an unallocatable physreg.\n"); - return false; // Not coalescable. - } - - // Don't join with physregs that have a ridiculous number of live - // ranges. The data structure performance is really bad when that - // happens. - if (li_->hasInterval(CP.getDstReg()) && - li_->getInterval(CP.getDstReg()).ranges.size() > 1000) { - ++numAborts; - DEBUG(dbgs() - << "\tPhysical register live interval too complicated, abort!\n"); - return false; - } - - // FIXME: Why are we skipping this test for partial copies? - // CodeGen/X86/phys_subreg_coalesce-3.ll needs it. - if (!CP.isPartial()) { - const TargetRegisterClass *RC = mri_->getRegClass(CP.getSrcReg()); - unsigned Threshold = RegClassInfo.getNumAllocatableRegs(RC) * 2; - unsigned Length = li_->getApproximateInstructionCount(JoinVInt); - if (Length > Threshold) { - ++numAborts; - DEBUG(dbgs() << "\tMay tie down a physical register, abort!\n"); - return false; - } - } - return true; -} - -/// isWinToJoinCrossClass - Return true if it's profitable to coalesce -/// two virtual registers from different register classes. -bool -SimpleRegisterCoalescing::isWinToJoinCrossClass(unsigned SrcReg, - unsigned DstReg, - const TargetRegisterClass *SrcRC, - const TargetRegisterClass *DstRC, - const TargetRegisterClass *NewRC) { - unsigned NewRCCount = RegClassInfo.getNumAllocatableRegs(NewRC); - // This heuristics is good enough in practice, but it's obviously not *right*. - // 4 is a magic number that works well enough for x86, ARM, etc. It filter - // out all but the most restrictive register classes. - if (NewRCCount > 4 || - // Early exit if the function is fairly small, coalesce aggressively if - // that's the case. For really special register classes with 3 or - // fewer registers, be a bit more careful. - (li_->getFuncInstructionCount() / NewRCCount) < 8) - return true; - LiveInterval &SrcInt = li_->getInterval(SrcReg); - LiveInterval &DstInt = li_->getInterval(DstReg); - unsigned SrcSize = li_->getApproximateInstructionCount(SrcInt); - unsigned DstSize = li_->getApproximateInstructionCount(DstInt); - - // Coalesce aggressively if the intervals are small compared to the number of - // registers in the new class. The number 4 is fairly arbitrary, chosen to be - // less aggressive than the 8 used for the whole function size. - const unsigned ThresSize = 4 * NewRCCount; - if (SrcSize <= ThresSize && DstSize <= ThresSize) - return true; - - // Estimate *register use density*. If it doubles or more, abort. - unsigned SrcUses = std::distance(mri_->use_nodbg_begin(SrcReg), - mri_->use_nodbg_end()); - unsigned DstUses = std::distance(mri_->use_nodbg_begin(DstReg), - mri_->use_nodbg_end()); - unsigned NewUses = SrcUses + DstUses; - unsigned NewSize = SrcSize + DstSize; - if (SrcRC != NewRC && SrcSize > ThresSize) { - unsigned SrcRCCount = RegClassInfo.getNumAllocatableRegs(SrcRC); - if (NewUses*SrcSize*SrcRCCount > 2*SrcUses*NewSize*NewRCCount) - return false; - } - if (DstRC != NewRC && DstSize > ThresSize) { - unsigned DstRCCount = RegClassInfo.getNumAllocatableRegs(DstRC); - if (NewUses*DstSize*DstRCCount > 2*DstUses*NewSize*NewRCCount) - return false; - } - return true; -} - - -/// JoinCopy - Attempt to join intervals corresponding to SrcReg/DstReg, -/// which are the src/dst of the copy instruction CopyMI. This returns true -/// if the copy was successfully coalesced away. If it is not currently -/// possible to coalesce this interval, but it may be possible if other -/// things get coalesced, then it returns true by reference in 'Again'. -bool SimpleRegisterCoalescing::JoinCopy(CopyRec &TheCopy, bool &Again) { - MachineInstr *CopyMI = TheCopy.MI; - - Again = false; - if (JoinedCopies.count(CopyMI) || ReMatCopies.count(CopyMI)) - return false; // Already done. - - DEBUG(dbgs() << li_->getInstructionIndex(CopyMI) << '\t' << *CopyMI); - - CoalescerPair CP(*tii_, *tri_); - if (!CP.setRegisters(CopyMI)) { - DEBUG(dbgs() << "\tNot coalescable.\n"); - return false; - } - - // If they are already joined we continue. - if (CP.getSrcReg() == CP.getDstReg()) { - markAsJoined(CopyMI); - DEBUG(dbgs() << "\tCopy already coalesced.\n"); - return false; // Not coalescable. - } - - DEBUG(dbgs() << "\tConsidering merging " << PrintReg(CP.getSrcReg(), tri_) - << " with " << PrintReg(CP.getDstReg(), tri_, CP.getSubIdx()) - << "\n"); - - // Enforce policies. - if (CP.isPhys()) { - if (!shouldJoinPhys(CP)) { - // Before giving up coalescing, if definition of source is defined by - // trivial computation, try rematerializing it. - if (!CP.isFlipped() && - ReMaterializeTrivialDef(li_->getInterval(CP.getSrcReg()), true, - CP.getDstReg(), 0, CopyMI)) - return true; - return false; - } - } else { - // Avoid constraining virtual register regclass too much. - if (CP.isCrossClass()) { - DEBUG(dbgs() << "\tCross-class to " << CP.getNewRC()->getName() << ".\n"); - if (DisableCrossClassJoin) { - DEBUG(dbgs() << "\tCross-class joins disabled.\n"); - return false; - } - if (!isWinToJoinCrossClass(CP.getSrcReg(), CP.getDstReg(), - mri_->getRegClass(CP.getSrcReg()), - mri_->getRegClass(CP.getDstReg()), - CP.getNewRC())) { - DEBUG(dbgs() << "\tAvoid coalescing to constrained register class.\n"); - Again = true; // May be possible to coalesce later. - return false; - } - } - - // When possible, let DstReg be the larger interval. - if (!CP.getSubIdx() && li_->getInterval(CP.getSrcReg()).ranges.size() > - li_->getInterval(CP.getDstReg()).ranges.size()) - CP.flip(); - } - - // Okay, attempt to join these two intervals. On failure, this returns false. - // Otherwise, if one of the intervals being joined is a physreg, this method - // always canonicalizes DstInt to be it. The output "SrcInt" will not have - // been modified, so we can use this information below to update aliases. - if (!JoinIntervals(CP)) { - // Coalescing failed. - - // If definition of source is defined by trivial computation, try - // rematerializing it. - if (!CP.isFlipped() && - ReMaterializeTrivialDef(li_->getInterval(CP.getSrcReg()), true, - CP.getDstReg(), 0, CopyMI)) - return true; - - // If we can eliminate the copy without merging the live ranges, do so now. - if (!CP.isPartial()) { - if (AdjustCopiesBackFrom(CP, CopyMI) || - RemoveCopyByCommutingDef(CP, CopyMI)) { - markAsJoined(CopyMI); - DEBUG(dbgs() << "\tTrivial!\n"); - return true; - } - } - - // Otherwise, we are unable to join the intervals. - DEBUG(dbgs() << "\tInterference!\n"); - Again = true; // May be possible to coalesce later. - return false; - } - - // Coalescing to a virtual register that is of a sub-register class of the - // other. Make sure the resulting register is set to the right register class. - if (CP.isCrossClass()) { - ++numCrossRCs; - mri_->setRegClass(CP.getDstReg(), CP.getNewRC()); - } - - // Remember to delete the copy instruction. - markAsJoined(CopyMI); - - UpdateRegDefsUses(CP); - - // If we have extended the live range of a physical register, make sure we - // update live-in lists as well. - if (CP.isPhys()) { - SmallVector<MachineBasicBlock*, 16> BlockSeq; - // JoinIntervals invalidates the VNInfos in SrcInt, but we only need the - // ranges for this, and they are preserved. - LiveInterval &SrcInt = li_->getInterval(CP.getSrcReg()); - for (LiveInterval::const_iterator I = SrcInt.begin(), E = SrcInt.end(); - I != E; ++I ) { - li_->findLiveInMBBs(I->start, I->end, BlockSeq); - for (unsigned idx = 0, size = BlockSeq.size(); idx != size; ++idx) { - MachineBasicBlock &block = *BlockSeq[idx]; - if (!block.isLiveIn(CP.getDstReg())) - block.addLiveIn(CP.getDstReg()); - } - BlockSeq.clear(); - } - } - - // SrcReg is guarateed to be the register whose live interval that is - // being merged. - li_->removeInterval(CP.getSrcReg()); - - // Update regalloc hint. - tri_->UpdateRegAllocHint(CP.getSrcReg(), CP.getDstReg(), *mf_); - - DEBUG({ - LiveInterval &DstInt = li_->getInterval(CP.getDstReg()); - dbgs() << "\tJoined. Result = "; - DstInt.print(dbgs(), tri_); - dbgs() << "\n"; - }); - - ++numJoins; - return true; -} - -/// ComputeUltimateVN - Assuming we are going to join two live intervals, -/// compute what the resultant value numbers for each value in the input two -/// ranges will be. This is complicated by copies between the two which can -/// and will commonly cause multiple value numbers to be merged into one. -/// -/// VN is the value number that we're trying to resolve. InstDefiningValue -/// keeps track of the new InstDefiningValue assignment for the result -/// LiveInterval. ThisFromOther/OtherFromThis are sets that keep track of -/// whether a value in this or other is a copy from the opposite set. -/// ThisValNoAssignments/OtherValNoAssignments keep track of value #'s that have -/// already been assigned. -/// -/// ThisFromOther[x] - If x is defined as a copy from the other interval, this -/// contains the value number the copy is from. -/// -static unsigned ComputeUltimateVN(VNInfo *VNI, - SmallVector<VNInfo*, 16> &NewVNInfo, - DenseMap<VNInfo*, VNInfo*> &ThisFromOther, - DenseMap<VNInfo*, VNInfo*> &OtherFromThis, - SmallVector<int, 16> &ThisValNoAssignments, - SmallVector<int, 16> &OtherValNoAssignments) { - unsigned VN = VNI->id; - - // If the VN has already been computed, just return it. - if (ThisValNoAssignments[VN] >= 0) - return ThisValNoAssignments[VN]; - assert(ThisValNoAssignments[VN] != -2 && "Cyclic value numbers"); - - // If this val is not a copy from the other val, then it must be a new value - // number in the destination. - DenseMap<VNInfo*, VNInfo*>::iterator I = ThisFromOther.find(VNI); - if (I == ThisFromOther.end()) { - NewVNInfo.push_back(VNI); - return ThisValNoAssignments[VN] = NewVNInfo.size()-1; - } - VNInfo *OtherValNo = I->second; - - // Otherwise, this *is* a copy from the RHS. If the other side has already - // been computed, return it. - if (OtherValNoAssignments[OtherValNo->id] >= 0) - return ThisValNoAssignments[VN] = OtherValNoAssignments[OtherValNo->id]; - - // Mark this value number as currently being computed, then ask what the - // ultimate value # of the other value is. - ThisValNoAssignments[VN] = -2; - unsigned UltimateVN = - ComputeUltimateVN(OtherValNo, NewVNInfo, OtherFromThis, ThisFromOther, - OtherValNoAssignments, ThisValNoAssignments); - return ThisValNoAssignments[VN] = UltimateVN; -} - -/// JoinIntervals - Attempt to join these two intervals. On failure, this -/// returns false. -bool SimpleRegisterCoalescing::JoinIntervals(CoalescerPair &CP) { - LiveInterval &RHS = li_->getInterval(CP.getSrcReg()); - DEBUG({ dbgs() << "\t\tRHS = "; RHS.print(dbgs(), tri_); dbgs() << "\n"; }); - - // If a live interval is a physical register, check for interference with any - // aliases. The interference check implemented here is a bit more conservative - // than the full interfeence check below. We allow overlapping live ranges - // only when one is a copy of the other. - if (CP.isPhys()) { - for (const unsigned *AS = tri_->getAliasSet(CP.getDstReg()); *AS; ++AS){ - if (!li_->hasInterval(*AS)) - continue; - const LiveInterval &LHS = li_->getInterval(*AS); - LiveInterval::const_iterator LI = LHS.begin(); - for (LiveInterval::const_iterator RI = RHS.begin(), RE = RHS.end(); - RI != RE; ++RI) { - LI = std::lower_bound(LI, LHS.end(), RI->start); - // Does LHS have an overlapping live range starting before RI? - if ((LI != LHS.begin() && LI[-1].end > RI->start) && - (RI->start != RI->valno->def || - !CP.isCoalescable(li_->getInstructionFromIndex(RI->start)))) { - DEBUG({ - dbgs() << "\t\tInterference from alias: "; - LHS.print(dbgs(), tri_); - dbgs() << "\n\t\tOverlap at " << RI->start << " and no copy.\n"; - }); - return false; - } - - // Check that LHS ranges beginning in this range are copies. - for (; LI != LHS.end() && LI->start < RI->end; ++LI) { - if (LI->start != LI->valno->def || - !CP.isCoalescable(li_->getInstructionFromIndex(LI->start))) { - DEBUG({ - dbgs() << "\t\tInterference from alias: "; - LHS.print(dbgs(), tri_); - dbgs() << "\n\t\tDef at " << LI->start << " is not a copy.\n"; - }); - return false; - } - } - } - } - } - - // Compute the final value assignment, assuming that the live ranges can be - // coalesced. - SmallVector<int, 16> LHSValNoAssignments; - SmallVector<int, 16> RHSValNoAssignments; - DenseMap<VNInfo*, VNInfo*> LHSValsDefinedFromRHS; - DenseMap<VNInfo*, VNInfo*> RHSValsDefinedFromLHS; - SmallVector<VNInfo*, 16> NewVNInfo; - - LiveInterval &LHS = li_->getOrCreateInterval(CP.getDstReg()); - DEBUG({ dbgs() << "\t\tLHS = "; LHS.print(dbgs(), tri_); dbgs() << "\n"; }); - - // Loop over the value numbers of the LHS, seeing if any are defined from - // the RHS. - for (LiveInterval::vni_iterator i = LHS.vni_begin(), e = LHS.vni_end(); - i != e; ++i) { - VNInfo *VNI = *i; - if (VNI->isUnused() || !VNI->isDefByCopy()) // Src not defined by a copy? - continue; - - // Never join with a register that has EarlyClobber redefs. - if (VNI->hasRedefByEC()) - return false; - - // DstReg is known to be a register in the LHS interval. If the src is - // from the RHS interval, we can use its value #. - if (!CP.isCoalescable(VNI->getCopy())) - continue; - - // Figure out the value # from the RHS. - LiveRange *lr = RHS.getLiveRangeContaining(VNI->def.getPrevSlot()); - // The copy could be to an aliased physreg. - if (!lr) continue; - LHSValsDefinedFromRHS[VNI] = lr->valno; - } - - // Loop over the value numbers of the RHS, seeing if any are defined from - // the LHS. - for (LiveInterval::vni_iterator i = RHS.vni_begin(), e = RHS.vni_end(); - i != e; ++i) { - VNInfo *VNI = *i; - if (VNI->isUnused() || !VNI->isDefByCopy()) // Src not defined by a copy? - continue; - - // Never join with a register that has EarlyClobber redefs. - if (VNI->hasRedefByEC()) - return false; - - // DstReg is known to be a register in the RHS interval. If the src is - // from the LHS interval, we can use its value #. - if (!CP.isCoalescable(VNI->getCopy())) - continue; - - // Figure out the value # from the LHS. - LiveRange *lr = LHS.getLiveRangeContaining(VNI->def.getPrevSlot()); - // The copy could be to an aliased physreg. - if (!lr) continue; - RHSValsDefinedFromLHS[VNI] = lr->valno; - } - - LHSValNoAssignments.resize(LHS.getNumValNums(), -1); - RHSValNoAssignments.resize(RHS.getNumValNums(), -1); - NewVNInfo.reserve(LHS.getNumValNums() + RHS.getNumValNums()); - - for (LiveInterval::vni_iterator i = LHS.vni_begin(), e = LHS.vni_end(); - i != e; ++i) { - VNInfo *VNI = *i; - unsigned VN = VNI->id; - if (LHSValNoAssignments[VN] >= 0 || VNI->isUnused()) - continue; - ComputeUltimateVN(VNI, NewVNInfo, - LHSValsDefinedFromRHS, RHSValsDefinedFromLHS, - LHSValNoAssignments, RHSValNoAssignments); - } - for (LiveInterval::vni_iterator i = RHS.vni_begin(), e = RHS.vni_end(); - i != e; ++i) { - VNInfo *VNI = *i; - unsigned VN = VNI->id; - if (RHSValNoAssignments[VN] >= 0 || VNI->isUnused()) - continue; - // If this value number isn't a copy from the LHS, it's a new number. - if (RHSValsDefinedFromLHS.find(VNI) == RHSValsDefinedFromLHS.end()) { - NewVNInfo.push_back(VNI); - RHSValNoAssignments[VN] = NewVNInfo.size()-1; - continue; - } - - ComputeUltimateVN(VNI, NewVNInfo, - RHSValsDefinedFromLHS, LHSValsDefinedFromRHS, - RHSValNoAssignments, LHSValNoAssignments); - } - - // Armed with the mappings of LHS/RHS values to ultimate values, walk the - // interval lists to see if these intervals are coalescable. - LiveInterval::const_iterator I = LHS.begin(); - LiveInterval::const_iterator IE = LHS.end(); - LiveInterval::const_iterator J = RHS.begin(); - LiveInterval::const_iterator JE = RHS.end(); - - // Skip ahead until the first place of potential sharing. - if (I != IE && J != JE) { - if (I->start < J->start) { - I = std::upper_bound(I, IE, J->start); - if (I != LHS.begin()) --I; - } else if (J->start < I->start) { - J = std::upper_bound(J, JE, I->start); - if (J != RHS.begin()) --J; - } - } - - while (I != IE && J != JE) { - // Determine if these two live ranges overlap. - bool Overlaps; - if (I->start < J->start) { - Overlaps = I->end > J->start; - } else { - Overlaps = J->end > I->start; - } - - // If so, check value # info to determine if they are really different. - if (Overlaps) { - // If the live range overlap will map to the same value number in the - // result liverange, we can still coalesce them. If not, we can't. - if (LHSValNoAssignments[I->valno->id] != - RHSValNoAssignments[J->valno->id]) - return false; - // If it's re-defined by an early clobber somewhere in the live range, - // then conservatively abort coalescing. - if (NewVNInfo[LHSValNoAssignments[I->valno->id]]->hasRedefByEC()) - return false; - } - - if (I->end < J->end) - ++I; - else - ++J; - } - - // Update kill info. Some live ranges are extended due to copy coalescing. - for (DenseMap<VNInfo*, VNInfo*>::iterator I = LHSValsDefinedFromRHS.begin(), - E = LHSValsDefinedFromRHS.end(); I != E; ++I) { - VNInfo *VNI = I->first; - unsigned LHSValID = LHSValNoAssignments[VNI->id]; - if (VNI->hasPHIKill()) - NewVNInfo[LHSValID]->setHasPHIKill(true); - } - - // Update kill info. Some live ranges are extended due to copy coalescing. - for (DenseMap<VNInfo*, VNInfo*>::iterator I = RHSValsDefinedFromLHS.begin(), - E = RHSValsDefinedFromLHS.end(); I != E; ++I) { - VNInfo *VNI = I->first; - unsigned RHSValID = RHSValNoAssignments[VNI->id]; - if (VNI->hasPHIKill()) - NewVNInfo[RHSValID]->setHasPHIKill(true); - } - - if (LHSValNoAssignments.empty()) - LHSValNoAssignments.push_back(-1); - if (RHSValNoAssignments.empty()) - RHSValNoAssignments.push_back(-1); - - // If we get here, we know that we can coalesce the live ranges. Ask the - // intervals to coalesce themselves now. - LHS.join(RHS, &LHSValNoAssignments[0], &RHSValNoAssignments[0], NewVNInfo, - mri_); - return true; -} - -namespace { - // DepthMBBCompare - Comparison predicate that sort first based on the loop - // depth of the basic block (the unsigned), and then on the MBB number. - struct DepthMBBCompare { - typedef std::pair<unsigned, MachineBasicBlock*> DepthMBBPair; - bool operator()(const DepthMBBPair &LHS, const DepthMBBPair &RHS) const { - // Deeper loops first - if (LHS.first != RHS.first) - return LHS.first > RHS.first; - - // Prefer blocks that are more connected in the CFG. This takes care of - // the most difficult copies first while intervals are short. - unsigned cl = LHS.second->pred_size() + LHS.second->succ_size(); - unsigned cr = RHS.second->pred_size() + RHS.second->succ_size(); - if (cl != cr) - return cl > cr; - - // As a last resort, sort by block number. - return LHS.second->getNumber() < RHS.second->getNumber(); - } - }; -} - -void SimpleRegisterCoalescing::CopyCoalesceInMBB(MachineBasicBlock *MBB, - std::vector<CopyRec> &TryAgain) { - DEBUG(dbgs() << MBB->getName() << ":\n"); - - SmallVector<CopyRec, 8> VirtCopies; - SmallVector<CopyRec, 8> PhysCopies; - SmallVector<CopyRec, 8> ImpDefCopies; - for (MachineBasicBlock::iterator MII = MBB->begin(), E = MBB->end(); - MII != E;) { - MachineInstr *Inst = MII++; - - // If this isn't a copy nor a extract_subreg, we can't join intervals. - unsigned SrcReg, DstReg; - if (Inst->isCopy()) { - DstReg = Inst->getOperand(0).getReg(); - SrcReg = Inst->getOperand(1).getReg(); - } else if (Inst->isSubregToReg()) { - DstReg = Inst->getOperand(0).getReg(); - SrcReg = Inst->getOperand(2).getReg(); - } else - continue; - - bool SrcIsPhys = TargetRegisterInfo::isPhysicalRegister(SrcReg); - bool DstIsPhys = TargetRegisterInfo::isPhysicalRegister(DstReg); - if (li_->hasInterval(SrcReg) && li_->getInterval(SrcReg).empty()) - ImpDefCopies.push_back(CopyRec(Inst, 0)); - else if (SrcIsPhys || DstIsPhys) - PhysCopies.push_back(CopyRec(Inst, 0)); - else - VirtCopies.push_back(CopyRec(Inst, 0)); - } - - // Try coalescing implicit copies and insert_subreg <undef> first, - // followed by copies to / from physical registers, then finally copies - // from virtual registers to virtual registers. - for (unsigned i = 0, e = ImpDefCopies.size(); i != e; ++i) { - CopyRec &TheCopy = ImpDefCopies[i]; - bool Again = false; - if (!JoinCopy(TheCopy, Again)) - if (Again) - TryAgain.push_back(TheCopy); - } - for (unsigned i = 0, e = PhysCopies.size(); i != e; ++i) { - CopyRec &TheCopy = PhysCopies[i]; - bool Again = false; - if (!JoinCopy(TheCopy, Again)) - if (Again) - TryAgain.push_back(TheCopy); - } - for (unsigned i = 0, e = VirtCopies.size(); i != e; ++i) { - CopyRec &TheCopy = VirtCopies[i]; - bool Again = false; - if (!JoinCopy(TheCopy, Again)) - if (Again) - TryAgain.push_back(TheCopy); - } -} - -void SimpleRegisterCoalescing::joinIntervals() { - DEBUG(dbgs() << "********** JOINING INTERVALS ***********\n"); - - std::vector<CopyRec> TryAgainList; - if (loopInfo->empty()) { - // If there are no loops in the function, join intervals in function order. - for (MachineFunction::iterator I = mf_->begin(), E = mf_->end(); - I != E; ++I) - CopyCoalesceInMBB(I, TryAgainList); - } else { - // Otherwise, join intervals in inner loops before other intervals. - // Unfortunately we can't just iterate over loop hierarchy here because - // there may be more MBB's than BB's. Collect MBB's for sorting. - - // Join intervals in the function prolog first. We want to join physical - // registers with virtual registers before the intervals got too long. - std::vector<std::pair<unsigned, MachineBasicBlock*> > MBBs; - for (MachineFunction::iterator I = mf_->begin(), E = mf_->end();I != E;++I){ - MachineBasicBlock *MBB = I; - MBBs.push_back(std::make_pair(loopInfo->getLoopDepth(MBB), I)); - } - - // Sort by loop depth. - std::sort(MBBs.begin(), MBBs.end(), DepthMBBCompare()); - - // Finally, join intervals in loop nest order. - for (unsigned i = 0, e = MBBs.size(); i != e; ++i) - CopyCoalesceInMBB(MBBs[i].second, TryAgainList); - } - - // Joining intervals can allow other intervals to be joined. Iteratively join - // until we make no progress. - bool ProgressMade = true; - while (ProgressMade) { - ProgressMade = false; - - for (unsigned i = 0, e = TryAgainList.size(); i != e; ++i) { - CopyRec &TheCopy = TryAgainList[i]; - if (!TheCopy.MI) - continue; - - bool Again = false; - bool Success = JoinCopy(TheCopy, Again); - if (Success || !Again) { - TheCopy.MI = 0; // Mark this one as done. - ProgressMade = true; - } - } - } -} - -void SimpleRegisterCoalescing::releaseMemory() { - JoinedCopies.clear(); - ReMatCopies.clear(); - ReMatDefs.clear(); -} - -bool SimpleRegisterCoalescing::runOnMachineFunction(MachineFunction &fn) { - mf_ = &fn; - mri_ = &fn.getRegInfo(); - tm_ = &fn.getTarget(); - tri_ = tm_->getRegisterInfo(); - tii_ = tm_->getInstrInfo(); - li_ = &getAnalysis<LiveIntervals>(); - ldv_ = &getAnalysis<LiveDebugVariables>(); - AA = &getAnalysis<AliasAnalysis>(); - loopInfo = &getAnalysis<MachineLoopInfo>(); - - DEBUG(dbgs() << "********** SIMPLE REGISTER COALESCING **********\n" - << "********** Function: " - << ((Value*)mf_->getFunction())->getName() << '\n'); - - if (VerifyCoalescing) - mf_->verify(this, "Before register coalescing"); - - RegClassInfo.runOnMachineFunction(fn); - - // Join (coalesce) intervals if requested. - if (EnableJoining) { - joinIntervals(); - DEBUG({ - dbgs() << "********** INTERVALS POST JOINING **********\n"; - for (LiveIntervals::iterator I = li_->begin(), E = li_->end(); - I != E; ++I){ - I->second->print(dbgs(), tri_); - dbgs() << "\n"; - } - }); - } - - // Perform a final pass over the instructions and compute spill weights - // and remove identity moves. - SmallVector<unsigned, 4> DeadDefs; - for (MachineFunction::iterator mbbi = mf_->begin(), mbbe = mf_->end(); - mbbi != mbbe; ++mbbi) { - MachineBasicBlock* mbb = mbbi; - for (MachineBasicBlock::iterator mii = mbb->begin(), mie = mbb->end(); - mii != mie; ) { - MachineInstr *MI = mii; - if (JoinedCopies.count(MI)) { - // Delete all coalesced copies. - bool DoDelete = true; - assert(MI->isCopyLike() && "Unrecognized copy instruction"); - unsigned SrcReg = MI->getOperand(MI->isSubregToReg() ? 2 : 1).getReg(); - if (TargetRegisterInfo::isPhysicalRegister(SrcReg) && - MI->getNumOperands() > 2) - // Do not delete extract_subreg, insert_subreg of physical - // registers unless the definition is dead. e.g. - // %DO<def> = INSERT_SUBREG %D0<undef>, %S0<kill>, 1 - // or else the scavenger may complain. LowerSubregs will - // delete them later. - DoDelete = false; - - if (MI->allDefsAreDead()) { - if (TargetRegisterInfo::isVirtualRegister(SrcReg) && - li_->hasInterval(SrcReg)) - li_->shrinkToUses(&li_->getInterval(SrcReg)); - DoDelete = true; - } - if (!DoDelete) { - // We need the instruction to adjust liveness, so make it a KILL. - if (MI->isSubregToReg()) { - MI->RemoveOperand(3); - MI->RemoveOperand(1); - } - MI->setDesc(tii_->get(TargetOpcode::KILL)); - mii = llvm::next(mii); - } else { - li_->RemoveMachineInstrFromMaps(MI); - mii = mbbi->erase(mii); - ++numPeep; - } - continue; - } - - // Now check if this is a remat'ed def instruction which is now dead. - if (ReMatDefs.count(MI)) { - bool isDead = true; - for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { - const MachineOperand &MO = MI->getOperand(i); - if (!MO.isReg()) - continue; - unsigned Reg = MO.getReg(); - if (!Reg) - continue; - if (TargetRegisterInfo::isVirtualRegister(Reg)) - DeadDefs.push_back(Reg); - if (MO.isDead()) - continue; - if (TargetRegisterInfo::isPhysicalRegister(Reg) || - !mri_->use_nodbg_empty(Reg)) { - isDead = false; - break; - } - } - if (isDead) { - while (!DeadDefs.empty()) { - unsigned DeadDef = DeadDefs.back(); - DeadDefs.pop_back(); - RemoveDeadDef(li_->getInterval(DeadDef), MI); - } - li_->RemoveMachineInstrFromMaps(mii); - mii = mbbi->erase(mii); - continue; - } else - DeadDefs.clear(); - } - - ++mii; - - // Check for now unnecessary kill flags. - if (li_->isNotInMIMap(MI)) continue; - SlotIndex DefIdx = li_->getInstructionIndex(MI).getDefIndex(); - for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { - MachineOperand &MO = MI->getOperand(i); - if (!MO.isReg() || !MO.isKill()) continue; - unsigned reg = MO.getReg(); - if (!reg || !li_->hasInterval(reg)) continue; - if (!li_->getInterval(reg).killedAt(DefIdx)) { - MO.setIsKill(false); - continue; - } - // When leaving a kill flag on a physreg, check if any subregs should - // remain alive. - if (!TargetRegisterInfo::isPhysicalRegister(reg)) - continue; - for (const unsigned *SR = tri_->getSubRegisters(reg); - unsigned S = *SR; ++SR) - if (li_->hasInterval(S) && li_->getInterval(S).liveAt(DefIdx)) - MI->addRegisterDefined(S, tri_); - } - } - } - - DEBUG(dump()); - DEBUG(ldv_->dump()); - if (VerifyCoalescing) - mf_->verify(this, "After register coalescing"); - return true; -} - -/// print - Implement the dump method. -void SimpleRegisterCoalescing::print(raw_ostream &O, const Module* m) const { - li_->print(O, m); -} - -RegisterCoalescer* llvm::createSimpleRegisterCoalescer() { - return new SimpleRegisterCoalescing(); -} - -// Make sure that anything that uses RegisterCoalescer pulls in this file... -DEFINING_FILE_FOR(SimpleRegisterCoalescing) diff --git a/lib/CodeGen/SplitKit.cpp b/lib/CodeGen/SplitKit.cpp index bf27cc8..a0952a0 100644 --- a/lib/CodeGen/SplitKit.cpp +++ b/lib/CodeGen/SplitKit.cpp @@ -76,12 +76,14 @@ SlotIndex SplitAnalysis::computeLastSplitPoint(unsigned Num) { return LSP.first; // There may not be a call instruction (?) in which case we ignore LPad. LSP.second = LSP.first; - for (MachineBasicBlock::const_iterator I = FirstTerm, E = MBB->begin(); - I != E; --I) + for (MachineBasicBlock::const_iterator I = MBB->end(), E = MBB->begin(); + I != E;) { + --I; if (I->getDesc().isCall()) { LSP.second = LIS.getInstructionIndex(I); break; } + } } // If CurLI is live into a landing pad successor, move the last split point @@ -122,7 +124,7 @@ void SplitAnalysis::analyzeUses() { // Compute per-live block info. if (!calcLiveBlockInfo()) { // FIXME: calcLiveBlockInfo found inconsistencies in the live range. - // I am looking at you, SimpleRegisterCoalescing! + // I am looking at you, RegisterCoalescer! DidRepairRange = true; ++NumRepairs; DEBUG(dbgs() << "*** Fixing inconsistent live interval! ***\n"); @@ -165,7 +167,7 @@ bool SplitAnalysis::calcLiveBlockInfo() { tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(BI.MBB); // If the block contains no uses, the range must be live through. At one - // point, SimpleRegisterCoalescing could create dangling ranges that ended + // point, RegisterCoalescer could create dangling ranges that ended // mid-block. if (UseI == UseE || *UseI >= Stop) { ++NumThroughBlocks; @@ -634,6 +636,7 @@ unsigned SplitEditor::openIntv() { void SplitEditor::selectIntv(unsigned Idx) { assert(Idx != 0 && "Cannot select the complement interval"); assert(Idx < Edit->size() && "Can only select previously opened interval"); + DEBUG(dbgs() << " selectIntv " << OpenIdx << " -> " << Idx << '\n'); OpenIdx = Idx; } @@ -654,6 +657,24 @@ SlotIndex SplitEditor::enterIntvBefore(SlotIndex Idx) { return VNI->def; } +SlotIndex SplitEditor::enterIntvAfter(SlotIndex Idx) { + assert(OpenIdx && "openIntv not called before enterIntvAfter"); + DEBUG(dbgs() << " enterIntvAfter " << Idx); + Idx = Idx.getBoundaryIndex(); + VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx); + if (!ParentVNI) { + DEBUG(dbgs() << ": not live\n"); + return Idx; + } + DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n'); + MachineInstr *MI = LIS.getInstructionFromIndex(Idx); + assert(MI && "enterIntvAfter called with invalid index"); + + VNInfo *VNI = defFromParent(OpenIdx, ParentVNI, Idx, *MI->getParent(), + llvm::next(MachineBasicBlock::iterator(MI))); + return VNI->def; +} + SlotIndex SplitEditor::enterIntvAtEnd(MachineBasicBlock &MBB) { assert(OpenIdx && "openIntv not called before enterIntvAtEnd"); SlotIndex End = LIS.getMBBEndIdx(&MBB); @@ -1005,12 +1026,6 @@ void SplitEditor::finish(SmallVectorImpl<unsigned> *LRMap) { markComplexMapped(i, ParentVNI); } -#ifndef NDEBUG - // Every new interval must have a def by now, otherwise the split is bogus. - for (LiveRangeEdit::iterator I = Edit->begin(), E = Edit->end(); I != E; ++I) - assert((*I)->hasAtLeastOneValue() && "Split interval has no value"); -#endif - // Transfer the simply mapped values, check if any are skipped. bool Skipped = transferValues(); if (Skipped) diff --git a/lib/CodeGen/SplitKit.h b/lib/CodeGen/SplitKit.h index 7174c0b..a9ccf40b 100644 --- a/lib/CodeGen/SplitKit.h +++ b/lib/CodeGen/SplitKit.h @@ -81,6 +81,12 @@ public: bool LiveThrough; ///< Live in whole block (Templ 5. above). bool LiveIn; ///< Current reg is live in. bool LiveOut; ///< Current reg is live out. + + /// isOneInstr - Returns true when this BlockInfo describes a single + /// instruction. + bool isOneInstr() const { + return SlotIndex::isSameInstr(FirstUse, LastUse); + } }; private: @@ -360,6 +366,10 @@ public: /// Return the beginning of the new live range. SlotIndex enterIntvBefore(SlotIndex Idx); + /// enterIntvAfter - Enter the open interval after the instruction at Idx. + /// Return the beginning of the new live range. + SlotIndex enterIntvAfter(SlotIndex Idx); + /// enterIntvAtEnd - Enter the open interval at the end of MBB. /// Use the open interval from he inserted copy to the MBB end. /// Return the beginning of the new live range. diff --git a/lib/CodeGen/Splitter.cpp b/lib/CodeGen/Splitter.cpp index 08aee82..ec75df4 100644 --- a/lib/CodeGen/Splitter.cpp +++ b/lib/CodeGen/Splitter.cpp @@ -11,7 +11,7 @@ #include "Splitter.h" -#include "SimpleRegisterCoalescing.h" +#include "RegisterCoalescer.h" #include "llvm/Module.h" #include "llvm/CodeGen/CalcSpillWeights.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" diff --git a/lib/CodeGen/StackSlotColoring.cpp b/lib/CodeGen/StackSlotColoring.cpp index 01f5b56..57cbe1b 100644 --- a/lib/CodeGen/StackSlotColoring.cpp +++ b/lib/CodeGen/StackSlotColoring.cpp @@ -504,7 +504,7 @@ bool StackSlotColoring::PropagateBackward(MachineBasicBlock::iterator MII, bool FoundDef = false; // Not counting 2address def. Uses.clear(); - const TargetInstrDesc &TID = MII->getDesc(); + const MCInstrDesc &MCID = MII->getDesc(); for (unsigned i = 0, e = MII->getNumOperands(); i != e; ++i) { MachineOperand &MO = MII->getOperand(i); if (!MO.isReg()) @@ -521,7 +521,7 @@ bool StackSlotColoring::PropagateBackward(MachineBasicBlock::iterator MII, if (MO.getSubReg() || MII->isSubregToReg()) return false; - const TargetRegisterClass *RC = TID.OpInfo[i].getRegClass(TRI); + const TargetRegisterClass *RC = TII->getRegClass(MCID, i, TRI); if (RC && !RC->contains(NewReg)) return false; @@ -566,7 +566,7 @@ bool StackSlotColoring::PropagateForward(MachineBasicBlock::iterator MII, SmallVector<MachineOperand*, 4> Uses; while (++MII != MBB->end()) { bool FoundKill = false; - const TargetInstrDesc &TID = MII->getDesc(); + const MCInstrDesc &MCID = MII->getDesc(); for (unsigned i = 0, e = MII->getNumOperands(); i != e; ++i) { MachineOperand &MO = MII->getOperand(i); if (!MO.isReg()) @@ -583,7 +583,7 @@ bool StackSlotColoring::PropagateForward(MachineBasicBlock::iterator MII, if (MO.getSubReg()) return false; - const TargetRegisterClass *RC = TID.OpInfo[i].getRegClass(TRI); + const TargetRegisterClass *RC = TII->getRegClass(MCID, i, TRI); if (RC && !RC->contains(NewReg)) return false; if (MO.isKill()) diff --git a/lib/CodeGen/TailDuplication.cpp b/lib/CodeGen/TailDuplication.cpp index 3111d59..6fe4bd7 100644 --- a/lib/CodeGen/TailDuplication.cpp +++ b/lib/CodeGen/TailDuplication.cpp @@ -97,8 +97,8 @@ namespace { bool shouldTailDuplicate(const MachineFunction &MF, bool IsSimple, MachineBasicBlock &TailBB); bool isSimpleBB(MachineBasicBlock *TailBB); - bool canCompletelyDuplicateBB(MachineBasicBlock &BB, bool IsSimple); - void duplicateSimpleBB(MachineBasicBlock *TailBB, + bool canCompletelyDuplicateBB(MachineBasicBlock &BB); + bool duplicateSimpleBB(MachineBasicBlock *TailBB, SmallVector<MachineBasicBlock*, 8> &TDBBs, const DenseSet<unsigned> &RegsUsedByPhi, SmallVector<MachineInstr*, 16> &Copies); @@ -529,8 +529,8 @@ TailDuplicatePass::shouldTailDuplicate(const MachineFunction &MF, bool hasIndirectBR = false; if (PreRegAlloc && !TailBB.empty()) { - const TargetInstrDesc &TID = TailBB.back().getDesc(); - if (TID.isIndirectBranch()) { + const MCInstrDesc &MCID = TailBB.back().getDesc(); + if (MCID.isIndirectBranch()) { MaxDuplicateCount = 20; hasIndirectBR = true; } @@ -568,12 +568,12 @@ TailDuplicatePass::shouldTailDuplicate(const MachineFunction &MF, return true; if (IsSimple) - return canCompletelyDuplicateBB(TailBB, IsSimple); + return true; if (!PreRegAlloc) return true; - return canCompletelyDuplicateBB(TailBB, IsSimple); + return canCompletelyDuplicateBB(TailBB); } /// isSimpleBB - True if this BB has only one unconditional jump. @@ -606,74 +606,59 @@ bothUsedInPHI(const MachineBasicBlock &A, } bool -TailDuplicatePass::canCompletelyDuplicateBB(MachineBasicBlock &BB, - bool isSimple) { +TailDuplicatePass::canCompletelyDuplicateBB(MachineBasicBlock &BB) { SmallPtrSet<MachineBasicBlock*, 8> Succs(BB.succ_begin(), BB.succ_end()); for (MachineBasicBlock::pred_iterator PI = BB.pred_begin(), PE = BB.pred_end(); PI != PE; ++PI) { MachineBasicBlock *PredBB = *PI; - if (isSimple) { - if (PredBB->getLandingPadSuccessor()) - return false; - if (bothUsedInPHI(*PredBB, Succs)) - return false; - } else { - if (PredBB->succ_size() > 1) - return false; - } + if (PredBB->succ_size() > 1) + return false; MachineBasicBlock *PredTBB = NULL, *PredFBB = NULL; SmallVector<MachineOperand, 4> PredCond; if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true)) return false; - if (!isSimple && !PredCond.empty()) + if (!PredCond.empty()) return false; } return true; } -void +bool TailDuplicatePass::duplicateSimpleBB(MachineBasicBlock *TailBB, SmallVector<MachineBasicBlock*, 8> &TDBBs, const DenseSet<unsigned> &UsedByPhi, SmallVector<MachineInstr*, 16> &Copies) { + SmallPtrSet<MachineBasicBlock*, 8> Succs(TailBB->succ_begin(), + TailBB->succ_end()); SmallVector<MachineBasicBlock*, 8> Preds(TailBB->pred_begin(), TailBB->pred_end()); + bool Changed = false; for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(), PE = Preds.end(); PI != PE; ++PI) { MachineBasicBlock *PredBB = *PI; + if (PredBB->getLandingPadSuccessor()) + continue; + + if (bothUsedInPHI(*PredBB, Succs)) + continue; + MachineBasicBlock *PredTBB = NULL, *PredFBB = NULL; SmallVector<MachineOperand, 4> PredCond; - bool NotAnalyzable = - TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true); - (void)NotAnalyzable; - assert(!NotAnalyzable && "Cannot duplicate this!"); + if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true)) + continue; + Changed = true; DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB << "From simple Succ: " << *TailBB); MachineBasicBlock *NewTarget = *TailBB->succ_begin(); MachineBasicBlock *NextBB = llvm::next(MachineFunction::iterator(PredBB)); - DenseMap<unsigned, unsigned> LocalVRMap; - SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos; - for (MachineBasicBlock::iterator I = TailBB->begin(); - I != TailBB->end() && I->isPHI();) { - MachineInstr *MI = &*I; - ++I; - ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, true); - } - MachineBasicBlock::iterator Loc = PredBB->getFirstTerminator(); - for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) { - Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(), - TII->get(TargetOpcode::COPY), - CopyInfos[i].first).addReg(CopyInfos[i].second)); - } - // Make PredFBB explicit. if (PredCond.empty()) PredFBB = PredTBB; @@ -715,6 +700,7 @@ TailDuplicatePass::duplicateSimpleBB(MachineBasicBlock *TailBB, TDBBs.push_back(PredBB); } + return Changed; } /// TailDuplicate - If it is profitable, duplicate TailBB's contents in each @@ -733,11 +719,8 @@ TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, MachineFunction &MF, DenseSet<unsigned> UsedByPhi; getRegsUsedByPHIs(*TailBB, &UsedByPhi); - if (IsSimple) { - duplicateSimpleBB(TailBB, TDBBs, UsedByPhi, Copies); - return true; - } - + if (IsSimple) + return duplicateSimpleBB(TailBB, TDBBs, UsedByPhi, Copies); // Iterate through all the unique predecessors and tail-duplicate this // block into them, if possible. Copying the list ahead of time also diff --git a/lib/CodeGen/TargetInstrInfoImpl.cpp b/lib/CodeGen/TargetInstrInfoImpl.cpp index 34e2b33..86e71d8 100644 --- a/lib/CodeGen/TargetInstrInfoImpl.cpp +++ b/lib/CodeGen/TargetInstrInfoImpl.cpp @@ -59,8 +59,8 @@ TargetInstrInfoImpl::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail, // the two operands returned by findCommutedOpIndices. MachineInstr *TargetInstrInfoImpl::commuteInstruction(MachineInstr *MI, bool NewMI) const { - const TargetInstrDesc &TID = MI->getDesc(); - bool HasDef = TID.getNumDefs(); + const MCInstrDesc &MCID = MI->getDesc(); + bool HasDef = MCID.getNumDefs(); if (HasDef && !MI->getOperand(0).isReg()) // No idea how to commute this instruction. Target should implement its own. return 0; @@ -81,7 +81,7 @@ MachineInstr *TargetInstrInfoImpl::commuteInstruction(MachineInstr *MI, bool ChangeReg0 = false; if (HasDef && MI->getOperand(0).getReg() == Reg1) { // Must be two address instruction! - assert(MI->getDesc().getOperandConstraint(0, TOI::TIED_TO) && + assert(MI->getDesc().getOperandConstraint(0, MCOI::TIED_TO) && "Expecting a two-address instruction!"); Reg2IsKill = false; ChangeReg0 = true; @@ -119,12 +119,12 @@ MachineInstr *TargetInstrInfoImpl::commuteInstruction(MachineInstr *MI, bool TargetInstrInfoImpl::findCommutedOpIndices(MachineInstr *MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const { - const TargetInstrDesc &TID = MI->getDesc(); - if (!TID.isCommutable()) + const MCInstrDesc &MCID = MI->getDesc(); + if (!MCID.isCommutable()) return false; // This assumes v0 = op v1, v2 and commuting would swap v1 and v2. If this // is not true, then the target must implement this. - SrcOpIdx1 = TID.getNumDefs(); + SrcOpIdx1 = MCID.getNumDefs(); SrcOpIdx2 = SrcOpIdx1 + 1; if (!MI->getOperand(SrcOpIdx1).isReg() || !MI->getOperand(SrcOpIdx2).isReg()) @@ -137,12 +137,12 @@ bool TargetInstrInfoImpl::findCommutedOpIndices(MachineInstr *MI, bool TargetInstrInfoImpl::PredicateInstruction(MachineInstr *MI, const SmallVectorImpl<MachineOperand> &Pred) const { bool MadeChange = false; - const TargetInstrDesc &TID = MI->getDesc(); - if (!TID.isPredicable()) + const MCInstrDesc &MCID = MI->getDesc(); + if (!MCID.isPredicable()) return false; for (unsigned j = 0, i = 0, e = MI->getNumOperands(); i != e; ++i) { - if (TID.OpInfo[i].isPredicate()) { + if (MCID.OpInfo[i].isPredicate()) { MachineOperand &MO = MI->getOperand(i); if (MO.isReg()) { MO.setReg(Pred[j].getReg()); @@ -332,10 +332,10 @@ isReallyTriviallyReMaterializableGeneric(const MachineInstr *MI, MF.getFrameInfo()->isImmutableObjectIndex(FrameIdx)) return true; - const TargetInstrDesc &TID = MI->getDesc(); + const MCInstrDesc &MCID = MI->getDesc(); // Avoid instructions obviously unsafe for remat. - if (TID.isNotDuplicable() || TID.mayStore() || + if (MCID.isNotDuplicable() || MCID.mayStore() || MI->hasUnmodeledSideEffects()) return false; @@ -345,7 +345,7 @@ isReallyTriviallyReMaterializableGeneric(const MachineInstr *MI, return false; // Avoid instructions which load from potentially varying memory. - if (TID.mayLoad() && !MI->isInvariantLoad(AA)) + if (MCID.mayLoad() && !MI->isInvariantLoad(AA)) return false; // If any of the registers accessed are non-constant, conservatively assume diff --git a/lib/CodeGen/TwoAddressInstructionPass.cpp b/lib/CodeGen/TwoAddressInstructionPass.cpp index 3860e0b..6d6244e 100644 --- a/lib/CodeGen/TwoAddressInstructionPass.cpp +++ b/lib/CodeGen/TwoAddressInstructionPass.cpp @@ -280,8 +280,8 @@ bool TwoAddressInstructionPass::Sink3AddrInstruction(MachineBasicBlock *MBB, /// isTwoAddrUse - Return true if the specified MI is using the specified /// register as a two-address operand. static bool isTwoAddrUse(MachineInstr *UseMI, unsigned Reg) { - const TargetInstrDesc &TID = UseMI->getDesc(); - for (unsigned i = 0, e = TID.getNumOperands(); i != e; ++i) { + const MCInstrDesc &MCID = UseMI->getDesc(); + for (unsigned i = 0, e = MCID.getNumOperands(); i != e; ++i) { MachineOperand &MO = UseMI->getOperand(i); if (MO.isReg() && MO.getReg() == Reg && (MO.isDef() || UseMI->isRegTiedToDefOperand(i))) @@ -443,8 +443,9 @@ static bool isKilled(MachineInstr &MI, unsigned Reg, /// isTwoAddrUse - Return true if the specified MI uses the specified register /// as a two-address use. If so, return the destination register by reference. static bool isTwoAddrUse(MachineInstr &MI, unsigned Reg, unsigned &DstReg) { - const TargetInstrDesc &TID = MI.getDesc(); - unsigned NumOps = MI.isInlineAsm() ? MI.getNumOperands():TID.getNumOperands(); + const MCInstrDesc &MCID = MI.getDesc(); + unsigned NumOps = MI.isInlineAsm() + ? MI.getNumOperands() : MCID.getNumOperands(); for (unsigned i = 0; i != NumOps; ++i) { const MachineOperand &MO = MI.getOperand(i); if (!MO.isReg() || !MO.isUse() || MO.getReg() != Reg) @@ -761,10 +762,10 @@ void TwoAddressInstructionPass::ProcessCopy(MachineInstr *MI, static bool isSafeToDelete(MachineInstr *MI, const TargetInstrInfo *TII, SmallVector<unsigned, 4> &Kills) { - const TargetInstrDesc &TID = MI->getDesc(); - if (TID.mayStore() || TID.isCall()) + const MCInstrDesc &MCID = MI->getDesc(); + if (MCID.mayStore() || MCID.isCall()) return false; - if (TID.isTerminator() || MI->hasUnmodeledSideEffects()) + if (MCID.isTerminator() || MI->hasUnmodeledSideEffects()) return false; for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { @@ -854,7 +855,7 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi, MachineFunction::iterator &mbbi, unsigned SrcIdx, unsigned DstIdx, unsigned Dist, SmallPtrSet<MachineInstr*, 8> &Processed) { - const TargetInstrDesc &TID = mi->getDesc(); + const MCInstrDesc &MCID = mi->getDesc(); unsigned regA = mi->getOperand(DstIdx).getReg(); unsigned regB = mi->getOperand(SrcIdx).getReg(); @@ -876,7 +877,7 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi, unsigned regCIdx = ~0U; bool TryCommute = false; bool AggressiveCommute = false; - if (TID.isCommutable() && mi->getNumOperands() >= 3 && + if (MCID.isCommutable() && mi->getNumOperands() >= 3 && TII->findCommutedOpIndices(mi, SrcOp1, SrcOp2)) { if (SrcIdx == SrcOp1) regCIdx = SrcOp2; @@ -907,7 +908,7 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi, if (TargetRegisterInfo::isVirtualRegister(regA)) ScanUses(regA, &*mbbi, Processed); - if (TID.isConvertibleTo3Addr()) { + if (MCID.isConvertibleTo3Addr()) { // This instruction is potentially convertible to a true // three-address instruction. Check if it is profitable. if (!regBKilled || isProfitableToConv3Addr(regA, regB)) { @@ -927,7 +928,7 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi, // movq (%rax), %rcx // addq %rdx, %rcx // because it's preferable to schedule a load than a register copy. - if (TID.mayLoad() && !regBKilled) { + if (MCID.mayLoad() && !regBKilled) { // Determine if a load can be unfolded. unsigned LoadRegIndex; unsigned NewOpc = @@ -936,14 +937,14 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi, /*UnfoldStore=*/false, &LoadRegIndex); if (NewOpc != 0) { - const TargetInstrDesc &UnfoldTID = TII->get(NewOpc); - if (UnfoldTID.getNumDefs() == 1) { + const MCInstrDesc &UnfoldMCID = TII->get(NewOpc); + if (UnfoldMCID.getNumDefs() == 1) { MachineFunction &MF = *mbbi->getParent(); // Unfold the load. DEBUG(dbgs() << "2addr: UNFOLDING: " << *mi); const TargetRegisterClass *RC = - UnfoldTID.OpInfo[LoadRegIndex].getRegClass(TRI); + TII->getRegClass(UnfoldMCID, LoadRegIndex, TRI); unsigned Reg = MRI->createVirtualRegister(RC); SmallVector<MachineInstr *, 2> NewMIs; if (!TII->unfoldMemoryOperand(MF, mi, Reg, @@ -1067,7 +1068,7 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &MF) { if (mi->isRegSequence()) RegSequences.push_back(&*mi); - const TargetInstrDesc &TID = mi->getDesc(); + const MCInstrDesc &MCID = mi->getDesc(); bool FirstTied = true; DistanceMap.insert(std::make_pair(mi, ++Dist)); @@ -1077,7 +1078,7 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &MF) { // First scan through all the tied register uses in this instruction // and record a list of pairs of tied operands for each register. unsigned NumOps = mi->isInlineAsm() - ? mi->getNumOperands() : TID.getNumOperands(); + ? mi->getNumOperands() : MCID.getNumOperands(); for (unsigned SrcIdx = 0; SrcIdx < NumOps; ++SrcIdx) { unsigned DstIdx = 0; if (!mi->isRegTiedToDefOperand(SrcIdx, &DstIdx)) diff --git a/lib/CodeGen/VirtRegRewriter.cpp b/lib/CodeGen/VirtRegRewriter.cpp index 1850658..a5ec797 100644 --- a/lib/CodeGen/VirtRegRewriter.cpp +++ b/lib/CodeGen/VirtRegRewriter.cpp @@ -679,8 +679,8 @@ static void ReMaterialize(MachineBasicBlock &MBB, VirtRegMap &VRM) { MachineInstr *ReMatDefMI = VRM.getReMaterializedMI(Reg); #ifndef NDEBUG - const TargetInstrDesc &TID = ReMatDefMI->getDesc(); - assert(TID.getNumDefs() == 1 && + const MCInstrDesc &MCID = ReMatDefMI->getDesc(); + assert(MCID.getNumDefs() == 1 && "Don't know how to remat instructions that define > 1 values!"); #endif TII->reMaterialize(MBB, MII, DestReg, 0, ReMatDefMI, *TRI); @@ -1483,11 +1483,11 @@ OptimizeByUnfold(MachineBasicBlock::iterator &MII, /// where SrcReg is r1 and it is tied to r0. Return true if after /// commuting this instruction it will be r0 = op r2, r1. static bool CommuteChangesDestination(MachineInstr *DefMI, - const TargetInstrDesc &TID, + const MCInstrDesc &MCID, unsigned SrcReg, const TargetInstrInfo *TII, unsigned &DstIdx) { - if (TID.getNumDefs() != 1 && TID.getNumOperands() != 3) + if (MCID.getNumDefs() != 1 && MCID.getNumOperands() != 3) return false; if (!DefMI->getOperand(1).isReg() || DefMI->getOperand(1).getReg() != SrcReg) @@ -1527,11 +1527,11 @@ CommuteToFoldReload(MachineBasicBlock::iterator &MII, MachineInstr &MI = *MII; MachineBasicBlock::iterator DefMII = prior(MII); MachineInstr *DefMI = DefMII; - const TargetInstrDesc &TID = DefMI->getDesc(); + const MCInstrDesc &MCID = DefMI->getDesc(); unsigned NewDstIdx; if (DefMII != MBB->begin() && - TID.isCommutable() && - CommuteChangesDestination(DefMI, TID, SrcReg, TII, NewDstIdx)) { + MCID.isCommutable() && + CommuteChangesDestination(DefMI, MCID, SrcReg, TII, NewDstIdx)) { MachineOperand &NewDstMO = DefMI->getOperand(NewDstIdx); unsigned NewReg = NewDstMO.getReg(); if (!NewDstMO.isKill() || TRI->regsOverlap(NewReg, SrcReg)) @@ -1658,9 +1658,9 @@ SpillRegToStackSlot(MachineBasicBlock::iterator &MII, /// isSafeToDelete - Return true if this instruction doesn't produce any side /// effect and all of its defs are dead. static bool isSafeToDelete(MachineInstr &MI) { - const TargetInstrDesc &TID = MI.getDesc(); - if (TID.mayLoad() || TID.mayStore() || TID.isTerminator() || - TID.isCall() || TID.isBarrier() || TID.isReturn() || + const MCInstrDesc &MCID = MI.getDesc(); + if (MCID.mayLoad() || MCID.mayStore() || MCID.isTerminator() || + MCID.isCall() || MCID.isBarrier() || MCID.isReturn() || MI.isLabel() || MI.isDebugValue() || MI.hasUnmodeledSideEffects()) return false; |