diff options
Diffstat (limited to 'lib/CodeGen')
70 files changed, 1759 insertions, 1526 deletions
diff --git a/lib/CodeGen/AggressiveAntiDepBreaker.cpp b/lib/CodeGen/AggressiveAntiDepBreaker.cpp index 25842a7..6cf4571 100644 --- a/lib/CodeGen/AggressiveAntiDepBreaker.cpp +++ b/lib/CodeGen/AggressiveAntiDepBreaker.cpp @@ -148,7 +148,7 @@ void AggressiveAntiDepBreaker::StartBlock(MachineBasicBlock *BB) { assert(State == NULL); State = new AggressiveAntiDepState(TRI->getNumRegs(), BB); - bool IsReturnBlock = (!BB->empty() && BB->back().getDesc().isReturn()); + bool IsReturnBlock = (!BB->empty() && BB->back().isReturn()); std::vector<unsigned> &KillIndices = State->GetKillIndices(); std::vector<unsigned> &DefIndices = State->GetDefIndices(); @@ -384,7 +384,7 @@ void AggressiveAntiDepBreaker::PrescanInstruction(MachineInstr *MI, // If MI's defs have a special allocation requirement, don't allow // any def registers to be changed. Also assume all registers // defined in a call must not be changed (ABI). - if (MI->getDesc().isCall() || MI->getDesc().hasExtraDefRegAllocReq() || + if (MI->isCall() || MI->hasExtraDefRegAllocReq() || TII->isPredicated(MI)) { DEBUG(if (State->GetGroup(Reg) != 0) dbgs() << "->g0(alloc-req)"); State->UnionGroups(Reg, 0); @@ -451,8 +451,8 @@ void AggressiveAntiDepBreaker::ScanInstruction(MachineInstr *MI, // instruction which may not be executed. The second R6 def may or may not // re-define R6 so it's not safe to change it since the last R6 use cannot be // changed. - bool Special = MI->getDesc().isCall() || - MI->getDesc().hasExtraSrcRegAllocReq() || + bool Special = MI->isCall() || + MI->hasExtraSrcRegAllocReq() || TII->isPredicated(MI); // Scan the register uses for this instruction and update diff --git a/lib/CodeGen/Analysis.cpp b/lib/CodeGen/Analysis.cpp index fafc010..0c84be5 100644 --- a/lib/CodeGen/Analysis.cpp +++ b/lib/CodeGen/Analysis.cpp @@ -1,4 +1,4 @@ -//===-- Analysis.cpp - CodeGen LLVM IR Analysis Utilities --*- C++ ------*-===// +//===-- Analysis.cpp - CodeGen LLVM IR Analysis Utilities -----------------===// // // The LLVM Compiler Infrastructure // @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/Analysis.h" +#include "llvm/Analysis/ValueTracking.h" #include "llvm/DerivedTypes.h" #include "llvm/Function.h" #include "llvm/Instructions.h" @@ -149,33 +150,40 @@ llvm::hasInlineAsmMemConstraint(InlineAsm::ConstraintInfoVector &CInfos, /// consideration of global floating-point math flags. /// ISD::CondCode llvm::getFCmpCondCode(FCmpInst::Predicate Pred) { - ISD::CondCode FPC, FOC; switch (Pred) { - case FCmpInst::FCMP_FALSE: FOC = FPC = ISD::SETFALSE; break; - case FCmpInst::FCMP_OEQ: FOC = ISD::SETEQ; FPC = ISD::SETOEQ; break; - case FCmpInst::FCMP_OGT: FOC = ISD::SETGT; FPC = ISD::SETOGT; break; - case FCmpInst::FCMP_OGE: FOC = ISD::SETGE; FPC = ISD::SETOGE; break; - case FCmpInst::FCMP_OLT: FOC = ISD::SETLT; FPC = ISD::SETOLT; break; - case FCmpInst::FCMP_OLE: FOC = ISD::SETLE; FPC = ISD::SETOLE; break; - case FCmpInst::FCMP_ONE: FOC = ISD::SETNE; FPC = ISD::SETONE; break; - case FCmpInst::FCMP_ORD: FOC = FPC = ISD::SETO; break; - case FCmpInst::FCMP_UNO: FOC = FPC = ISD::SETUO; break; - case FCmpInst::FCMP_UEQ: FOC = ISD::SETEQ; FPC = ISD::SETUEQ; break; - case FCmpInst::FCMP_UGT: FOC = ISD::SETGT; FPC = ISD::SETUGT; break; - case FCmpInst::FCMP_UGE: FOC = ISD::SETGE; FPC = ISD::SETUGE; break; - case FCmpInst::FCMP_ULT: FOC = ISD::SETLT; FPC = ISD::SETULT; break; - case FCmpInst::FCMP_ULE: FOC = ISD::SETLE; FPC = ISD::SETULE; break; - case FCmpInst::FCMP_UNE: FOC = ISD::SETNE; FPC = ISD::SETUNE; break; - case FCmpInst::FCMP_TRUE: FOC = FPC = ISD::SETTRUE; break; - default: - llvm_unreachable("Invalid FCmp predicate opcode!"); - FOC = FPC = ISD::SETFALSE; - break; + case FCmpInst::FCMP_FALSE: return ISD::SETFALSE; + case FCmpInst::FCMP_OEQ: return ISD::SETOEQ; + case FCmpInst::FCMP_OGT: return ISD::SETOGT; + case FCmpInst::FCMP_OGE: return ISD::SETOGE; + case FCmpInst::FCMP_OLT: return ISD::SETOLT; + case FCmpInst::FCMP_OLE: return ISD::SETOLE; + case FCmpInst::FCMP_ONE: return ISD::SETONE; + case FCmpInst::FCMP_ORD: return ISD::SETO; + case FCmpInst::FCMP_UNO: return ISD::SETUO; + case FCmpInst::FCMP_UEQ: return ISD::SETUEQ; + case FCmpInst::FCMP_UGT: return ISD::SETUGT; + case FCmpInst::FCMP_UGE: return ISD::SETUGE; + case FCmpInst::FCMP_ULT: return ISD::SETULT; + case FCmpInst::FCMP_ULE: return ISD::SETULE; + case FCmpInst::FCMP_UNE: return ISD::SETUNE; + case FCmpInst::FCMP_TRUE: return ISD::SETTRUE; + default: break; + } + llvm_unreachable("Invalid FCmp predicate opcode!"); + return ISD::SETFALSE; +} + +ISD::CondCode llvm::getFCmpCodeWithoutNaN(ISD::CondCode CC) { + switch (CC) { + case ISD::SETOEQ: case ISD::SETUEQ: return ISD::SETEQ; + case ISD::SETONE: case ISD::SETUNE: return ISD::SETNE; + case ISD::SETOLT: case ISD::SETULT: return ISD::SETLT; + case ISD::SETOLE: case ISD::SETULE: return ISD::SETLE; + case ISD::SETOGT: case ISD::SETUGT: return ISD::SETGT; + case ISD::SETOGE: case ISD::SETUGE: return ISD::SETGE; + default: break; } - if (NoNaNsFPMath) - return FOC; - else - return FPC; + return CC; } /// getICmpCondCode - Return the ISD condition code corresponding to @@ -221,12 +229,13 @@ bool llvm::isInTailCallPosition(ImmutableCallSite CS, Attributes CalleeRetAttr, // longjmp on x86), it can end up causing miscompilation that has not // been fully understood. if (!Ret && - (!GuaranteedTailCallOpt || !isa<UnreachableInst>(Term))) return false; + (!TLI.getTargetMachine().Options.GuaranteedTailCallOpt || + !isa<UnreachableInst>(Term))) return false; // If I will have a chain, make sure no other instruction that will have a // chain interposes between I and the return. if (I->mayHaveSideEffects() || I->mayReadFromMemory() || - !I->isSafeToSpeculativelyExecute()) + !isSafeToSpeculativelyExecute(I)) for (BasicBlock::const_iterator BBI = prior(prior(ExitBB->end())); ; --BBI) { if (&*BBI == I) @@ -235,7 +244,7 @@ bool llvm::isInTailCallPosition(ImmutableCallSite CS, Attributes CalleeRetAttr, if (isa<DbgInfoIntrinsic>(BBI)) continue; if (BBI->mayHaveSideEffects() || BBI->mayReadFromMemory() || - !BBI->isSafeToSpeculativelyExecute()) + !isSafeToSpeculativelyExecute(BBI)) return false; } diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 711b796..0c4d0d5 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -1308,7 +1308,7 @@ void AsmPrinter::EmitXXStructorList(const Constant *List) { } // Emit the function pointers in reverse priority order. - switch (MAI->getStructorOutputOrder()) { + switch (getObjFileLowering().getStructorOutputOrder()) { case Structors::None: break; case Structors::PriorityOrder: @@ -1659,6 +1659,28 @@ static void EmitGlobalConstantVector(const ConstantVector *CV, AP.OutStreamer.EmitZeros(Padding, AddrSpace); } +static void LowerVectorConstant(const Constant *CV, unsigned AddrSpace, + AsmPrinter &AP) { + // Look through bitcasts + if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(CV)) + if (CE->getOpcode() == Instruction::BitCast) + CV = CE->getOperand(0); + + if (const ConstantVector *V = dyn_cast<ConstantVector>(CV)) + return EmitGlobalConstantVector(V, AddrSpace, AP); + + // If we get here, we're stuck; report the problem to the user. + // FIXME: Are there any other useful tricks for vectors? + { + std::string S; + raw_string_ostream OS(S); + OS << "Unsupported vector expression in static initializer: "; + WriteAsOperand(OS, CV, /*PrintType=*/false, + !AP.MF ? 0 : AP.MF->getFunction()->getParent()); + report_fatal_error(OS.str()); + } +} + static void EmitGlobalConstantStruct(const ConstantStruct *CS, unsigned AddrSpace, AsmPrinter &AP) { // Print the fields in successive locations. Pad to align if needed! @@ -1813,8 +1835,8 @@ static void EmitGlobalConstantImpl(const Constant *CV, unsigned AddrSpace, return; } - if (const ConstantVector *V = dyn_cast<ConstantVector>(CV)) - return EmitGlobalConstantVector(V, AddrSpace, AP); + if (CV->getType()->isVectorTy()) + return LowerVectorConstant(CV, AddrSpace, AP); // Otherwise, it must be a ConstantExpr. Lower it to an MCExpr, then emit it // thread the streamer with EmitValue. @@ -1987,7 +2009,7 @@ static void EmitBasicBlockLoopComments(const MachineBasicBlock &MBB, void AsmPrinter::EmitBasicBlockStart(const MachineBasicBlock *MBB) const { // Emit an alignment directive for this block, if needed. if (unsigned Align = MBB->getAlignment()) - EmitAlignment(Log2_32(Align)); + EmitAlignment(Align); // If the block has its address taken, emit any labels that were used to // reference the block. It is possible that there is more than one label @@ -2082,7 +2104,7 @@ isBlockOnlyReachableByFallthrough(const MachineBasicBlock *MBB) const { MachineInstr &MI = *II; // If it is not a simple branch, we are in a table somewhere. - if (!MI.getDesc().isBranch() || MI.getDesc().isIndirectBranch()) + if (!MI.isBranch() || MI.isIndirectBranch()) return false; // If we are the operands of one of the branches, this is not diff --git a/lib/CodeGen/AsmPrinter/CMakeLists.txt b/lib/CodeGen/AsmPrinter/CMakeLists.txt index f6ce17d..58fe2ed 100644 --- a/lib/CodeGen/AsmPrinter/CMakeLists.txt +++ b/lib/CodeGen/AsmPrinter/CMakeLists.txt @@ -12,13 +12,3 @@ add_llvm_library(LLVMAsmPrinter OcamlGCPrinter.cpp Win64Exception.cpp ) - -add_llvm_library_dependencies(LLVMAsmPrinter - LLVMAnalysis - LLVMCodeGen - LLVMCore - LLVMMC - LLVMMCParser - LLVMSupport - LLVMTarget - ) diff --git a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp index 237998a..8cb5156 100644 --- a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp @@ -791,13 +791,13 @@ void CompileUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) { DISubprogram SP(Element); ElemDie = getOrCreateSubprogramDIE(DISubprogram(Element)); if (SP.isProtected()) - addUInt(ElemDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_flag, + addUInt(ElemDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_protected); else if (SP.isPrivate()) - addUInt(ElemDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_flag, + addUInt(ElemDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_private); else - addUInt(ElemDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_flag, + addUInt(ElemDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_public); if (SP.isExplicit()) addUInt(ElemDie, dwarf::DW_AT_explicit, dwarf::DW_FORM_flag, 1); @@ -988,7 +988,7 @@ DIE *CompileUnit::getOrCreateSubprogramDIE(DISubprogram SP) { unsigned VK = SP.getVirtuality(); if (VK) { - addUInt(SPDie, dwarf::DW_AT_virtuality, dwarf::DW_FORM_flag, VK); + addUInt(SPDie, dwarf::DW_AT_virtuality, dwarf::DW_FORM_data1, VK); DIEBlock *Block = getDIEBlock(); addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_constu); addUInt(Block, 0, dwarf::DW_FORM_udata, SP.getVirtualIndex()); @@ -1398,17 +1398,17 @@ DIE *CompileUnit::createMemberDIE(DIDerivedType DT) { addBlock(MemberDie, dwarf::DW_AT_data_member_location, 0, MemLocationDie); if (DT.isProtected()) - addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_flag, + addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_protected); else if (DT.isPrivate()) - addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_flag, + addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_private); // Otherwise C++ member and base classes are considered public. else - addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_flag, + addUInt(MemberDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1, dwarf::DW_ACCESS_public); if (DT.isVirtual()) - addUInt(MemberDie, dwarf::DW_AT_virtuality, dwarf::DW_FORM_flag, + addUInt(MemberDie, dwarf::DW_AT_virtuality, dwarf::DW_FORM_data1, dwarf::DW_VIRTUALITY_virtual); // Objective-C properties. diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp index dc46a58..a3db96a 100644 --- a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp @@ -442,6 +442,10 @@ DIE *DwarfDebug::constructInlinedScopeDIE(CompileUnit *TheCU, TheCU->addUInt(ScopeDIE, dwarf::DW_AT_call_file, 0, TheCU->getID()); TheCU->addUInt(ScopeDIE, dwarf::DW_AT_call_line, 0, DL.getLineNumber()); + // Add name to the name table, we do this here because we're guaranteed + // to have concrete versions of our DW_TAG_inlined_subprogram nodes. + addSubprogramNames(TheCU, InlinedSP, ScopeDIE); + return ScopeDIE; } @@ -1414,7 +1418,7 @@ void DwarfDebug::endFunction(const MachineFunction *MF) { DIE *CurFnDIE = constructScopeDIE(TheCU, FnScope); - if (!DisableFramePointerElim(*MF)) + if (!MF->getTarget().Options.DisableFramePointerElim(*MF)) TheCU->addUInt(CurFnDIE, dwarf::DW_AT_APPLE_omit_frame_ptr, dwarf::DW_FORM_flag, 1); diff --git a/lib/CodeGen/AsmPrinter/DwarfException.cpp b/lib/CodeGen/AsmPrinter/DwarfException.cpp index e0f2e85..bf7f7ee 100644 --- a/lib/CodeGen/AsmPrinter/DwarfException.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfException.cpp @@ -184,7 +184,7 @@ ComputeActionsTable(const SmallVectorImpl<const LandingPadInfo*> &LandingPads, /// CallToNoUnwindFunction - Return `true' if this is a call to a function /// marked `nounwind'. Return `false' otherwise. bool DwarfException::CallToNoUnwindFunction(const MachineInstr *MI) { - assert(MI->getDesc().isCall() && "This should be a call instruction!"); + assert(MI->isCall() && "This should be a call instruction!"); bool MarkedNoUnwind = false; bool SawFunc = false; @@ -243,7 +243,7 @@ ComputeCallSiteTable(SmallVectorImpl<CallSiteEntry> &CallSites, for (MachineBasicBlock::const_iterator MI = I->begin(), E = I->end(); MI != E; ++MI) { if (!MI->isLabel()) { - if (MI->getDesc().isCall()) + if (MI->isCall()) SawPotentiallyThrowing |= !CallToNoUnwindFunction(MI); continue; } diff --git a/lib/CodeGen/AsmPrinter/LLVMBuild.txt b/lib/CodeGen/AsmPrinter/LLVMBuild.txt index 0f2059f..20b1f7b 100644 --- a/lib/CodeGen/AsmPrinter/LLVMBuild.txt +++ b/lib/CodeGen/AsmPrinter/LLVMBuild.txt @@ -20,4 +20,3 @@ type = Library name = AsmPrinter parent = Libraries required_libraries = Analysis CodeGen Core MC MCParser Support Target - diff --git a/lib/CodeGen/BranchFolding.cpp b/lib/CodeGen/BranchFolding.cpp index 5dec368..89894c3 100644 --- a/lib/CodeGen/BranchFolding.cpp +++ b/lib/CodeGen/BranchFolding.cpp @@ -432,10 +432,9 @@ static unsigned EstimateRuntime(MachineBasicBlock::iterator I, for (; I != E; ++I) { if (I->isDebugValue()) continue; - const MCInstrDesc &MCID = I->getDesc(); - if (MCID.isCall()) + if (I->isCall()) Time += 10; - else if (MCID.mayLoad() || MCID.mayStore()) + else if (I->mayLoad() || I->mayStore()) Time += 2; else ++Time; @@ -502,7 +501,7 @@ static unsigned CountTerminators(MachineBasicBlock *MBB, break; } --I; - if (!I->getDesc().isTerminator()) break; + if (!I->isTerminator()) break; ++NumTerms; } return NumTerms; @@ -550,8 +549,8 @@ static bool ProfitableToMerge(MachineBasicBlock *MBB1, // heuristics. unsigned EffectiveTailLen = CommonTailLen; if (SuccBB && MBB1 != PredBB && MBB2 != PredBB && - !MBB1->back().getDesc().isBarrier() && - !MBB2->back().getDesc().isBarrier()) + !MBB1->back().isBarrier() && + !MBB2->back().isBarrier()) ++EffectiveTailLen; // Check if the common tail is long enough to be worthwhile. @@ -927,8 +926,9 @@ bool BranchFolder::TailMergeBlocks(MachineFunction &MF) { if (MergePotentials.size() >= 2) MadeChange |= TryTailMergeBlocks(IBB, PredBB); // Reinsert an unconditional branch if needed. - // The 1 below can occur as a result of removing blocks in TryTailMergeBlocks. - PredBB = prior(I); // this may have been changed in TryTailMergeBlocks + // The 1 below can occur as a result of removing blocks in + // TryTailMergeBlocks. + PredBB = prior(I); // this may have been changed in TryTailMergeBlocks if (MergePotentials.size() == 1 && MergePotentials.begin()->getBlock() != PredBB) FixTail(MergePotentials.begin()->getBlock(), IBB, TII); @@ -983,7 +983,7 @@ static bool IsBranchOnlyBlock(MachineBasicBlock *MBB) { if (!MBBI->isDebugValue()) break; } - return (MBBI->getDesc().isBranch()); + return (MBBI->isBranch()); } /// IsBetterFallthrough - Return true if it would be clearly better to @@ -1011,7 +1011,7 @@ static bool IsBetterFallthrough(MachineBasicBlock *MBB1, MachineBasicBlock::iterator MBB2I = --MBB2->end(); while (MBB2I->isDebugValue()) --MBB2I; - return MBB2I->getDesc().isCall() && !MBB1I->getDesc().isCall(); + return MBB2I->isCall() && !MBB1I->isCall(); } /// OptimizeBlock - Analyze and optimize control flow related to the specified diff --git a/lib/CodeGen/CMakeLists.txt b/lib/CodeGen/CMakeLists.txt index c8d4dcf..7aee3bb 100644 --- a/lib/CodeGen/CMakeLists.txt +++ b/lib/CodeGen/CMakeLists.txt @@ -9,6 +9,7 @@ add_llvm_library(LLVMCodeGen CodePlacementOpt.cpp CriticalAntiDepBreaker.cpp DeadMachineInstructionElim.cpp + DFAPacketizer.cpp DwarfEHPrepare.cpp EdgeBundles.cpp ELFCodeEmitter.cpp @@ -46,6 +47,7 @@ add_llvm_library(LLVMCodeGen MachineFunctionPass.cpp MachineFunctionPrinterPass.cpp MachineInstr.cpp + MachineInstrBundle.cpp MachineLICM.cpp MachineLoopInfo.cpp MachineLoopRanges.cpp @@ -87,27 +89,18 @@ add_llvm_library(LLVMCodeGen Spiller.cpp SpillPlacement.cpp SplitKit.cpp - Splitter.cpp StackProtector.cpp StackSlotColoring.cpp StrongPHIElimination.cpp TailDuplication.cpp + TargetFrameLoweringImpl.cpp TargetInstrInfoImpl.cpp TargetLoweringObjectFileImpl.cpp + TargetOptionsImpl.cpp TwoAddressInstructionPass.cpp UnreachableBlockElim.cpp VirtRegMap.cpp ) -add_llvm_library_dependencies(LLVMCodeGen - LLVMAnalysis - LLVMCore - LLVMMC - LLVMScalarOpts - LLVMSupport - LLVMTarget - LLVMTransformUtils - ) - add_subdirectory(SelectionDAG) add_subdirectory(AsmPrinter) diff --git a/lib/CodeGen/CodeGen.cpp b/lib/CodeGen/CodeGen.cpp index 3112c22..48b71d9 100644 --- a/lib/CodeGen/CodeGen.cpp +++ b/lib/CodeGen/CodeGen.cpp @@ -45,7 +45,6 @@ void llvm::initializeCodeGen(PassRegistry &Registry) { initializeRegisterCoalescerPass(Registry); initializeRenderMachineFunctionPass(Registry); initializeSlotIndexesPass(Registry); - initializeLoopSplitterPass(Registry); initializeStackProtectorPass(Registry); initializeStackSlotColoringPass(Registry); initializeStrongPHIEliminationPass(Registry); diff --git a/lib/CodeGen/CriticalAntiDepBreaker.cpp b/lib/CodeGen/CriticalAntiDepBreaker.cpp index 84c4d59..128143e 100644 --- a/lib/CodeGen/CriticalAntiDepBreaker.cpp +++ b/lib/CodeGen/CriticalAntiDepBreaker.cpp @@ -54,7 +54,7 @@ void CriticalAntiDepBreaker::StartBlock(MachineBasicBlock *BB) { // Clear "do not change" set. KeepRegs.clear(); - bool IsReturnBlock = (!BB->empty() && BB->back().getDesc().isReturn()); + bool IsReturnBlock = (!BB->empty() && BB->back().isReturn()); // Determine the live-out physregs for this block. if (IsReturnBlock) { @@ -193,8 +193,8 @@ void CriticalAntiDepBreaker::PrescanInstruction(MachineInstr *MI) { // instruction which may not be executed. The second R6 def may or may not // re-define R6 so it's not safe to change it since the last R6 use cannot be // changed. - bool Special = MI->getDesc().isCall() || - MI->getDesc().hasExtraSrcRegAllocReq() || + bool Special = MI->isCall() || + MI->hasExtraSrcRegAllocReq() || TII->isPredicated(MI); // Scan the register operands for this instruction and update @@ -572,7 +572,7 @@ BreakAntiDependencies(const std::vector<SUnit>& SUnits, // If MI's defs have a special allocation requirement, don't allow // any def registers to be changed. Also assume all registers // defined in a call must not be changed (ABI). - if (MI->getDesc().isCall() || MI->getDesc().hasExtraDefRegAllocReq() || + if (MI->isCall() || MI->hasExtraDefRegAllocReq() || TII->isPredicated(MI)) // If this instruction's defs have special allocation requirement, don't // break this anti-dependency. diff --git a/lib/CodeGen/DFAPacketizer.cpp b/lib/CodeGen/DFAPacketizer.cpp new file mode 100644 index 0000000..16276bd --- /dev/null +++ b/lib/CodeGen/DFAPacketizer.cpp @@ -0,0 +1,98 @@ +//=- llvm/CodeGen/DFAPacketizer.cpp - DFA Packetizer for VLIW -*- C++ -*-=====// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// This class implements a deterministic finite automaton (DFA) based +// packetizing mechanism for VLIW architectures. It provides APIs to +// determine whether there exists a legal mapping of instructions to +// functional unit assignments in a packet. The DFA is auto-generated from +// the target's Schedule.td file. +// +// A DFA consists of 3 major elements: states, inputs, and transitions. For +// the packetizing mechanism, the input is the set of instruction classes for +// a target. The state models all possible combinations of functional unit +// consumption for a given set of instructions in a packet. A transition +// models the addition of an instruction to a packet. In the DFA constructed +// by this class, if an instruction can be added to a packet, then a valid +// transition exists from the corresponding state. Invalid transitions +// indicate that the instruction cannot be added to the current packet. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/DFAPacketizer.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/MC/MCInstrItineraries.h" +using namespace llvm; + +DFAPacketizer::DFAPacketizer(const InstrItineraryData *I, const int (*SIT)[2], + const unsigned *SET): + InstrItins(I), CurrentState(0), DFAStateInputTable(SIT), + DFAStateEntryTable(SET) {} + + +// +// ReadTable - Read the DFA transition table and update CachedTable. +// +// Format of the transition tables: +// DFAStateInputTable[][2] = pairs of <Input, Transition> for all valid +// transitions +// DFAStateEntryTable[i] = Index of the first entry in DFAStateInputTable +// for the ith state +// +void DFAPacketizer::ReadTable(unsigned int state) { + unsigned ThisState = DFAStateEntryTable[state]; + unsigned NextStateInTable = DFAStateEntryTable[state+1]; + // Early exit in case CachedTable has already contains this + // state's transitions. + if (CachedTable.count(UnsignPair(state, + DFAStateInputTable[ThisState][0]))) + return; + + for (unsigned i = ThisState; i < NextStateInTable; i++) + CachedTable[UnsignPair(state, DFAStateInputTable[i][0])] = + DFAStateInputTable[i][1]; +} + + +// canReserveResources - Check if the resources occupied by a MCInstrDesc +// are available in the current state. +bool DFAPacketizer::canReserveResources(const llvm::MCInstrDesc *MID) { + unsigned InsnClass = MID->getSchedClass(); + const llvm::InstrStage *IS = InstrItins->beginStage(InsnClass); + unsigned FuncUnits = IS->getUnits(); + UnsignPair StateTrans = UnsignPair(CurrentState, FuncUnits); + ReadTable(CurrentState); + return (CachedTable.count(StateTrans) != 0); +} + + +// reserveResources - Reserve the resources occupied by a MCInstrDesc and +// change the current state to reflect that change. +void DFAPacketizer::reserveResources(const llvm::MCInstrDesc *MID) { + unsigned InsnClass = MID->getSchedClass(); + const llvm::InstrStage *IS = InstrItins->beginStage(InsnClass); + unsigned FuncUnits = IS->getUnits(); + UnsignPair StateTrans = UnsignPair(CurrentState, FuncUnits); + ReadTable(CurrentState); + assert(CachedTable.count(StateTrans) != 0); + CurrentState = CachedTable[StateTrans]; +} + + +// canReserveResources - Check if the resources occupied by a machine +// instruction are available in the current state. +bool DFAPacketizer::canReserveResources(llvm::MachineInstr *MI) { + const llvm::MCInstrDesc &MID = MI->getDesc(); + return canReserveResources(&MID); +} + +// reserveResources - Reserve the resources occupied by a machine +// instruction and change the current state to reflect that change. +void DFAPacketizer::reserveResources(llvm::MachineInstr *MI) { + const llvm::MCInstrDesc &MID = MI->getDesc(); + reserveResources(&MID); +} diff --git a/lib/CodeGen/DeadMachineInstructionElim.cpp b/lib/CodeGen/DeadMachineInstructionElim.cpp index 6de6c0c..ba135e1 100644 --- a/lib/CodeGen/DeadMachineInstructionElim.cpp +++ b/lib/CodeGen/DeadMachineInstructionElim.cpp @@ -102,7 +102,7 @@ bool DeadMachineInstructionElim::runOnMachineFunction(MachineFunction &MF) { LivePhysRegs = ReservedRegs; // Also add any explicit live-out physregs for this block. - if (!MBB->empty() && MBB->back().getDesc().isReturn()) + if (!MBB->empty() && MBB->back().isReturn()) for (MachineRegisterInfo::liveout_iterator LOI = MRI->liveout_begin(), LOE = MRI->liveout_end(); LOI != LOE; ++LOI) { unsigned Reg = *LOI; diff --git a/lib/CodeGen/ExecutionDepsFix.cpp b/lib/CodeGen/ExecutionDepsFix.cpp index 300f037..4ec75cd 100644 --- a/lib/CodeGen/ExecutionDepsFix.cpp +++ b/lib/CodeGen/ExecutionDepsFix.cpp @@ -454,7 +454,7 @@ void ExeDepsFix::processDefs(MachineInstr *MI, bool Kill) { assert(!MI->isDebugValue() && "Won't process debug values"); const MCInstrDesc &MCID = MI->getDesc(); for (unsigned i = 0, - e = MCID.isVariadic() ? MI->getNumOperands() : MCID.getNumDefs(); + e = MI->isVariadic() ? MI->getNumOperands() : MCID.getNumDefs(); i != e; ++i) { MachineOperand &MO = MI->getOperand(i); if (!MO.isReg()) diff --git a/lib/CodeGen/ExpandISelPseudos.cpp b/lib/CodeGen/ExpandISelPseudos.cpp index a67140e..b5f107d 100644 --- a/lib/CodeGen/ExpandISelPseudos.cpp +++ b/lib/CodeGen/ExpandISelPseudos.cpp @@ -62,8 +62,7 @@ bool ExpandISelPseudos::runOnMachineFunction(MachineFunction &MF) { MachineInstr *MI = MBBI++; // If MI is a pseudo, expand it. - const MCInstrDesc &MCID = MI->getDesc(); - if (MCID.usesCustomInsertionHook()) { + if (MI->usesCustomInsertionHook()) { Changed = true; MachineBasicBlock *NewMBB = TLI->EmitInstrWithCustomInserter(MI, MBB); diff --git a/lib/CodeGen/ExpandPostRAPseudos.cpp b/lib/CodeGen/ExpandPostRAPseudos.cpp index e2a14a8..3d23db0 100644 --- a/lib/CodeGen/ExpandPostRAPseudos.cpp +++ b/lib/CodeGen/ExpandPostRAPseudos.cpp @@ -207,7 +207,7 @@ bool ExpandPostRA::runOnMachineFunction(MachineFunction &MF) { ++mi; // Only expand pseudos. - if (!MI->getDesc().isPseudo()) + if (!MI->isPseudo()) continue; // Give targets a chance to expand even standard pseudos. diff --git a/lib/CodeGen/GCStrategy.cpp b/lib/CodeGen/GCStrategy.cpp index 9349797..e2c7132 100644 --- a/lib/CodeGen/GCStrategy.cpp +++ b/lib/CodeGen/GCStrategy.cpp @@ -386,7 +386,7 @@ void MachineCodeAnalysis::FindSafePoints(MachineFunction &MF) { BBE = MF.end(); BBI != BBE; ++BBI) for (MachineBasicBlock::iterator MI = BBI->begin(), ME = BBI->end(); MI != ME; ++MI) - if (MI->getDesc().isCall()) + if (MI->isCall()) VisitCallPoint(MI); } diff --git a/lib/CodeGen/IfConversion.cpp b/lib/CodeGen/IfConversion.cpp index d888939..bd31fdf 100644 --- a/lib/CodeGen/IfConversion.cpp +++ b/lib/CodeGen/IfConversion.cpp @@ -573,12 +573,12 @@ bool IfConverter::ValidDiamond(BBInfo &TrueBBI, BBInfo &FalseBBI, // blocks, move the end iterators up past any branch instructions. while (TIE != TIB) { --TIE; - if (!TIE->getDesc().isBranch()) + if (!TIE->isBranch()) break; } while (FIE != FIB) { --FIE; - if (!FIE->getDesc().isBranch()) + if (!FIE->isBranch()) break; } @@ -651,12 +651,11 @@ void IfConverter::ScanInstructions(BBInfo &BBI) { if (I->isDebugValue()) continue; - const MCInstrDesc &MCID = I->getDesc(); - if (MCID.isNotDuplicable()) + if (I->isNotDuplicable()) BBI.CannotBeCopied = true; bool isPredicated = TII->isPredicated(I); - bool isCondBr = BBI.IsBrAnalyzable && MCID.isConditionalBranch(); + bool isCondBr = BBI.IsBrAnalyzable && I->isConditionalBranch(); if (!isCondBr) { if (!isPredicated) { @@ -1395,9 +1394,8 @@ void IfConverter::CopyAndPredicateBlock(BBInfo &ToBBI, BBInfo &FromBBI, for (MachineBasicBlock::iterator I = FromBBI.BB->begin(), E = FromBBI.BB->end(); I != E; ++I) { - const MCInstrDesc &MCID = I->getDesc(); // Do not copy the end of the block branches. - if (IgnoreBr && MCID.isBranch()) + if (IgnoreBr && I->isBranch()) break; MachineInstr *MI = MF.CloneMachineInstr(I); diff --git a/lib/CodeGen/InlineSpiller.cpp b/lib/CodeGen/InlineSpiller.cpp index 59907d9..9bf810e 100644 --- a/lib/CodeGen/InlineSpiller.cpp +++ b/lib/CodeGen/InlineSpiller.cpp @@ -759,7 +759,7 @@ void InlineSpiller::eliminateRedundantSpills(LiveInterval &SLI, VNInfo *VNI) { // Find all spills and copies of VNI. for (MachineRegisterInfo::use_nodbg_iterator UI = MRI.use_nodbg_begin(Reg); MachineInstr *MI = UI.skipInstruction();) { - if (!MI->isCopy() && !MI->getDesc().mayStore()) + if (!MI->isCopy() && !MI->mayStore()) continue; SlotIndex Idx = LIS.getInstructionIndex(MI); if (LI->getVNInfoAt(Idx) != VNI) @@ -878,7 +878,7 @@ bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, // Before rematerializing into a register for a single instruction, try to // fold a load into the instruction. That avoids allocating a new register. - if (RM.OrigMI->getDesc().canFoldAsLoad() && + if (RM.OrigMI->canFoldAsLoad() && foldMemoryOperand(MI, Ops, RM.OrigMI)) { Edit->markRematerialized(RM.ParentVNI); ++NumFoldedLoads; @@ -957,7 +957,7 @@ void InlineSpiller::reMaterializeAll() { if (DeadDefs.empty()) return; DEBUG(dbgs() << "Remat created " << DeadDefs.size() << " dead defs.\n"); - Edit->eliminateDeadDefs(DeadDefs, LIS, VRM, TII); + Edit->eliminateDeadDefs(DeadDefs, LIS, VRM, TII, RegsToSpill); // Get rid of deleted and empty intervals. for (unsigned i = RegsToSpill.size(); i != 0; --i) { @@ -1240,7 +1240,7 @@ void InlineSpiller::spillAll() { // Hoisted spills may cause dead code. if (!DeadDefs.empty()) { DEBUG(dbgs() << "Eliminating " << DeadDefs.size() << " dead defs\n"); - Edit->eliminateDeadDefs(DeadDefs, LIS, VRM, TII); + Edit->eliminateDeadDefs(DeadDefs, LIS, VRM, TII, RegsToSpill); } // Finally delete the SnippetCopies. diff --git a/lib/CodeGen/LLVMBuild.txt b/lib/CodeGen/LLVMBuild.txt index 2eebb08..fee0347 100644 --- a/lib/CodeGen/LLVMBuild.txt +++ b/lib/CodeGen/LLVMBuild.txt @@ -15,9 +15,11 @@ ; ;===------------------------------------------------------------------------===; +[common] +subdirectories = AsmPrinter SelectionDAG + [component_0] type = Library name = CodeGen parent = Libraries required_libraries = Analysis Core MC Scalar Support Target TransformUtils - diff --git a/lib/CodeGen/LLVMTargetMachine.cpp b/lib/CodeGen/LLVMTargetMachine.cpp index 03b5693..62227fd 100644 --- a/lib/CodeGen/LLVMTargetMachine.cpp +++ b/lib/CodeGen/LLVMTargetMachine.cpp @@ -41,10 +41,6 @@ #include "llvm/Support/TargetRegistry.h" using namespace llvm; -namespace llvm { - bool EnableFastISel; -} - static cl::opt<bool> DisablePostRA("disable-post-ra", cl::Hidden, cl::desc("Disable Post Regalloc")); static cl::opt<bool> DisableBranchFold("disable-branch-fold", cl::Hidden, @@ -114,9 +110,10 @@ EnableFastISelOption("fast-isel", cl::Hidden, LLVMTargetMachine::LLVMTargetMachine(const Target &T, StringRef Triple, StringRef CPU, StringRef FS, + TargetOptions Options, Reloc::Model RM, CodeModel::Model CM, CodeGenOpt::Level OL) - : TargetMachine(T, Triple, CPU, FS) { + : TargetMachine(T, Triple, CPU, FS, Options) { CodeGenInfo = T.createMCCodeGenInfo(Triple, RM, CM, OL); AsmInfo = T.createMCAsmInfo(Triple); // TargetSelect.h moved to a different directory between LLVM 2.9 and 3.0, @@ -275,14 +272,15 @@ bool LLVMTargetMachine::addPassesToEmitMC(PassManagerBase &PM, return false; // success! } -static void printNoVerify(PassManagerBase &PM, const char *Banner) { - if (PrintMachineCode) +void LLVMTargetMachine::printNoVerify(PassManagerBase &PM, + const char *Banner) const { + if (Options.PrintMachineCode) PM.add(createMachineFunctionPrinterPass(dbgs(), Banner)); } -static void printAndVerify(PassManagerBase &PM, - const char *Banner) { - if (PrintMachineCode) +void LLVMTargetMachine::printAndVerify(PassManagerBase &PM, + const char *Banner) const { + if (Options.PrintMachineCode) PM.add(createMachineFunctionPrinterPass(dbgs(), Banner)); if (VerifyMachineCode) @@ -380,7 +378,7 @@ bool LLVMTargetMachine::addCommonCodeGenPasses(PassManagerBase &PM, if (EnableFastISelOption == cl::BOU_TRUE || (getOptLevel() == CodeGenOpt::None && EnableFastISelOption != cl::BOU_FALSE)) - EnableFastISel = true; + Options.EnableFastISel = true; // Ask the target for an isel. if (addInstSelector(PM)) diff --git a/lib/CodeGen/LiveDebugVariables.cpp b/lib/CodeGen/LiveDebugVariables.cpp index eb54baa7..c35302a 100644 --- a/lib/CodeGen/LiveDebugVariables.cpp +++ b/lib/CodeGen/LiveDebugVariables.cpp @@ -920,8 +920,8 @@ findInsertLocation(MachineBasicBlock *MBB, SlotIndex Idx, } // Don't insert anything after the first terminator, though. - return MI->getDesc().isTerminator() ? MBB->getFirstTerminator() : - llvm::next(MachineBasicBlock::iterator(MI)); + return MI->isTerminator() ? MBB->getFirstTerminator() : + llvm::next(MachineBasicBlock::iterator(MI)); } DebugLoc UserValue::findDebugLoc() { diff --git a/lib/CodeGen/LiveIntervalAnalysis.cpp b/lib/CodeGen/LiveIntervalAnalysis.cpp index edcfebe..1e58173 100644 --- a/lib/CodeGen/LiveIntervalAnalysis.cpp +++ b/lib/CodeGen/LiveIntervalAnalysis.cpp @@ -794,7 +794,7 @@ LiveIntervals::getLastSplitPoint(const LiveInterval &li, MachineBasicBlock::iterator I = mbb->end(), B = mbb->begin(); while (I != B) { --I; - if (I->getDesc().isCall()) + if (I->isCall()) return I; } // The block contains no calls that can throw, so use the first terminator. diff --git a/lib/CodeGen/LiveRangeEdit.cpp b/lib/CodeGen/LiveRangeEdit.cpp index 2f283b2..a470877 100644 --- a/lib/CodeGen/LiveRangeEdit.cpp +++ b/lib/CodeGen/LiveRangeEdit.cpp @@ -129,7 +129,7 @@ bool LiveRangeEdit::canRematerializeAt(Remat &RM, } // If only cheap remats were requested, bail out early. - if (cheapAsAMove && !RM.OrigMI->getDesc().isAsCheapAsAMove()) + if (cheapAsAMove && !RM.OrigMI->isAsCheapAsAMove()) return false; // Verify that all used registers are available with the same values. @@ -174,7 +174,7 @@ bool LiveRangeEdit::foldAsLoad(LiveInterval *LI, if (MO.isDef()) { if (DefMI && DefMI != MI) return false; - if (!MI->getDesc().canFoldAsLoad()) + if (!MI->canFoldAsLoad()) return false; DefMI = MI; } else if (!MO.isUndef()) { @@ -210,7 +210,8 @@ bool LiveRangeEdit::foldAsLoad(LiveInterval *LI, void LiveRangeEdit::eliminateDeadDefs(SmallVectorImpl<MachineInstr*> &Dead, LiveIntervals &LIS, VirtRegMap &VRM, - const TargetInstrInfo &TII) { + const TargetInstrInfo &TII, + ArrayRef<unsigned> RegsBeingSpilled) { SetVector<LiveInterval*, SmallVector<LiveInterval*, 8>, SmallPtrSet<LiveInterval*, 8> > ToShrink; @@ -290,6 +291,21 @@ void LiveRangeEdit::eliminateDeadDefs(SmallVectorImpl<MachineInstr*> &Dead, delegate_->LRE_WillShrinkVirtReg(LI->reg); if (!LIS.shrinkToUses(LI, &Dead)) continue; + + // Don't create new intervals for a register being spilled. + // The new intervals would have to be spilled anyway so its not worth it. + // Also they currently aren't spilled so creating them and not spilling + // them results in incorrect code. + bool BeingSpilled = false; + for (unsigned i = 0, e = RegsBeingSpilled.size(); i != e; ++i) { + if (LI->reg == RegsBeingSpilled[i]) { + BeingSpilled = true; + break; + } + } + + if (BeingSpilled) continue; + // LI may have been separated, create new intervals. LI->RenumberValues(LIS); diff --git a/lib/CodeGen/LiveRangeEdit.h b/lib/CodeGen/LiveRangeEdit.h index 9b0a671..057d9bb 100644 --- a/lib/CodeGen/LiveRangeEdit.h +++ b/lib/CodeGen/LiveRangeEdit.h @@ -191,9 +191,14 @@ public: /// eliminateDeadDefs - Try to delete machine instructions that are now dead /// (allDefsAreDead returns true). This may cause live intervals to be trimmed /// and further dead efs to be eliminated. + /// RegsBeingSpilled lists registers currently being spilled by the register + /// allocator. These registers should not be split into new intervals + /// as currently those new intervals are not guaranteed to spill. void eliminateDeadDefs(SmallVectorImpl<MachineInstr*> &Dead, LiveIntervals&, VirtRegMap&, - const TargetInstrInfo&); + const TargetInstrInfo&, + ArrayRef<unsigned> RegsBeingSpilled + = ArrayRef<unsigned>()); /// calculateRegClassAndHint - Recompute register class and hint for each new /// register. diff --git a/lib/CodeGen/LiveVariables.cpp b/lib/CodeGen/LiveVariables.cpp index 2ca90f9..7477d91 100644 --- a/lib/CodeGen/LiveVariables.cpp +++ b/lib/CodeGen/LiveVariables.cpp @@ -590,8 +590,8 @@ bool LiveVariables::runOnMachineFunction(MachineFunction &mf) { // them. The tail callee need not take the same registers as input // that it produces as output, and there are dependencies for its input // registers elsewhere. - if (!MBB->empty() && MBB->back().getDesc().isReturn() - && !MBB->back().getDesc().isCall()) { + if (!MBB->empty() && MBB->back().isReturn() + && !MBB->back().isCall()) { MachineInstr *Ret = &MBB->back(); for (MachineRegisterInfo::liveout_iterator @@ -754,7 +754,7 @@ void LiveVariables::addNewBlock(MachineBasicBlock *BB, const unsigned NumNew = BB->getNumber(); // All registers used by PHI nodes in SuccBB must be live through BB. - for (MachineBasicBlock::const_iterator BBI = SuccBB->begin(), + for (MachineBasicBlock::iterator BBI = SuccBB->begin(), BBE = SuccBB->end(); BBI != BBE && BBI->isPHI(); ++BBI) for (unsigned i = 1, e = BBI->getNumOperands(); i != e; i += 2) if (BBI->getOperand(i+1).getMBB() == BB) diff --git a/lib/CodeGen/MachineBasicBlock.cpp b/lib/CodeGen/MachineBasicBlock.cpp index b9d1ef7..6734916 100644 --- a/lib/CodeGen/MachineBasicBlock.cpp +++ b/lib/CodeGen/MachineBasicBlock.cpp @@ -73,7 +73,8 @@ void ilist_traits<MachineBasicBlock>::addNodeToList(MachineBasicBlock *N) { // Make sure the instructions have their operands in the reginfo lists. MachineRegisterInfo &RegInfo = MF.getRegInfo(); - for (MachineBasicBlock::iterator I = N->begin(), E = N->end(); I != E; ++I) + for (MachineBasicBlock::instr_iterator + I = N->instr_begin(), E = N->instr_end(); I != E; ++I) I->AddRegOperandsToUseLists(RegInfo); LeakDetector::removeGarbageObject(N); @@ -120,8 +121,8 @@ void ilist_traits<MachineInstr>::removeNodeFromList(MachineInstr *N) { /// lists. void ilist_traits<MachineInstr>:: transferNodesFromList(ilist_traits<MachineInstr> &fromList, - MachineBasicBlock::iterator first, - MachineBasicBlock::iterator last) { + ilist_iterator<MachineInstr> first, + ilist_iterator<MachineInstr> last) { assert(Parent->getParent() == fromList.Parent->getParent() && "MachineInstr parent mismatch!"); @@ -140,9 +141,10 @@ void ilist_traits<MachineInstr>::deleteNode(MachineInstr* MI) { } MachineBasicBlock::iterator MachineBasicBlock::getFirstNonPHI() { - iterator I = begin(); + instr_iterator I = instr_begin(); while (I != end() && I->isPHI()) ++I; + assert(!I->isInsideBundle() && "First non-phi MI cannot be inside a bundle!"); return I; } @@ -150,23 +152,63 @@ MachineBasicBlock::iterator MachineBasicBlock::SkipPHIsAndLabels(MachineBasicBlock::iterator I) { while (I != end() && (I->isPHI() || I->isLabel() || I->isDebugValue())) ++I; + // FIXME: This needs to change if we wish to bundle labels / dbg_values + // inside the bundle. + assert(!I->isInsideBundle() && + "First non-phi / non-label instruction is inside a bundle!"); return I; } MachineBasicBlock::iterator MachineBasicBlock::getFirstTerminator() { iterator I = end(); - while (I != begin() && ((--I)->getDesc().isTerminator() || I->isDebugValue())) + while (I != begin() && ((--I)->isTerminator() || I->isDebugValue())) ; /*noop */ - while (I != end() && !I->getDesc().isTerminator()) + while (I != end() && !I->isTerminator()) + ++I; + return I; +} + +MachineBasicBlock::const_iterator +MachineBasicBlock::getFirstTerminator() const { + const_iterator I = end(); + while (I != begin() && ((--I)->isTerminator() || I->isDebugValue())) + ; /*noop */ + while (I != end() && !I->isTerminator()) + ++I; + return I; +} + +MachineBasicBlock::instr_iterator MachineBasicBlock::getFirstInstrTerminator() { + instr_iterator I = instr_end(); + while (I != instr_begin() && ((--I)->isTerminator() || I->isDebugValue())) + ; /*noop */ + while (I != instr_end() && !I->isTerminator()) ++I; return I; } MachineBasicBlock::iterator MachineBasicBlock::getLastNonDebugInstr() { - iterator B = begin(), I = end(); + // Skip over end-of-block dbg_value instructions. + instr_iterator B = instr_begin(), I = instr_end(); while (I != B) { --I; - if (I->isDebugValue()) + // Return instruction that starts a bundle. + if (I->isDebugValue() || I->isInsideBundle()) + continue; + return I; + } + // The block is all debug values. + return end(); +} + +MachineBasicBlock::const_iterator +MachineBasicBlock::getLastNonDebugInstr() const { + // Skip over end-of-block dbg_value instructions. + const_instr_iterator B = instr_begin(), I = instr_end(); + while (I != B) { + --I; + // Return instruction that starts a bundle. + if (I->isDebugValue() || I->isInsideBundle()) continue; return I; } @@ -203,8 +245,6 @@ void MachineBasicBlock::print(raw_ostream &OS, SlotIndexes *Indexes) const { return; } - if (Alignment) { OS << "Alignment " << Alignment << "\n"; } - if (Indexes) OS << Indexes->getMBBStartIdx(this) << '\t'; @@ -218,6 +258,12 @@ void MachineBasicBlock::print(raw_ostream &OS, SlotIndexes *Indexes) const { } if (isLandingPad()) { OS << Comma << "EH LANDING PAD"; Comma = ", "; } if (hasAddressTaken()) { OS << Comma << "ADDRESS TAKEN"; Comma = ", "; } + if (Alignment) { + OS << Comma << "Align " << Alignment << " (" << (1u << Alignment) + << " bytes)"; + Comma = ", "; + } + OS << '\n'; const TargetRegisterInfo *TRI = MF->getTarget().getRegisterInfo(); @@ -237,13 +283,15 @@ void MachineBasicBlock::print(raw_ostream &OS, SlotIndexes *Indexes) const { OS << '\n'; } - for (const_iterator I = begin(); I != end(); ++I) { + for (const_instr_iterator I = instr_begin(); I != instr_end(); ++I) { if (Indexes) { if (Indexes->hasIndex(I)) OS << Indexes->getInstructionIndex(I); OS << '\t'; } OS << '\t'; + if (I->isInsideBundle()) + OS << " * "; I->print(OS, &getParent()->getTarget()); } @@ -449,8 +497,8 @@ MachineBasicBlock::transferSuccessorsAndUpdatePHIs(MachineBasicBlock *fromMBB) { fromMBB->removeSuccessor(Succ); // Fix up any PHI nodes in the successor. - for (MachineBasicBlock::iterator MI = Succ->begin(), ME = Succ->end(); - MI != ME && MI->isPHI(); ++MI) + for (MachineBasicBlock::instr_iterator MI = Succ->instr_begin(), + ME = Succ->instr_end(); MI != ME && MI->isPHI(); ++MI) for (unsigned i = 2, e = MI->getNumOperands()+1; i != e; i += 2) { MachineOperand &MO = MI->getOperand(i); if (MO.getMBB() == fromMBB) @@ -492,8 +540,8 @@ bool MachineBasicBlock::canFallThrough() { // Barrier is predicated and thus no longer an actual control barrier. This // is over-conservative though, because if an instruction isn't actually // predicated we could still treat it like a barrier. - return empty() || !back().getDesc().isBarrier() || - back().getDesc().isPredicable(); + return empty() || !back().isBarrier() || + back().isPredicable(); } // If there is no branch, control always falls through. @@ -552,7 +600,8 @@ MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { // Collect a list of virtual registers killed by the terminators. SmallVector<unsigned, 4> KilledRegs; if (LV) - for (iterator I = getFirstTerminator(), E = end(); I != E; ++I) { + for (instr_iterator I = getFirstInstrTerminator(), E = instr_end(); + I != E; ++I) { MachineInstr *MI = I; for (MachineInstr::mop_iterator OI = MI->operands_begin(), OE = MI->operands_end(); OI != OE; ++OI) { @@ -579,7 +628,8 @@ MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { } // Fix PHI nodes in Succ so they refer to NMBB instead of this - for (MachineBasicBlock::iterator i = Succ->begin(), e = Succ->end(); + for (MachineBasicBlock::instr_iterator + i = Succ->instr_begin(),e = Succ->instr_end(); i != e && i->isPHI(); ++i) for (unsigned ni = 1, ne = i->getNumOperands(); ni != ne; ni += 2) if (i->getOperand(ni+1).getMBB() == this) @@ -595,7 +645,7 @@ MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { // Restore kills of virtual registers that were killed by the terminators. while (!KilledRegs.empty()) { unsigned Reg = KilledRegs.pop_back_val(); - for (iterator I = end(), E = begin(); I != E;) { + for (instr_iterator I = instr_end(), E = instr_begin(); I != E;) { if (!(--I)->addRegisterKilled(Reg, NULL, /* addIfNotFound= */ false)) continue; LV->getVarInfo(Reg).Kills.push_back(I); @@ -664,6 +714,41 @@ MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { return NMBB; } +MachineBasicBlock::iterator +MachineBasicBlock::erase(MachineBasicBlock::iterator I) { + if (I->isBundle()) { + MachineBasicBlock::iterator E = llvm::next(I); + return Insts.erase(I.getInstrIterator(), E.getInstrIterator()); + } + + return Insts.erase(I.getInstrIterator()); +} + +MachineInstr *MachineBasicBlock::remove(MachineInstr *I) { + if (I->isBundle()) { + MachineBasicBlock::instr_iterator MII = I; ++MII; + while (MII != end() && MII->isInsideBundle()) { + MachineInstr *MI = &*MII++; + Insts.remove(MI); + } + } + + return Insts.remove(I); +} + +void MachineBasicBlock::splice(MachineBasicBlock::iterator where, + MachineBasicBlock *Other, + MachineBasicBlock::iterator From) { + if (From->isBundle()) { + MachineBasicBlock::iterator To = llvm::next(From); + Insts.splice(where.getInstrIterator(), Other->Insts, + From.getInstrIterator(), To.getInstrIterator()); + return; + } + + Insts.splice(where.getInstrIterator(), Other->Insts, From.getInstrIterator()); +} + /// removeFromParent - This method unlinks 'this' from the containing function, /// and returns it, but does not delete it. MachineBasicBlock *MachineBasicBlock::removeFromParent() { @@ -687,10 +772,10 @@ void MachineBasicBlock::ReplaceUsesOfBlockWith(MachineBasicBlock *Old, MachineBasicBlock *New) { assert(Old != New && "Cannot replace self with self!"); - MachineBasicBlock::iterator I = end(); - while (I != begin()) { + MachineBasicBlock::instr_iterator I = instr_end(); + while (I != instr_begin()) { --I; - if (!I->getDesc().isTerminator()) break; + if (!I->isTerminator()) break; // Scan the operands of this machine instruction, replacing any uses of Old // with New. @@ -769,17 +854,17 @@ bool MachineBasicBlock::CorrectExtraCFGEdges(MachineBasicBlock *DestA, /// findDebugLoc - find the next valid DebugLoc starting at MBBI, skipping /// any DBG_VALUE instructions. Return UnknownLoc if there is none. DebugLoc -MachineBasicBlock::findDebugLoc(MachineBasicBlock::iterator &MBBI) { +MachineBasicBlock::findDebugLoc(instr_iterator MBBI) { DebugLoc DL; - MachineBasicBlock::iterator E = end(); - if (MBBI != E) { - // Skip debug declarations, we don't want a DebugLoc from them. - MachineBasicBlock::iterator MBBI2 = MBBI; - while (MBBI2 != E && MBBI2->isDebugValue()) - MBBI2++; - if (MBBI2 != E) - DL = MBBI2->getDebugLoc(); - } + instr_iterator E = instr_end(); + if (MBBI == E) + return DL; + + // Skip debug declarations, we don't want a DebugLoc from them. + while (MBBI != E && MBBI->isDebugValue()) + MBBI++; + if (MBBI != E) + DL = MBBI->getDebugLoc(); return DL; } diff --git a/lib/CodeGen/MachineBlockPlacement.cpp b/lib/CodeGen/MachineBlockPlacement.cpp index 55d804b..638d895 100644 --- a/lib/CodeGen/MachineBlockPlacement.cpp +++ b/lib/CodeGen/MachineBlockPlacement.cpp @@ -36,10 +36,7 @@ #include "llvm/CodeGen/Passes.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/PostOrderIterator.h" -#include "llvm/ADT/SCCIterator.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" @@ -56,22 +53,6 @@ STATISTIC(UncondBranchTakenFreq, "Potential frequency of taking unconditional branches"); namespace { -/// \brief A structure for storing a weighted edge. -/// -/// This stores an edge and its weight, computed as the product of the -/// frequency that the starting block is entered with the probability of -/// a particular exit block. -struct WeightedEdge { - BlockFrequency EdgeFrequency; - MachineBasicBlock *From, *To; - - bool operator<(const WeightedEdge &RHS) const { - return EdgeFrequency < RHS.EdgeFrequency; - } -}; -} - -namespace { class BlockChain; /// \brief Type for our function-wide basic block -> block chain mapping. typedef DenseMap<MachineBasicBlock *, BlockChain *> BlockToChainMapType; @@ -222,6 +203,9 @@ class MachineBlockPlacement : public MachineFunctionPass { void buildChain(MachineBasicBlock *BB, BlockChain &Chain, SmallVectorImpl<MachineBasicBlock *> &BlockWorkList, const BlockFilterSet *BlockFilter = 0); + MachineBasicBlock *findBestLoopTop(MachineFunction &F, + MachineLoop &L, + const BlockFilterSet &LoopBlockSet); void buildLoopChains(MachineFunction &F, MachineLoop &L); void buildCFGChains(MachineFunction &F); void AlignLoops(MachineFunction &F); @@ -546,12 +530,134 @@ void MachineBlockPlacement::buildChain( markChainSuccessors(SuccChain, LoopHeaderBB, BlockWorkList, BlockFilter); Chain.merge(BestSucc, &SuccChain); BB = *llvm::prior(Chain.end()); - }; + } DEBUG(dbgs() << "Finished forming chain for header block " << getBlockNum(*Chain.begin()) << "\n"); } +/// \brief Find the best loop top block for layout. +/// +/// This routine implements the logic to analyze the loop looking for the best +/// block to layout at the top of the loop. Typically this is done to maximize +/// fallthrough opportunities. +MachineBasicBlock * +MachineBlockPlacement::findBestLoopTop(MachineFunction &F, + MachineLoop &L, + const BlockFilterSet &LoopBlockSet) { + BlockFrequency BestExitEdgeFreq; + MachineBasicBlock *ExitingBB = 0; + MachineBasicBlock *LoopingBB = 0; + // If there are exits to outer loops, loop rotation can severely limit + // fallthrough opportunites unless it selects such an exit. Keep a set of + // blocks where rotating to exit with that block will reach an outer loop. + SmallPtrSet<MachineBasicBlock *, 4> BlocksExitingToOuterLoop; + + DEBUG(dbgs() << "Finding best loop exit for: " + << getBlockName(L.getHeader()) << "\n"); + for (MachineLoop::block_iterator I = L.block_begin(), + E = L.block_end(); + I != E; ++I) { + BlockChain &Chain = *BlockToChain[*I]; + // Ensure that this block is at the end of a chain; otherwise it could be + // mid-way through an inner loop or a successor of an analyzable branch. + if (*I != *llvm::prior(Chain.end())) + continue; + + // Now walk the successors. We need to establish whether this has a viable + // exiting successor and whether it has a viable non-exiting successor. + // We store the old exiting state and restore it if a viable looping + // successor isn't found. + MachineBasicBlock *OldExitingBB = ExitingBB; + BlockFrequency OldBestExitEdgeFreq = BestExitEdgeFreq; + // We also compute and store the best looping successor for use in layout. + MachineBasicBlock *BestLoopSucc = 0; + // FIXME: Due to the performance of the probability and weight routines in + // the MBPI analysis, we use the internal weights. This is only valid + // because it is purely a ranking function, we don't care about anything + // but the relative values. + uint32_t BestLoopSuccWeight = 0; + // FIXME: We also manually compute the probabilities to avoid quadratic + // behavior. + uint32_t WeightScale = 0; + uint32_t SumWeight = MBPI->getSumForBlock(*I, WeightScale); + for (MachineBasicBlock::succ_iterator SI = (*I)->succ_begin(), + SE = (*I)->succ_end(); + SI != SE; ++SI) { + if ((*SI)->isLandingPad()) + continue; + if (*SI == *I) + continue; + BlockChain &SuccChain = *BlockToChain[*SI]; + // Don't split chains, either this chain or the successor's chain. + if (&Chain == &SuccChain || *SI != *SuccChain.begin()) { + DEBUG(dbgs() << " " << (LoopBlockSet.count(*SI) ? "looping: " + : "exiting: ") + << getBlockName(*I) << " -> " + << getBlockName(*SI) << " (chain conflict)\n"); + continue; + } + + uint32_t SuccWeight = MBPI->getEdgeWeight(*I, *SI); + if (LoopBlockSet.count(*SI)) { + DEBUG(dbgs() << " looping: " << getBlockName(*I) << " -> " + << getBlockName(*SI) << " (" << SuccWeight << ")\n"); + if (BestLoopSucc && BestLoopSuccWeight >= SuccWeight) + continue; + + BestLoopSucc = *SI; + BestLoopSuccWeight = SuccWeight; + continue; + } + + BranchProbability SuccProb(SuccWeight / WeightScale, SumWeight); + BlockFrequency ExitEdgeFreq = MBFI->getBlockFreq(*I) * SuccProb; + DEBUG(dbgs() << " exiting: " << getBlockName(*I) << " -> " + << getBlockName(*SI) << " (" << ExitEdgeFreq << ")\n"); + // Note that we slightly bias this toward an existing layout successor to + // retain incoming order in the absence of better information. + // FIXME: Should we bias this more strongly? It's pretty weak. + if (!ExitingBB || ExitEdgeFreq > BestExitEdgeFreq || + ((*I)->isLayoutSuccessor(*SI) && + !(ExitEdgeFreq < BestExitEdgeFreq))) { + BestExitEdgeFreq = ExitEdgeFreq; + ExitingBB = *I; + } + + if (MachineLoop *ExitLoop = MLI->getLoopFor(*SI)) + if (ExitLoop->contains(&L)) + BlocksExitingToOuterLoop.insert(*I); + } + + // Restore the old exiting state, no viable looping successor was found. + if (!BestLoopSucc) { + ExitingBB = OldExitingBB; + BestExitEdgeFreq = OldBestExitEdgeFreq; + continue; + } + + // If this was best exiting block thus far, also record the looping block. + if (ExitingBB == *I) + LoopingBB = BestLoopSucc; + } + // Without a candidate exitting block or with only a single block in the + // loop, just use the loop header to layout the loop. + if (!ExitingBB || L.getNumBlocks() == 1) + return L.getHeader(); + + // Also, if we have exit blocks which lead to outer loops but didn't select + // one of them as the exiting block we are rotating toward, disable loop + // rotation altogether. + if (!BlocksExitingToOuterLoop.empty() && + !BlocksExitingToOuterLoop.count(ExitingBB)) + return L.getHeader(); + + assert(LoopingBB && "All successors of a loop block are exit blocks!"); + DEBUG(dbgs() << " Best exiting block: " << getBlockName(ExitingBB) << "\n"); + DEBUG(dbgs() << " Best top block: " << getBlockName(LoopingBB) << "\n"); + return LoopingBB; +} + /// \brief Forms basic block chains from the natural loop structures. /// /// These chains are designed to preserve the existing *structure* of the code @@ -567,17 +673,21 @@ void MachineBlockPlacement::buildLoopChains(MachineFunction &F, SmallVector<MachineBasicBlock *, 16> BlockWorkList; BlockFilterSet LoopBlockSet(L.block_begin(), L.block_end()); - BlockChain &LoopChain = *BlockToChain[L.getHeader()]; + + MachineBasicBlock *LayoutTop = findBestLoopTop(F, L, LoopBlockSet); + BlockChain &LoopChain = *BlockToChain[LayoutTop]; // FIXME: This is a really lame way of walking the chains in the loop: we // walk the blocks, and use a set to prevent visiting a particular chain // twice. SmallPtrSet<BlockChain *, 4> UpdatedPreds; + assert(LoopChain.LoopPredecessors == 0); + UpdatedPreds.insert(&LoopChain); for (MachineLoop::block_iterator BI = L.block_begin(), BE = L.block_end(); BI != BE; ++BI) { BlockChain &Chain = *BlockToChain[*BI]; - if (!UpdatedPreds.insert(&Chain) || BI == L.block_begin()) + if (!UpdatedPreds.insert(&Chain)) continue; assert(Chain.LoopPredecessors == 0); @@ -597,7 +707,7 @@ void MachineBlockPlacement::buildLoopChains(MachineFunction &F, BlockWorkList.push_back(*Chain.begin()); } - buildChain(*L.block_begin(), LoopChain, BlockWorkList, &LoopBlockSet); + buildChain(LayoutTop, LoopChain, BlockWorkList, &LoopBlockSet); DEBUG({ // Crash at the end so we get all of the debugging output first. diff --git a/lib/CodeGen/MachineCSE.cpp b/lib/CodeGen/MachineCSE.cpp index 7eda8c1..8c02cd7 100644 --- a/lib/CodeGen/MachineCSE.cpp +++ b/lib/CodeGen/MachineCSE.cpp @@ -260,12 +260,11 @@ bool MachineCSE::isCSECandidate(MachineInstr *MI) { return false; // Ignore stuff that we obviously can't move. - const MCInstrDesc &MCID = MI->getDesc(); - if (MCID.mayStore() || MCID.isCall() || MCID.isTerminator() || + if (MI->mayStore() || MI->isCall() || MI->isTerminator() || MI->hasUnmodeledSideEffects()) return false; - if (MCID.mayLoad()) { + if (MI->mayLoad()) { // Okay, this instruction does a load. As a refinement, we allow the target // to decide whether the loaded value is actually a constant. If so, we can // actually use it as a load. @@ -287,7 +286,7 @@ bool MachineCSE::isProfitableToCSE(unsigned CSReg, unsigned Reg, // Heuristics #1: Don't CSE "cheap" computation if the def is not local or in // an immediate predecessor. We don't want to increase register pressure and // end up causing other computation to be spilled. - if (MI->getDesc().isAsCheapAsAMove()) { + if (MI->isAsCheapAsAMove()) { MachineBasicBlock *CSBB = CSMI->getParent(); MachineBasicBlock *BB = MI->getParent(); if (CSBB != BB && !CSBB->isSuccessor(BB)) @@ -376,7 +375,7 @@ bool MachineCSE::ProcessBlock(MachineBasicBlock *MBB) { // Commute commutable instructions. bool Commuted = false; - if (!FoundCSE && MI->getDesc().isCommutable()) { + if (!FoundCSE && MI->isCommutable()) { MachineInstr *NewMI = TII->commuteInstruction(MI); if (NewMI) { Commuted = true; diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp index b0ef9d4..ec5a1cd 100644 --- a/lib/CodeGen/MachineInstr.cpp +++ b/lib/CodeGen/MachineInstr.cpp @@ -178,6 +178,7 @@ void MachineOperand::ChangeToRegister(unsigned Reg, bool isDef, bool isImp, IsKill = isKill; IsDead = isDead; IsUndef = isUndef; + IsInternalRead = false; IsEarlyClobber = false; IsDebug = isDebug; SubReg = 0; @@ -240,7 +241,7 @@ void MachineOperand::print(raw_ostream &OS, const TargetMachine *TM) const { OS << PrintReg(getReg(), TRI, getSubReg()); if (isDef() || isKill() || isDead() || isImplicit() || isUndef() || - isEarlyClobber()) { + isInternalRead() || isEarlyClobber()) { OS << '<'; bool NeedComma = false; if (isDef()) { @@ -256,14 +257,26 @@ void MachineOperand::print(raw_ostream &OS, const TargetMachine *TM) const { NeedComma = true; } - if (isKill() || isDead() || isUndef()) { + if (isKill() || isDead() || isUndef() || isInternalRead()) { if (NeedComma) OS << ','; - if (isKill()) OS << "kill"; - if (isDead()) OS << "dead"; + NeedComma = false; + if (isKill()) { + OS << "kill"; + NeedComma = true; + } + if (isDead()) { + OS << "dead"; + NeedComma = true; + } if (isUndef()) { - if (isKill() || isDead()) - OS << ','; + if (NeedComma) OS << ','; OS << "undef"; + NeedComma = true; + } + if (isInternalRead()) { + if (NeedComma) OS << ','; + OS << "internal"; + NeedComma = true; } } OS << '>'; @@ -735,6 +748,27 @@ void MachineInstr::addMemOperand(MachineFunction &MF, MemRefsEnd = NewMemRefsEnd; } +bool +MachineInstr::hasProperty(unsigned MCFlag, QueryType Type) const { + if (Type == IgnoreBundle || !isBundle()) + return getDesc().getFlags() & (1 << MCFlag); + + const MachineBasicBlock *MBB = getParent(); + MachineBasicBlock::const_instr_iterator MII = *this; ++MII; + while (MII != MBB->end() && MII->isInsideBundle()) { + if (MII->getDesc().getFlags() & (1 << MCFlag)) { + if (Type == AnyInBundle) + return true; + } else { + if (Type == AllInBundle) + return false; + } + ++MII; + } + + return Type == AllInBundle; +} + bool MachineInstr::isIdenticalTo(const MachineInstr *Other, MICheckType Check) const { // If opcodes or number of operands are not the same then the two @@ -743,6 +777,19 @@ bool MachineInstr::isIdenticalTo(const MachineInstr *Other, Other->getNumOperands() != getNumOperands()) return false; + if (isBundle()) { + // Both instructions are bundles, compare MIs inside the bundle. + MachineBasicBlock::const_instr_iterator I1 = *this; + MachineBasicBlock::const_instr_iterator E1 = getParent()->instr_end(); + MachineBasicBlock::const_instr_iterator I2 = *Other; + MachineBasicBlock::const_instr_iterator E2= Other->getParent()->instr_end(); + while (++I1 != E1 && I1->isInsideBundle()) { + ++I2; + if (I2 == E2 || !I2->isInsideBundle() || !I1->isIdenticalTo(I2, Check)) + return false; + } + } + // Check operands to make sure they match. for (unsigned i = 0, e = getNumOperands(); i != e; ++i) { const MachineOperand &MO = getOperand(i); @@ -789,6 +836,18 @@ bool MachineInstr::isIdenticalTo(const MachineInstr *Other, /// block, and returns it, but does not delete it. MachineInstr *MachineInstr::removeFromParent() { assert(getParent() && "Not embedded in a basic block!"); + + // If it's a bundle then remove the MIs inside the bundle as well. + if (isBundle()) { + MachineBasicBlock *MBB = getParent(); + MachineBasicBlock::instr_iterator MII = *this; ++MII; + MachineBasicBlock::instr_iterator E = MBB->instr_end(); + while (MII != E && MII->isInsideBundle()) { + MachineInstr *MI = &*MII; + ++MII; + MBB->remove(MI); + } + } getParent()->remove(this); return this; } @@ -798,6 +857,17 @@ MachineInstr *MachineInstr::removeFromParent() { /// block, and deletes it. void MachineInstr::eraseFromParent() { assert(getParent() && "Not embedded in a basic block!"); + // If it's a bundle then remove the MIs inside the bundle as well. + if (isBundle()) { + MachineBasicBlock *MBB = getParent(); + MachineBasicBlock::instr_iterator MII = *this; ++MII; + MachineBasicBlock::instr_iterator E = MBB->instr_end(); + while (MII != E && MII->isInsideBundle()) { + MachineInstr *MI = &*MII; + ++MII; + MBB->erase(MI); + } + } getParent()->erase(this); } @@ -887,6 +957,20 @@ MachineInstr::getRegClassConstraint(unsigned OpIdx, return NULL; } +/// getBundleSize - Return the number of instructions inside the MI bundle. +unsigned MachineInstr::getBundleSize() const { + assert(isBundle() && "Expecting a bundle"); + + MachineBasicBlock::const_instr_iterator I = *this; + unsigned Size = 0; + while ((++I)->isInsideBundle()) { + ++Size; + } + assert(Size > 1 && "Malformed bundle"); + + return Size; +} + /// findRegisterUseOperandIdx() - Returns the MachineOperand that is a use of /// the specific register or -1 if it is not found. It further tightens /// the search criteria to a use that kills the register if isKill is true. @@ -1118,6 +1202,8 @@ void MachineInstr::copyKillDeadInfo(const MachineInstr *MI) { /// copyPredicates - Copies predicate operand(s) from MI. void MachineInstr::copyPredicates(const MachineInstr *MI) { + assert(!isBundle() && "MachineInstr::copyPredicates() can't handle bundles"); + const MCInstrDesc &MCID = MI->getDesc(); if (!MCID.isPredicable()) return; @@ -1159,13 +1245,13 @@ bool MachineInstr::isSafeToMove(const TargetInstrInfo *TII, AliasAnalysis *AA, bool &SawStore) const { // Ignore stuff that we obviously can't move. - if (MCID->mayStore() || MCID->isCall()) { + if (mayStore() || isCall()) { SawStore = true; return false; } if (isLabel() || isDebugValue() || - MCID->isTerminator() || hasUnmodeledSideEffects()) + isTerminator() || hasUnmodeledSideEffects()) return false; // See if this instruction does a load. If so, we have to guarantee that the @@ -1173,7 +1259,7 @@ bool MachineInstr::isSafeToMove(const TargetInstrInfo *TII, // destination. The check for isInvariantLoad gives the targe the chance to // classify the load as always returning a constant, e.g. a constant pool // load. - if (MCID->mayLoad() && !isInvariantLoad(AA)) + if (mayLoad() && !isInvariantLoad(AA)) // Otherwise, this is a real load. If there is a store between the load and // end of block, or if the load is volatile, we can't move it. return !SawStore && !hasVolatileMemoryRef(); @@ -1213,9 +1299,9 @@ bool MachineInstr::isSafeToReMat(const TargetInstrInfo *TII, /// have no volatile memory references. bool MachineInstr::hasVolatileMemoryRef() const { // An instruction known never to access memory won't have a volatile access. - if (!MCID->mayStore() && - !MCID->mayLoad() && - !MCID->isCall() && + if (!mayStore() && + !mayLoad() && + !isCall() && !hasUnmodeledSideEffects()) return false; @@ -1239,7 +1325,7 @@ bool MachineInstr::hasVolatileMemoryRef() const { /// *all* loads the instruction does are invariant (if it does multiple loads). bool MachineInstr::isInvariantLoad(AliasAnalysis *AA) const { // If the instruction doesn't load at all, it isn't an invariant load. - if (!MCID->mayLoad()) + if (!mayLoad()) return false; // If the instruction has lost its memoperands, conservatively assume that @@ -1292,7 +1378,7 @@ unsigned MachineInstr::isConstantValuePHI() const { } bool MachineInstr::hasUnmodeledSideEffects() const { - if (getDesc().hasUnmodeledSideEffects()) + if (hasProperty(MCID::UnmodeledSideEffects)) return true; if (isInlineAsm()) { unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); @@ -1420,7 +1506,7 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const { // call instructions much less noisy on targets where calls clobber lots // of registers. Don't rely on MO.isDead() because we may be called before // LiveVariables is run, or we may be looking at a non-allocatable reg. - if (MF && getDesc().isCall() && + if (MF && isCall() && MO.isReg() && MO.isImplicit() && MO.isDef()) { unsigned Reg = MO.getReg(); if (TargetRegisterInfo::isPhysicalRegister(Reg)) { diff --git a/lib/CodeGen/MachineInstrBundle.cpp b/lib/CodeGen/MachineInstrBundle.cpp new file mode 100644 index 0000000..b766d08 --- /dev/null +++ b/lib/CodeGen/MachineInstrBundle.cpp @@ -0,0 +1,180 @@ +//===-- lib/CodeGen/MachineInstrBundle.cpp --------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/MachineInstrBundle.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +using namespace llvm; + +namespace { + class UnpackMachineBundles : public MachineFunctionPass { + public: + static char ID; // Pass identification + UnpackMachineBundles() : MachineFunctionPass(ID) { + initializeUnpackMachineBundlesPass(*PassRegistry::getPassRegistry()); + } + + virtual bool runOnMachineFunction(MachineFunction &MF); + }; +} // end anonymous namespace + +char UnpackMachineBundles::ID = 0; +INITIALIZE_PASS(UnpackMachineBundles, "unpack-mi-bundle", + "Unpack machine instruction bundles", false, false) + +FunctionPass *llvm::createUnpackMachineBundlesPass() { + return new UnpackMachineBundles(); +} + +bool UnpackMachineBundles::runOnMachineFunction(MachineFunction &MF) { + bool Changed = false; + for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) { + MachineBasicBlock *MBB = &*I; + + for (MachineBasicBlock::instr_iterator MII = MBB->instr_begin(), + MIE = MBB->instr_end(); MII != MIE; ) { + MachineInstr *MI = &*MII; + + // Remove BUNDLE instruction and the InsideBundle flags from bundled + // instructions. + if (MI->isBundle()) { + while (++MII != MIE && MII->isInsideBundle()) { + MII->setIsInsideBundle(false); + for (unsigned i = 0, e = MII->getNumOperands(); i != e; ++i) { + MachineOperand &MO = MII->getOperand(i); + if (MO.isReg() && MO.isInternalRead()) + MO.setIsInternalRead(false); + } + } + MI->eraseFromParent(); + + Changed = true; + continue; + } + + ++MII; + } + } + + return Changed; +} + +/// FinalizeBundle - Finalize a machine instruction bundle which includes +/// a sequence of instructions starting from FirstMI to LastMI (inclusive). +/// This routine adds a BUNDLE instruction to represent the bundle, it adds +/// IsInternalRead markers to MachineOperands which are defined inside the +/// bundle, and it copies externally visible defs and uses to the BUNDLE +/// instruction. +void llvm::FinalizeBundle(MachineBasicBlock &MBB, + MachineBasicBlock::instr_iterator FirstMI, + MachineBasicBlock::instr_iterator LastMI) { + const TargetMachine &TM = MBB.getParent()->getTarget(); + const TargetInstrInfo *TII = TM.getInstrInfo(); + const TargetRegisterInfo *TRI = TM.getRegisterInfo(); + + MachineInstrBuilder MIB = BuildMI(MBB, FirstMI, FirstMI->getDebugLoc(), + TII->get(TargetOpcode::BUNDLE)); + + SmallVector<unsigned, 8> LocalDefs; + SmallSet<unsigned, 8> LocalDefSet; + SmallSet<unsigned, 8> DeadDefSet; + SmallSet<unsigned, 8> KilledDefSet; + SmallVector<unsigned, 8> ExternUses; + SmallSet<unsigned, 8> ExternUseSet; + SmallSet<unsigned, 8> KilledUseSet; + SmallSet<unsigned, 8> UndefUseSet; + SmallVector<MachineOperand*, 4> Defs; + do { + for (unsigned i = 0, e = FirstMI->getNumOperands(); i != e; ++i) { + MachineOperand &MO = FirstMI->getOperand(i); + if (!MO.isReg()) + continue; + if (MO.isDef()) { + Defs.push_back(&MO); + continue; + } + + unsigned Reg = MO.getReg(); + if (!Reg) + continue; + assert(TargetRegisterInfo::isPhysicalRegister(Reg)); + if (LocalDefSet.count(Reg)) { + MO.setIsInternalRead(); + if (MO.isKill()) + // Internal def is now killed. + KilledDefSet.insert(Reg); + } else { + if (ExternUseSet.insert(Reg)) { + ExternUses.push_back(Reg); + if (MO.isUndef()) + UndefUseSet.insert(Reg); + } + if (MO.isKill()) + // External def is now killed. + KilledUseSet.insert(Reg); + } + } + + for (unsigned i = 0, e = Defs.size(); i != e; ++i) { + MachineOperand &MO = *Defs[i]; + unsigned Reg = MO.getReg(); + if (!Reg) + continue; + + if (LocalDefSet.insert(Reg)) { + LocalDefs.push_back(Reg); + if (MO.isDead()) { + DeadDefSet.insert(Reg); + } + } else { + // Re-defined inside the bundle, it's no longer killed. + KilledDefSet.erase(Reg); + if (!MO.isDead()) + // Previously defined but dead. + DeadDefSet.erase(Reg); + } + + if (!MO.isDead()) { + for (const unsigned *SubRegs = TRI->getSubRegisters(Reg); + unsigned SubReg = *SubRegs; ++SubRegs) { + if (LocalDefSet.insert(SubReg)) + LocalDefs.push_back(SubReg); + } + } + } + + FirstMI->setIsInsideBundle(); + Defs.clear(); + } while (FirstMI++ != LastMI); + + SmallSet<unsigned, 8> Added; + for (unsigned i = 0, e = LocalDefs.size(); i != e; ++i) { + unsigned Reg = LocalDefs[i]; + if (Added.insert(Reg)) { + // If it's not live beyond end of the bundle, mark it dead. + bool isDead = DeadDefSet.count(Reg) || KilledDefSet.count(Reg); + MIB.addReg(Reg, getDefRegState(true) | getDeadRegState(isDead) | + getImplRegState(true)); + } + } + + for (unsigned i = 0, e = ExternUses.size(); i != e; ++i) { + unsigned Reg = ExternUses[i]; + bool isKill = KilledUseSet.count(Reg); + bool isUndef = UndefUseSet.count(Reg); + MIB.addReg(Reg, getKillRegState(isKill) | getUndefRegState(isUndef) | + getImplRegState(true)); + } +} diff --git a/lib/CodeGen/MachineLICM.cpp b/lib/CodeGen/MachineLICM.cpp index e5e8c51..764429d 100644 --- a/lib/CodeGen/MachineLICM.cpp +++ b/lib/CodeGen/MachineLICM.cpp @@ -765,7 +765,7 @@ void MachineLICM::UpdateRegPressure(const MachineInstr *MI) { /// isLoadFromGOTOrConstantPool - Return true if this machine instruction /// loads from global offset table or constant pool. static bool isLoadFromGOTOrConstantPool(MachineInstr &MI) { - assert (MI.getDesc().mayLoad() && "Expected MI that loads!"); + assert (MI.mayLoad() && "Expected MI that loads!"); for (MachineInstr::mmo_iterator I = MI.memoperands_begin(), E = MI.memoperands_end(); I != E; ++I) { if (const Value *V = (*I)->getValue()) { @@ -792,7 +792,7 @@ bool MachineLICM::IsLICMCandidate(MachineInstr &I) { // from constant memory are not safe to speculate all the time, for example // indexed load from a jump table. // Stores and side effects are already checked by isSafeToMove. - if (I.getDesc().mayLoad() && !isLoadFromGOTOrConstantPool(I) && + if (I.mayLoad() && !isLoadFromGOTOrConstantPool(I) && !IsGuaranteedToExecute(I.getParent())) return false; @@ -921,7 +921,7 @@ bool MachineLICM::HasHighOperandLatency(MachineInstr &MI, /// IsCheapInstruction - Return true if the instruction is marked "cheap" or /// the operand latency between its def and a use is one or less. bool MachineLICM::IsCheapInstruction(MachineInstr &MI) const { - if (MI.getDesc().isAsCheapAsAMove() || MI.isCopyLike()) + if (MI.isAsCheapAsAMove() || MI.isCopyLike()) return true; if (!InstrItins || InstrItins->isEmpty()) return false; @@ -1105,7 +1105,7 @@ bool MachineLICM::IsProfitableToHoist(MachineInstr &MI) { MachineInstr *MachineLICM::ExtractHoistableLoad(MachineInstr *MI) { // Don't unfold simple loads. - if (MI->getDesc().canFoldAsLoad()) + if (MI->canFoldAsLoad()) return 0; // If not, we may be able to unfold a load and hoist that. @@ -1141,8 +1141,9 @@ MachineInstr *MachineLICM::ExtractHoistableLoad(MachineInstr *MI) { assert(NewMIs.size() == 2 && "Unfolded a load into multiple instructions!"); MachineBasicBlock *MBB = MI->getParent(); - MBB->insert(MI, NewMIs[0]); - MBB->insert(MI, NewMIs[1]); + MachineBasicBlock::iterator Pos = MI; + MBB->insert(Pos, NewMIs[0]); + MBB->insert(Pos, NewMIs[1]); // If unfolding produced a load that wasn't loop-invariant or profitable to // hoist, discard the new instructions and bail. if (!IsLoopInvariantInst(*NewMIs[0]) || !IsProfitableToHoist(*NewMIs[0])) { diff --git a/lib/CodeGen/MachineSSAUpdater.cpp b/lib/CodeGen/MachineSSAUpdater.cpp index 84d6df2..8cb6112 100644 --- a/lib/CodeGen/MachineSSAUpdater.cpp +++ b/lib/CodeGen/MachineSSAUpdater.cpp @@ -81,7 +81,7 @@ unsigned LookForIdenticalPHI(MachineBasicBlock *BB, if (BB->empty()) return 0; - MachineBasicBlock::iterator I = BB->front(); + MachineBasicBlock::iterator I = BB->begin(); if (!I->isPHI()) return 0; @@ -182,7 +182,7 @@ unsigned MachineSSAUpdater::GetValueInMiddleOfBlock(MachineBasicBlock *BB) { return DupPHI; // Otherwise, we do need a PHI: insert one now. - MachineBasicBlock::iterator Loc = BB->empty() ? BB->end() : BB->front(); + MachineBasicBlock::iterator Loc = BB->empty() ? BB->end() : BB->begin(); MachineInstr *InsertedPHI = InsertNewDef(TargetOpcode::PHI, BB, Loc, VRC, MRI, TII); @@ -311,7 +311,7 @@ public: /// Add it into the specified block and return the register. static unsigned CreateEmptyPHI(MachineBasicBlock *BB, unsigned NumPreds, MachineSSAUpdater *Updater) { - MachineBasicBlock::iterator Loc = BB->empty() ? BB->end() : BB->front(); + MachineBasicBlock::iterator Loc = BB->empty() ? BB->end() : BB->begin(); MachineInstr *PHI = InsertNewDef(TargetOpcode::PHI, BB, Loc, Updater->VRC, Updater->MRI, Updater->TII); diff --git a/lib/CodeGen/MachineSink.cpp b/lib/CodeGen/MachineSink.cpp index 29cfb49..e47360d 100644 --- a/lib/CodeGen/MachineSink.cpp +++ b/lib/CodeGen/MachineSink.cpp @@ -90,6 +90,12 @@ namespace { bool AllUsesDominatedByBlock(unsigned Reg, MachineBasicBlock *MBB, MachineBasicBlock *DefMBB, bool &BreakPHIEdge, bool &LocalUse) const; + MachineBasicBlock *FindSuccToSinkTo(MachineInstr *MI, MachineBasicBlock *MBB, + bool &BreakPHIEdge); + bool isProfitableToSinkTo(unsigned Reg, MachineInstr *MI, + MachineBasicBlock *MBB, + MachineBasicBlock *SuccToSinkTo); + bool PerformTrivialForwardCoalescing(MachineInstr *MI, MachineBasicBlock *MBB); }; @@ -147,14 +153,10 @@ MachineSinking::AllUsesDominatedByBlock(unsigned Reg, assert(TargetRegisterInfo::isVirtualRegister(Reg) && "Only makes sense for vregs"); + // Ignore debug uses because debug info doesn't affect the code. if (MRI->use_nodbg_empty(Reg)) return true; - // Ignoring debug uses is necessary so debug info doesn't affect the code. - // This may leave a referencing dbg_value in the original block, before - // the definition of the vreg. Dwarf generator handles this although the - // user might not get the right info at runtime. - // BreakPHIEdge is true if all the uses are in the successor MBB being sunken // into and they are all PHI nodes. In this case, machine-sink must break // the critical edge first. e.g. @@ -291,7 +293,7 @@ bool MachineSinking::isWorthBreakingCriticalEdge(MachineInstr *MI, if (!CEBCandidates.insert(std::make_pair(From, To))) return true; - if (!MI->isCopy() && !MI->getDesc().isAsCheapAsAMove()) + if (!MI->isCopy() && !MI->isAsCheapAsAMove()) return true; // MI is cheap, we probably don't want to break the critical edge for it. @@ -401,35 +403,76 @@ static void collectDebugValues(MachineInstr *MI, } } -/// SinkInstruction - Determine whether it is safe to sink the specified machine -/// instruction out of its current block into a successor. -bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) { - // Don't sink insert_subreg, subreg_to_reg, reg_sequence. These are meant to - // be close to the source to make it easier to coalesce. - if (AvoidsSinking(MI, MRI)) +/// isPostDominatedBy - Return true if A is post dominated by B. +static bool isPostDominatedBy(MachineBasicBlock *A, MachineBasicBlock *B) { + + // FIXME - Use real post dominator. + if (A->succ_size() != 2) + return false; + MachineBasicBlock::succ_iterator I = A->succ_begin(); + if (B == *I) + ++I; + MachineBasicBlock *OtherSuccBlock = *I; + if (OtherSuccBlock->succ_size() != 1 || + *(OtherSuccBlock->succ_begin()) != B) return false; - // Check if it's safe to move the instruction. - if (!MI->isSafeToMove(TII, AA, SawStore)) + return true; +} + +/// isProfitableToSinkTo - Return true if it is profitable to sink MI. +bool MachineSinking::isProfitableToSinkTo(unsigned Reg, MachineInstr *MI, + MachineBasicBlock *MBB, + MachineBasicBlock *SuccToSinkTo) { + assert (MI && "Invalid MachineInstr!"); + assert (SuccToSinkTo && "Invalid SinkTo Candidate BB"); + + if (MBB == SuccToSinkTo) return false; - // FIXME: This should include support for sinking instructions within the - // block they are currently in to shorten the live ranges. We often get - // instructions sunk into the top of a large block, but it would be better to - // also sink them down before their first use in the block. This xform has to - // be careful not to *increase* register pressure though, e.g. sinking - // "x = y + z" down if it kills y and z would increase the live ranges of y - // and z and only shrink the live range of x. + // It is profitable if SuccToSinkTo does not post dominate current block. + if (!isPostDominatedBy(MBB, SuccToSinkTo)) + return true; + + // Check if only use in post dominated block is PHI instruction. + bool NonPHIUse = false; + for (MachineRegisterInfo::use_nodbg_iterator + I = MRI->use_nodbg_begin(Reg), E = MRI->use_nodbg_end(); + I != E; ++I) { + MachineInstr *UseInst = &*I; + MachineBasicBlock *UseBlock = UseInst->getParent(); + if (UseBlock == SuccToSinkTo && !UseInst->isPHI()) + NonPHIUse = true; + } + if (!NonPHIUse) + return true; + + // If SuccToSinkTo post dominates then also it may be profitable if MI + // can further profitably sinked into another block in next round. + bool BreakPHIEdge = false; + // FIXME - If finding successor is compile time expensive then catch results. + if (MachineBasicBlock *MBB2 = FindSuccToSinkTo(MI, SuccToSinkTo, BreakPHIEdge)) + return isProfitableToSinkTo(Reg, MI, SuccToSinkTo, MBB2); + + // If SuccToSinkTo is final destination and it is a post dominator of current + // block then it is not profitable to sink MI into SuccToSinkTo block. + return false; +} + +/// FindSuccToSinkTo - Find a successor to sink this instruction to. +MachineBasicBlock *MachineSinking::FindSuccToSinkTo(MachineInstr *MI, + MachineBasicBlock *MBB, + bool &BreakPHIEdge) { + + assert (MI && "Invalid MachineInstr!"); + assert (MBB && "Invalid MachineBasicBlock!"); // Loop over all the operands of the specified instruction. If there is // anything we can't handle, bail out. - MachineBasicBlock *ParentBlock = MI->getParent(); // SuccToSinkTo - This is the successor to sink this instruction to, once we // decide. MachineBasicBlock *SuccToSinkTo = 0; - - bool BreakPHIEdge = false; for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { const MachineOperand &MO = MI->getOperand(i); if (!MO.isReg()) continue; // Ignore non-register operands. @@ -443,23 +486,23 @@ bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) { // and we can freely move its uses. Alternatively, if it's allocatable, // it could get allocated to something with a def during allocation. if (!MRI->def_empty(Reg)) - return false; + return NULL; if (AllocatableSet.test(Reg)) - return false; + return NULL; // Check for a def among the register's aliases too. for (const unsigned *Alias = TRI->getAliasSet(Reg); *Alias; ++Alias) { unsigned AliasReg = *Alias; if (!MRI->def_empty(AliasReg)) - return false; + return NULL; if (AllocatableSet.test(AliasReg)) - return false; + return NULL; } } else if (!MO.isDead()) { // A def that isn't dead. We can't move it. - return false; + return NULL; } } else { // Virtual register uses are always safe to sink. @@ -467,7 +510,7 @@ bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) { // If it's not safe to move defs of the register class, then abort. if (!TII->isSafeToMoveRegClassDefs(MRI->getRegClass(Reg))) - return false; + return NULL; // FIXME: This picks a successor to sink into based on having one // successor that dominates all the uses. However, there are cases where @@ -488,48 +531,79 @@ bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) { // If a previous operand picked a block to sink to, then this operand // must be sinkable to the same block. bool LocalUse = false; - if (!AllUsesDominatedByBlock(Reg, SuccToSinkTo, ParentBlock, + if (!AllUsesDominatedByBlock(Reg, SuccToSinkTo, MBB, BreakPHIEdge, LocalUse)) - return false; + return NULL; continue; } // Otherwise, we should look at all the successors and decide which one // we should sink to. - for (MachineBasicBlock::succ_iterator SI = ParentBlock->succ_begin(), - E = ParentBlock->succ_end(); SI != E; ++SI) { + for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(), + E = MBB->succ_end(); SI != E; ++SI) { + MachineBasicBlock *SuccBlock = *SI; bool LocalUse = false; - if (AllUsesDominatedByBlock(Reg, *SI, ParentBlock, + if (AllUsesDominatedByBlock(Reg, SuccBlock, MBB, BreakPHIEdge, LocalUse)) { - SuccToSinkTo = *SI; + SuccToSinkTo = SuccBlock; break; } if (LocalUse) // Def is used locally, it's never safe to move this def. - return false; + return NULL; } // If we couldn't find a block to sink to, ignore this instruction. if (SuccToSinkTo == 0) - return false; + return NULL; + else if (!isProfitableToSinkTo(Reg, MI, MBB, SuccToSinkTo)) + return NULL; } } - // If there are no outputs, it must have side-effects. - if (SuccToSinkTo == 0) - return false; + // It is not possible to sink an instruction into its own block. This can + // happen with loops. + if (MBB == SuccToSinkTo) + return NULL; // It's not safe to sink instructions to EH landing pad. Control flow into // landing pad is implicitly defined. - if (SuccToSinkTo->isLandingPad()) + if (SuccToSinkTo && SuccToSinkTo->isLandingPad()) + return NULL; + + return SuccToSinkTo; +} + +/// SinkInstruction - Determine whether it is safe to sink the specified machine +/// instruction out of its current block into a successor. +bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) { + // Don't sink insert_subreg, subreg_to_reg, reg_sequence. These are meant to + // be close to the source to make it easier to coalesce. + if (AvoidsSinking(MI, MRI)) return false; - // It is not possible to sink an instruction into its own block. This can - // happen with loops. - if (MI->getParent() == SuccToSinkTo) + // Check if it's safe to move the instruction. + if (!MI->isSafeToMove(TII, AA, SawStore)) return false; + // FIXME: This should include support for sinking instructions within the + // block they are currently in to shorten the live ranges. We often get + // instructions sunk into the top of a large block, but it would be better to + // also sink them down before their first use in the block. This xform has to + // be careful not to *increase* register pressure though, e.g. sinking + // "x = y + z" down if it kills y and z would increase the live ranges of y + // and z and only shrink the live range of x. + + bool BreakPHIEdge = false; + MachineBasicBlock *ParentBlock = MI->getParent(); + MachineBasicBlock *SuccToSinkTo = FindSuccToSinkTo(MI, ParentBlock, BreakPHIEdge); + + // If there are no outputs, it must have side-effects. + if (SuccToSinkTo == 0) + return false; + + // If the instruction to move defines a dead physical register which is live // when leaving the basic block, don't move it because it could turn into a // "zombie" define of that preg. E.g., EFLAGS. (<rdar://problem/8030636>) diff --git a/lib/CodeGen/MachineVerifier.cpp b/lib/CodeGen/MachineVerifier.cpp index f231e3c..0a2c2f8 100644 --- a/lib/CodeGen/MachineVerifier.cpp +++ b/lib/CodeGen/MachineVerifier.cpp @@ -279,13 +279,17 @@ bool MachineVerifier::runOnMachineFunction(MachineFunction &MF) { for (MachineFunction::const_iterator MFI = MF.begin(), MFE = MF.end(); MFI!=MFE; ++MFI) { visitMachineBasicBlockBefore(MFI); - for (MachineBasicBlock::const_iterator MBBI = MFI->begin(), - MBBE = MFI->end(); MBBI != MBBE; ++MBBI) { + for (MachineBasicBlock::const_instr_iterator MBBI = MFI->instr_begin(), + MBBE = MFI->instr_end(); MBBI != MBBE; ++MBBI) { if (MBBI->getParent() != MFI) { report("Bad instruction parent pointer", MFI); *OS << "Instruction: " << *MBBI; continue; } + // Skip BUNDLE instruction for now. FIXME: We should add code to verify + // the BUNDLE's specifically. + if (MBBI->isBundle()) + continue; visitMachineInstrBefore(MBBI); for (unsigned I = 0, E = MBBI->getNumOperands(); I != E; ++I) visitMachineOperand(&MBBI->getOperand(I), I); @@ -435,7 +439,7 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) { report("MBB exits via unconditional fall-through but its successor " "differs from its CFG successor!", MBB); } - if (!MBB->empty() && MBB->back().getDesc().isBarrier() && + if (!MBB->empty() && MBB->back().isBarrier() && !TII->isPredicated(&MBB->back())) { report("MBB exits via unconditional fall-through but ends with a " "barrier instruction!", MBB); @@ -456,10 +460,10 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) { if (MBB->empty()) { report("MBB exits via unconditional branch but doesn't contain " "any instructions!", MBB); - } else if (!MBB->back().getDesc().isBarrier()) { + } else if (!MBB->back().isBarrier()) { report("MBB exits via unconditional branch but doesn't end with a " "barrier instruction!", MBB); - } else if (!MBB->back().getDesc().isTerminator()) { + } else if (!MBB->back().isTerminator()) { report("MBB exits via unconditional branch but the branch isn't a " "terminator instruction!", MBB); } @@ -479,10 +483,10 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) { if (MBB->empty()) { report("MBB exits via conditional branch/fall-through but doesn't " "contain any instructions!", MBB); - } else if (MBB->back().getDesc().isBarrier()) { + } else if (MBB->back().isBarrier()) { report("MBB exits via conditional branch/fall-through but ends with a " "barrier instruction!", MBB); - } else if (!MBB->back().getDesc().isTerminator()) { + } else if (!MBB->back().isTerminator()) { report("MBB exits via conditional branch/fall-through but the branch " "isn't a terminator instruction!", MBB); } @@ -499,10 +503,10 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) { if (MBB->empty()) { report("MBB exits via conditional branch/branch but doesn't " "contain any instructions!", MBB); - } else if (!MBB->back().getDesc().isBarrier()) { + } else if (!MBB->back().isBarrier()) { report("MBB exits via conditional branch/branch but doesn't end with a " "barrier instruction!", MBB); - } else if (!MBB->back().getDesc().isTerminator()) { + } else if (!MBB->back().isTerminator()) { report("MBB exits via conditional branch/branch but the branch " "isn't a terminator instruction!", MBB); } @@ -555,9 +559,9 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) { // Check the MachineMemOperands for basic consistency. for (MachineInstr::mmo_iterator I = MI->memoperands_begin(), E = MI->memoperands_end(); I != E; ++I) { - if ((*I)->isLoad() && !MCID.mayLoad()) + if ((*I)->isLoad() && !MI->mayLoad()) report("Missing mayLoad flag", MI); - if ((*I)->isStore() && !MCID.mayStore()) + if ((*I)->isStore() && !MI->mayStore()) report("Missing mayStore flag", MI); } @@ -575,7 +579,7 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) { } // Ensure non-terminators don't follow terminators. - if (MCID.isTerminator()) { + if (MI->isTerminator()) { if (!FirstTerminator) FirstTerminator = MI; } else if (FirstTerminator) { @@ -606,7 +610,7 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { // Don't check if it's the last operand in a variadic instruction. See, // e.g., LDM_RET in the arm back end. if (MO->isReg() && - !(MCID.isVariadic() && MONum == MCID.getNumOperands()-1)) { + !(MI->isVariadic() && MONum == MCID.getNumOperands()-1)) { if (MO->isDef() && !MCOI.isOptionalDef()) report("Explicit operand marked as def", MO, MONum); if (MO->isImplicit()) @@ -614,7 +618,7 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { } } else { // ARM adds %reg0 operands to indicate predicates. We'll allow that. - if (MO->isReg() && !MO->isImplicit() && !MCID.isVariadic() && MO->getReg()) + if (MO->isReg() && !MO->isImplicit() && !MI->isVariadic() && MO->getReg()) report("Extra explicit operand on non-variadic instruction", MO, MONum); } @@ -800,11 +804,11 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { LiveInts && !LiveInts->isNotInMIMap(MI)) { LiveInterval &LI = LiveStks->getInterval(MO->getIndex()); SlotIndex Idx = LiveInts->getInstructionIndex(MI); - if (MCID.mayLoad() && !LI.liveAt(Idx.getRegSlot(true))) { + if (MI->mayLoad() && !LI.liveAt(Idx.getRegSlot(true))) { report("Instruction loads from dead spill slot", MO, MONum); *OS << "Live stack: " << LI << '\n'; } - if (MCID.mayStore() && !LI.liveAt(Idx.getRegSlot())) { + if (MI->mayStore() && !LI.liveAt(Idx.getRegSlot())) { report("Instruction stores to dead spill slot", MO, MONum); *OS << "Live stack: " << LI << '\n'; } diff --git a/lib/CodeGen/PHIElimination.cpp b/lib/CodeGen/PHIElimination.cpp index 6994aa5..0e52496 100644 --- a/lib/CodeGen/PHIElimination.cpp +++ b/lib/CodeGen/PHIElimination.cpp @@ -410,7 +410,7 @@ bool PHIElimination::SplitPHIEdges(MachineFunction &MF, return false; // Quick exit for basic blocks without PHIs. bool Changed = false; - for (MachineBasicBlock::const_iterator BBI = MBB.begin(), BBE = MBB.end(); + for (MachineBasicBlock::iterator BBI = MBB.begin(), BBE = MBB.end(); BBI != BBE && BBI->isPHI(); ++BBI) { for (unsigned i = 1, e = BBI->getNumOperands(); i != e; i += 2) { unsigned Reg = BBI->getOperand(i).getReg(); diff --git a/lib/CodeGen/PeepholeOptimizer.cpp b/lib/CodeGen/PeepholeOptimizer.cpp index bbc7ce2..2a5652a 100644 --- a/lib/CodeGen/PeepholeOptimizer.cpp +++ b/lib/CodeGen/PeepholeOptimizer.cpp @@ -292,7 +292,7 @@ bool PeepholeOptimizer::OptimizeBitcastInstr(MachineInstr *MI, assert(Def && Src && "Malformed bitcast instruction!"); MachineInstr *DefMI = MRI->getVRegDef(Src); - if (!DefMI || !DefMI->getDesc().isBitcast()) + if (!DefMI || !DefMI->isBitcast()) return false; unsigned SrcSrc = 0; @@ -353,7 +353,7 @@ bool PeepholeOptimizer::isMoveImmediate(MachineInstr *MI, SmallSet<unsigned, 4> &ImmDefRegs, DenseMap<unsigned, MachineInstr*> &ImmDefMIs) { const MCInstrDesc &MCID = MI->getDesc(); - if (!MCID.isMoveImmediate()) + if (!MI->isMoveImmediate()) return false; if (MCID.getNumDefs() != 1) return false; @@ -428,9 +428,7 @@ bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) { continue; } - const MCInstrDesc &MCID = MI->getDesc(); - - if (MCID.isBitcast()) { + if (MI->isBitcast()) { if (OptimizeBitcastInstr(MI, MBB)) { // MI is deleted. LocalMIs.erase(MI); @@ -438,7 +436,7 @@ bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) { MII = First ? I->begin() : llvm::next(PMII); continue; } - } else if (MCID.isCompare()) { + } else if (MI->isCompare()) { if (OptimizeCmpInstr(MI, MBB)) { // MI is deleted. LocalMIs.erase(MI); diff --git a/lib/CodeGen/PostRASchedulerList.cpp b/lib/CodeGen/PostRASchedulerList.cpp index 7205ed6..fa832c8 100644 --- a/lib/CodeGen/PostRASchedulerList.cpp +++ b/lib/CodeGen/PostRASchedulerList.cpp @@ -212,7 +212,8 @@ bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) { RegClassInfo.runOnMachineFunction(Fn); // Check for explicit enable/disable of post-ra scheduling. - TargetSubtargetInfo::AntiDepBreakMode AntiDepMode = TargetSubtargetInfo::ANTIDEP_NONE; + TargetSubtargetInfo::AntiDepBreakMode AntiDepMode = + TargetSubtargetInfo::ANTIDEP_NONE; SmallVector<TargetRegisterClass*, 4> CriticalPathRCs; if (EnablePostRAScheduler.getPosition() > 0) { if (!EnablePostRAScheduler) @@ -271,6 +272,8 @@ bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) { } I = MI; --Count; + if (MI->isBundle()) + Count -= MI->getBundleSize(); } assert(Count == 0 && "Instruction count mismatch!"); assert((MBB->begin() == Current || CurrentCount != 0) && @@ -364,7 +367,7 @@ void SchedulePostRATDList::StartBlockForKills(MachineBasicBlock *BB) { KillIndices[i] = ~0u; // Determine the live-out physregs for this block. - if (!BB->empty() && BB->back().getDesc().isReturn()) { + if (!BB->empty() && BB->back().isReturn()) { // In a return block, examine the function live-out regs. for (MachineRegisterInfo::liveout_iterator I = MRI.liveout_begin(), E = MRI.liveout_end(); I != E; ++I) { diff --git a/lib/CodeGen/PrologEpilogInserter.cpp b/lib/CodeGen/PrologEpilogInserter.cpp index 32c9325..b4fd1cb 100644 --- a/lib/CodeGen/PrologEpilogInserter.cpp +++ b/lib/CodeGen/PrologEpilogInserter.cpp @@ -332,7 +332,7 @@ void PEI::insertCSRSpillsAndRestores(MachineFunction &Fn) { // Skip over all terminator instructions, which are part of the return // sequence. MachineBasicBlock::iterator I2 = I; - while (I2 != MBB->begin() && (--I2)->getDesc().isTerminator()) + while (I2 != MBB->begin() && (--I2)->isTerminator()) I = I2; bool AtStart = I == MBB->begin(); @@ -426,11 +426,11 @@ void PEI::insertCSRSpillsAndRestores(MachineFunction &Fn) { // Skip over all terminator instructions, which are part of the // return sequence. - if (! I->getDesc().isTerminator()) { + if (! I->isTerminator()) { ++I; } else { MachineBasicBlock::iterator I2 = I; - while (I2 != MBB->begin() && (--I2)->getDesc().isTerminator()) + while (I2 != MBB->begin() && (--I2)->isTerminator()) I = I2; } } @@ -698,7 +698,7 @@ void PEI::insertPrologEpilogCode(MachineFunction &Fn) { // Add epilogue to restore the callee-save registers in each exiting block for (MachineFunction::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) { // If last instruction is a return instruction, add an epilogue - if (!I->empty() && I->back().getDesc().isReturn()) + if (!I->empty() && I->back().isReturn()) TFI.emitEpilogue(Fn, *I); } @@ -706,7 +706,7 @@ void PEI::insertPrologEpilogCode(MachineFunction &Fn) { // we've been asked for it. This, when linked with a runtime with support // for segmented stacks (libgcc is one), will result in allocating stack // space in small chunks instead of one large contiguous block. - if (EnableSegmentedStacks) + if (Fn.getTarget().Options.EnableSegmentedStacks) TFI.adjustForSegmentedStacks(Fn); } diff --git a/lib/CodeGen/RegAllocFast.cpp b/lib/CodeGen/RegAllocFast.cpp index 4664a3c..c2656c5 100644 --- a/lib/CodeGen/RegAllocFast.cpp +++ b/lib/CodeGen/RegAllocFast.cpp @@ -748,8 +748,8 @@ void RAFast::AllocateBasicBlock() { // and return are tail calls; do not do this for them. The tail callee need // not take the same registers as input that it produces as output, and there // are dependencies for its input registers elsewhere. - if (!MBB->empty() && MBB->back().getDesc().isReturn() && - !MBB->back().getDesc().isCall()) { + if (!MBB->empty() && MBB->back().isReturn() && + !MBB->back().isCall()) { MachineInstr *Ret = &MBB->back(); for (MachineRegisterInfo::liveout_iterator @@ -968,7 +968,7 @@ void RAFast::AllocateBasicBlock() { } unsigned DefOpEnd = MI->getNumOperands(); - if (MCID.isCall()) { + if (MI->isCall()) { // Spill all virtregs before a call. This serves two purposes: 1. If an // exception is thrown, the landing pad is going to expect to find // registers in their spill slots, and 2. we don't have to wade through diff --git a/lib/CodeGen/RegAllocPBQP.cpp b/lib/CodeGen/RegAllocPBQP.cpp index 845ee12..a053ccc 100644 --- a/lib/CodeGen/RegAllocPBQP.cpp +++ b/lib/CodeGen/RegAllocPBQP.cpp @@ -34,13 +34,14 @@ #include "LiveRangeEdit.h" #include "RenderMachineFunction.h" #include "Spiller.h" -#include "Splitter.h" #include "VirtRegMap.h" #include "RegisterCoalescer.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/CalcSpillWeights.h" #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/LiveStackAnalysis.h" #include "llvm/CodeGen/RegAllocPBQP.h" +#include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" @@ -68,11 +69,6 @@ pbqpCoalescing("pbqp-coalescing", cl::desc("Attempt coalescing during PBQP register allocation."), cl::init(false), cl::Hidden); -static cl::opt<bool> -pbqpPreSplitting("pbqp-pre-splitting", - cl::desc("Pre-split before PBQP register allocation."), - cl::init(false), cl::Hidden); - namespace { /// @@ -93,7 +89,6 @@ public: initializeCalculateSpillWeightsPass(*PassRegistry::getPassRegistry()); initializeLiveStacksPass(*PassRegistry::getPassRegistry()); initializeMachineLoopInfoPass(*PassRegistry::getPassRegistry()); - initializeLoopSplitterPass(*PassRegistry::getPassRegistry()); initializeVirtRegMapPass(*PassRegistry::getPassRegistry()); initializeRenderMachineFunctionPass(*PassRegistry::getPassRegistry()); } @@ -444,6 +439,9 @@ void PBQPBuilderWithCoalescing::addVirtRegCoalesce( void RegAllocPBQP::getAnalysisUsage(AnalysisUsage &au) const { + au.setPreservesCFG(); + au.addRequired<AliasAnalysis>(); + au.addPreserved<AliasAnalysis>(); au.addRequired<SlotIndexes>(); au.addPreserved<SlotIndexes>(); au.addRequired<LiveIntervals>(); @@ -454,10 +452,10 @@ void RegAllocPBQP::getAnalysisUsage(AnalysisUsage &au) const { au.addRequired<CalculateSpillWeights>(); au.addRequired<LiveStacks>(); au.addPreserved<LiveStacks>(); + au.addRequired<MachineDominatorTree>(); + au.addPreserved<MachineDominatorTree>(); au.addRequired<MachineLoopInfo>(); au.addPreserved<MachineLoopInfo>(); - if (pbqpPreSplitting) - au.addRequired<LoopSplitter>(); au.addRequired<VirtRegMap>(); au.addRequired<RenderMachineFunction>(); MachineFunctionPass::getAnalysisUsage(au); diff --git a/lib/CodeGen/RegisterCoalescer.cpp b/lib/CodeGen/RegisterCoalescer.cpp index 22d6a3b..cd181cd 100644 --- a/lib/CodeGen/RegisterCoalescer.cpp +++ b/lib/CodeGen/RegisterCoalescer.cpp @@ -651,8 +651,7 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP, MachineInstr *DefMI = LIS->getInstructionFromIndex(AValNo->def); if (!DefMI) return false; - const MCInstrDesc &MCID = DefMI->getDesc(); - if (!MCID.isCommutable()) + if (!DefMI->isCommutable()) return false; // If DefMI is a two-address instruction then commuting it will change the // destination register. @@ -718,7 +717,8 @@ bool RegisterCoalescer::RemoveCopyByCommutingDef(const CoalescerPair &CP, return false; if (NewMI != DefMI) { LIS->ReplaceMachineInstrInMaps(DefMI, NewMI); - MBB->insert(DefMI, NewMI); + MachineBasicBlock::iterator Pos = DefMI; + MBB->insert(Pos, NewMI); MBB->erase(DefMI); } unsigned OpIdx = NewMI->findRegisterUseOperandIdx(IntA.reg, false); @@ -809,14 +809,14 @@ bool RegisterCoalescer::ReMaterializeTrivialDef(LiveInterval &SrcInt, if (!DefMI) return false; assert(DefMI && "Defining instruction disappeared"); - const MCInstrDesc &MCID = DefMI->getDesc(); - if (!MCID.isAsCheapAsAMove()) + if (!DefMI->isAsCheapAsAMove()) return false; if (!TII->isTriviallyReMaterializable(DefMI, AA)) return false; bool SawStore = false; if (!DefMI->isSafeToMove(TII, AA, SawStore)) return false; + const MCInstrDesc &MCID = DefMI->getDesc(); if (MCID.getNumDefs() != 1) return false; if (!DefMI->isImplicitDef()) { diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp index 34b8ab0..4418f40 100644 --- a/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -157,7 +157,7 @@ void ScheduleDAGInstrs::AddSchedBarrierDeps() { MachineInstr *ExitMI = InsertPos != BB->end() ? &*InsertPos : 0; ExitSU.setInstr(ExitMI); bool AllDepKnown = ExitMI && - (ExitMI->getDesc().isCall() || ExitMI->getDesc().isBarrier()); + (ExitMI->isCall() || ExitMI->isBarrier()); if (ExitMI && AllDepKnown) { // If it's a call or a barrier, add dependencies on the defs and uses of // instruction. @@ -238,13 +238,12 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) { continue; } - const MCInstrDesc &MCID = MI->getDesc(); - assert(!MCID.isTerminator() && !MI->isLabel() && + assert(!MI->isTerminator() && !MI->isLabel() && "Cannot schedule terminators or labels!"); // Create the SUnit for this MI. SUnit *SU = NewSUnit(MI); - SU->isCall = MCID.isCall(); - SU->isCommutable = MCID.isCommutable(); + SU->isCall = MI->isCall(); + SU->isCommutable = MI->isCommutable(); // Assign the Latency field of SU using target-provided information. if (UnitLatencies) @@ -278,8 +277,15 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) { continue; if (DefSU != SU && (Kind != SDep::Output || !MO.isDead() || - !DefSU->getInstr()->registerDefIsDead(Reg))) - DefSU->addPred(SDep(SU, Kind, AOLatency, /*Reg=*/Reg)); + !DefSU->getInstr()->registerDefIsDead(Reg))) { + if (Kind == SDep::Anti) + DefSU->addPred(SDep(SU, Kind, 0, /*Reg=*/Reg)); + else { + unsigned AOLat = TII->getOutputLatency(InstrItins, MI, j, + DefSU->getInstr()); + DefSU->addPred(SDep(SU, Kind, AOLat, /*Reg=*/Reg)); + } + } } for (const unsigned *Alias = TRI->getAliasSet(Reg); *Alias; ++Alias) { std::vector<SUnit *> &MemDefList = Defs[*Alias]; @@ -315,7 +321,7 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) { int RegUseIndex = UseMI->findRegisterUseOperandIdx(Reg); assert(RegUseIndex >= 0 && "UseMI doesn's use register!"); if (RegUseIndex >= 0 && - (UseMCID.mayLoad() || UseMCID.mayStore()) && + (UseMI->mayLoad() || UseMI->mayStore()) && (unsigned)RegUseIndex < UseMCID.getNumOperands() && UseMCID.OpInfo[RegUseIndex].isLookupPtrRegClass()) LDataLatency += SpecialAddressLatency; @@ -419,9 +425,9 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) { // produce more precise dependence information. #define STORE_LOAD_LATENCY 1 unsigned TrueMemOrderLatency = 0; - if (MCID.isCall() || MI->hasUnmodeledSideEffects() || + if (MI->isCall() || MI->hasUnmodeledSideEffects() || (MI->hasVolatileMemoryRef() && - (!MCID.mayLoad() || !MI->isInvariantLoad(AA)))) { + (!MI->mayLoad() || !MI->isInvariantLoad(AA)))) { // Be conservative with these and add dependencies on all memory // references, even those that are known to not alias. for (std::map<const Value *, SUnit *>::iterator I = @@ -460,7 +466,7 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) { PendingLoads.clear(); AliasMemDefs.clear(); AliasMemUses.clear(); - } else if (MCID.mayStore()) { + } else if (MI->mayStore()) { bool MayAlias = true; TrueMemOrderLatency = STORE_LOAD_LATENCY; if (const Value *V = getUnderlyingObjectForInstr(MI, MFI, MayAlias)) { @@ -516,7 +522,7 @@ void ScheduleDAGInstrs::BuildSchedGraph(AliasAnalysis *AA) { /*Reg=*/0, /*isNormalMemory=*/false, /*isMustAlias=*/false, /*isArtificial=*/true)); - } else if (MCID.mayLoad()) { + } else if (MI->mayLoad()) { bool MayAlias = true; TrueMemOrderLatency = 0; if (MI->isInvariantLoad(AA)) { @@ -576,7 +582,7 @@ void ScheduleDAGInstrs::ComputeLatency(SUnit *SU) { // Simplistic target-independent heuristic: assume that loads take // extra time. - if (SU->getInstr()->getDesc().mayLoad()) + if (SU->getInstr()->mayLoad()) SU->Latency += 2; } else { SU->Latency = TII->getInstrLatency(InstrItins, SU->getInstr()); @@ -658,39 +664,33 @@ std::string ScheduleDAGInstrs::getGraphNodeLabel(const SUnit *SU) const { // EmitSchedule - Emit the machine code in scheduled order. MachineBasicBlock *ScheduleDAGInstrs::EmitSchedule() { - // For MachineInstr-based scheduling, we're rescheduling the instructions in - // the block, so start by removing them from the block. - while (Begin != InsertPos) { - MachineBasicBlock::iterator I = Begin; - ++Begin; - BB->remove(I); - } + Begin = InsertPos; // If first instruction was a DBG_VALUE then put it back. if (FirstDbgValue) - BB->insert(InsertPos, FirstDbgValue); + BB->splice(InsertPos, BB, FirstDbgValue); // Then re-insert them according to the given schedule. for (unsigned i = 0, e = Sequence.size(); i != e; i++) { if (SUnit *SU = Sequence[i]) - BB->insert(InsertPos, SU->getInstr()); + BB->splice(InsertPos, BB, SU->getInstr()); else // Null SUnit* is a noop. EmitNoop(); - } - // Update the Begin iterator, as the first instruction in the block - // may have been scheduled later. - if (!Sequence.empty()) - Begin = Sequence[0]->getInstr(); + // Update the Begin iterator, as the first instruction in the block + // may have been scheduled later. + if (i == 0) + Begin = prior(InsertPos); + } // Reinsert any remaining debug_values. for (std::vector<std::pair<MachineInstr *, MachineInstr *> >::iterator DI = DbgValues.end(), DE = DbgValues.begin(); DI != DE; --DI) { std::pair<MachineInstr *, MachineInstr *> P = *prior(DI); MachineInstr *DbgValue = P.first; - MachineInstr *OrigPrivMI = P.second; - BB->insertAfter(OrigPrivMI, DbgValue); + MachineBasicBlock::iterator OrigPrivMI = P.second; + BB->splice(++OrigPrivMI, BB, DbgValue); } DbgValues.clear(); FirstDbgValue = NULL; diff --git a/lib/CodeGen/SelectionDAG/CMakeLists.txt b/lib/CodeGen/SelectionDAG/CMakeLists.txt index ff4184f..6023326 100644 --- a/lib/CodeGen/SelectionDAG/CMakeLists.txt +++ b/lib/CodeGen/SelectionDAG/CMakeLists.txt @@ -20,13 +20,3 @@ add_llvm_library(LLVMSelectionDAG TargetLowering.cpp TargetSelectionDAGInfo.cpp ) - -add_llvm_library_dependencies(LLVMSelectionDAG - LLVMAnalysis - LLVMCodeGen - LLVMCore - LLVMMC - LLVMSupport - LLVMTarget - LLVMTransformUtils - ) diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index d8208a4..80cf0a8 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -180,7 +180,9 @@ namespace { SDValue visitSRA(SDNode *N); SDValue visitSRL(SDNode *N); SDValue visitCTLZ(SDNode *N); + SDValue visitCTLZ_ZERO_UNDEF(SDNode *N); SDValue visitCTTZ(SDNode *N); + SDValue visitCTTZ_ZERO_UNDEF(SDNode *N); SDValue visitCTPOP(SDNode *N); SDValue visitSELECT(SDNode *N); SDValue visitSELECT_CC(SDNode *N); @@ -361,6 +363,7 @@ CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) { /// specified expression for the same cost as the expression itself, or 2 if we /// can compute the negated form more cheaply than the expression itself. static char isNegatibleForFree(SDValue Op, bool LegalOperations, + const TargetOptions *Options, unsigned Depth = 0) { // No compile time optimizations on this type. if (Op.getValueType() == MVT::ppcf128) @@ -383,34 +386,39 @@ static char isNegatibleForFree(SDValue Op, bool LegalOperations, return LegalOperations ? 0 : 1; case ISD::FADD: // FIXME: determine better conditions for this xform. - if (!UnsafeFPMath) return 0; + if (!Options->UnsafeFPMath) return 0; // fold (fsub (fadd A, B)) -> (fsub (fneg A), B) - if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, Depth+1)) + if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, Options, + Depth + 1)) return V; // fold (fneg (fadd A, B)) -> (fsub (fneg B), A) - return isNegatibleForFree(Op.getOperand(1), LegalOperations, Depth+1); + return isNegatibleForFree(Op.getOperand(1), LegalOperations, Options, + Depth + 1); case ISD::FSUB: // We can't turn -(A-B) into B-A when we honor signed zeros. - if (!UnsafeFPMath) return 0; + if (!Options->UnsafeFPMath) return 0; // fold (fneg (fsub A, B)) -> (fsub B, A) return 1; case ISD::FMUL: case ISD::FDIV: - if (HonorSignDependentRoundingFPMath()) return 0; + if (Options->HonorSignDependentRoundingFPMath()) return 0; // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y)) - if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, Depth+1)) + if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, Options, + Depth + 1)) return V; - return isNegatibleForFree(Op.getOperand(1), LegalOperations, Depth+1); + return isNegatibleForFree(Op.getOperand(1), LegalOperations, Options, + Depth + 1); case ISD::FP_EXTEND: case ISD::FP_ROUND: case ISD::FSIN: - return isNegatibleForFree(Op.getOperand(0), LegalOperations, Depth+1); + return isNegatibleForFree(Op.getOperand(0), LegalOperations, Options, + Depth + 1); } } @@ -434,10 +442,11 @@ static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG, } case ISD::FADD: // FIXME: determine better conditions for this xform. - assert(UnsafeFPMath); + assert(DAG.getTarget().Options.UnsafeFPMath); // fold (fneg (fadd A, B)) -> (fsub (fneg A), B) - if (isNegatibleForFree(Op.getOperand(0), LegalOperations, Depth+1)) + if (isNegatibleForFree(Op.getOperand(0), LegalOperations, + &DAG.getTarget().Options, Depth+1)) return DAG.getNode(ISD::FSUB, Op.getDebugLoc(), Op.getValueType(), GetNegatedExpression(Op.getOperand(0), DAG, LegalOperations, Depth+1), @@ -449,7 +458,7 @@ static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG, Op.getOperand(0)); case ISD::FSUB: // We can't turn -(A-B) into B-A when we honor signed zeros. - assert(UnsafeFPMath); + assert(DAG.getTarget().Options.UnsafeFPMath); // fold (fneg (fsub 0, B)) -> B if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0))) @@ -462,10 +471,11 @@ static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG, case ISD::FMUL: case ISD::FDIV: - assert(!HonorSignDependentRoundingFPMath()); + assert(!DAG.getTarget().Options.HonorSignDependentRoundingFPMath()); // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) - if (isNegatibleForFree(Op.getOperand(0), LegalOperations, Depth+1)) + if (isNegatibleForFree(Op.getOperand(0), LegalOperations, + &DAG.getTarget().Options, Depth+1)) return DAG.getNode(Op.getOpcode(), Op.getDebugLoc(), Op.getValueType(), GetNegatedExpression(Op.getOperand(0), DAG, LegalOperations, Depth+1), @@ -1070,7 +1080,9 @@ SDValue DAGCombiner::visit(SDNode *N) { case ISD::SRA: return visitSRA(N); case ISD::SRL: return visitSRL(N); case ISD::CTLZ: return visitCTLZ(N); + case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N); case ISD::CTTZ: return visitCTTZ(N); + case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N); case ISD::CTPOP: return visitCTPOP(N); case ISD::SELECT: return visitSELECT(N); case ISD::SELECT_CC: return visitSELECT_CC(N); @@ -1769,7 +1781,7 @@ SDValue DAGCombiner::visitSDIV(SDNode *N) { N0, N1); } // fold (sdiv X, pow2) -> simple ops after legalize - if (N1C && !N1C->isNullValue() && !TLI.isIntDivCheap() && + if (N1C && !N1C->isNullValue() && (N1C->getAPIntValue().isPowerOf2() || (-N1C->getAPIntValue()).isPowerOf2())) { // If dividing by powers of two is cheap, then don't perform the following @@ -3709,6 +3721,16 @@ SDValue DAGCombiner::visitCTLZ(SDNode *N) { return SDValue(); } +SDValue DAGCombiner::visitCTLZ_ZERO_UNDEF(SDNode *N) { + SDValue N0 = N->getOperand(0); + EVT VT = N->getValueType(0); + + // fold (ctlz_zero_undef c1) -> c2 + if (isa<ConstantSDNode>(N0)) + return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, N->getDebugLoc(), VT, N0); + return SDValue(); +} + SDValue DAGCombiner::visitCTTZ(SDNode *N) { SDValue N0 = N->getOperand(0); EVT VT = N->getValueType(0); @@ -3719,6 +3741,16 @@ SDValue DAGCombiner::visitCTTZ(SDNode *N) { return SDValue(); } +SDValue DAGCombiner::visitCTTZ_ZERO_UNDEF(SDNode *N) { + SDValue N0 = N->getOperand(0); + EVT VT = N->getValueType(0); + + // fold (cttz_zero_undef c1) -> c2 + if (isa<ConstantSDNode>(N0)) + return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, N->getDebugLoc(), VT, N0); + return SDValue(); +} + SDValue DAGCombiner::visitCTPOP(SDNode *N) { SDValue N0 = N->getOperand(0); EVT VT = N->getValueType(0); @@ -5254,20 +5286,22 @@ SDValue DAGCombiner::visitFADD(SDNode *N) { if (N0CFP && !N1CFP) return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N1, N0); // fold (fadd A, 0) -> A - if (UnsafeFPMath && N1CFP && N1CFP->getValueAPF().isZero()) + if (DAG.getTarget().Options.UnsafeFPMath && N1CFP && + N1CFP->getValueAPF().isZero()) return N0; // fold (fadd A, (fneg B)) -> (fsub A, B) - if (isNegatibleForFree(N1, LegalOperations) == 2) + if (isNegatibleForFree(N1, LegalOperations, &DAG.getTarget().Options) == 2) return DAG.getNode(ISD::FSUB, N->getDebugLoc(), VT, N0, GetNegatedExpression(N1, DAG, LegalOperations)); // fold (fadd (fneg A), B) -> (fsub B, A) - if (isNegatibleForFree(N0, LegalOperations) == 2) + if (isNegatibleForFree(N0, LegalOperations, &DAG.getTarget().Options) == 2) return DAG.getNode(ISD::FSUB, N->getDebugLoc(), VT, N1, GetNegatedExpression(N0, DAG, LegalOperations)); // If allowed, fold (fadd (fadd x, c1), c2) -> (fadd x, (fadd c1, c2)) - if (UnsafeFPMath && N1CFP && N0.getOpcode() == ISD::FADD && - N0.getNode()->hasOneUse() && isa<ConstantFPSDNode>(N0.getOperand(1))) + if (DAG.getTarget().Options.UnsafeFPMath && N1CFP && + N0.getOpcode() == ISD::FADD && N0.getNode()->hasOneUse() && + isa<ConstantFPSDNode>(N0.getOperand(1))) return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0.getOperand(0), DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0.getOperand(1), N1)); @@ -5292,17 +5326,19 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) { if (N0CFP && N1CFP && VT != MVT::ppcf128) return DAG.getNode(ISD::FSUB, N->getDebugLoc(), VT, N0, N1); // fold (fsub A, 0) -> A - if (UnsafeFPMath && N1CFP && N1CFP->getValueAPF().isZero()) + if (DAG.getTarget().Options.UnsafeFPMath && + N1CFP && N1CFP->getValueAPF().isZero()) return N0; // fold (fsub 0, B) -> -B - if (UnsafeFPMath && N0CFP && N0CFP->getValueAPF().isZero()) { - if (isNegatibleForFree(N1, LegalOperations)) + if (DAG.getTarget().Options.UnsafeFPMath && + N0CFP && N0CFP->getValueAPF().isZero()) { + if (isNegatibleForFree(N1, LegalOperations, &DAG.getTarget().Options)) return GetNegatedExpression(N1, DAG, LegalOperations); if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) return DAG.getNode(ISD::FNEG, N->getDebugLoc(), VT, N1); } // fold (fsub A, (fneg B)) -> (fadd A, B) - if (isNegatibleForFree(N1, LegalOperations)) + if (isNegatibleForFree(N1, LegalOperations, &DAG.getTarget().Options)) return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0, GetNegatedExpression(N1, DAG, LegalOperations)); @@ -5329,10 +5365,12 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) { if (N0CFP && !N1CFP) return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, N1, N0); // fold (fmul A, 0) -> 0 - if (UnsafeFPMath && N1CFP && N1CFP->getValueAPF().isZero()) + if (DAG.getTarget().Options.UnsafeFPMath && + N1CFP && N1CFP->getValueAPF().isZero()) return N1; // fold (fmul A, 0) -> 0, vector edition. - if (UnsafeFPMath && ISD::isBuildVectorAllZeros(N1.getNode())) + if (DAG.getTarget().Options.UnsafeFPMath && + ISD::isBuildVectorAllZeros(N1.getNode())) return N1; // fold (fmul X, 2.0) -> (fadd X, X) if (N1CFP && N1CFP->isExactlyValue(+2.0)) @@ -5343,8 +5381,10 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) { return DAG.getNode(ISD::FNEG, N->getDebugLoc(), VT, N0); // fold (fmul (fneg X), (fneg Y)) -> (fmul X, Y) - if (char LHSNeg = isNegatibleForFree(N0, LegalOperations)) { - if (char RHSNeg = isNegatibleForFree(N1, LegalOperations)) { + if (char LHSNeg = isNegatibleForFree(N0, LegalOperations, + &DAG.getTarget().Options)) { + if (char RHSNeg = isNegatibleForFree(N1, LegalOperations, + &DAG.getTarget().Options)) { // Both can be negated for free, check to see if at least one is cheaper // negated. if (LHSNeg == 2 || RHSNeg == 2) @@ -5355,7 +5395,8 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) { } // If allowed, fold (fmul (fmul x, c1), c2) -> (fmul x, (fmul c1, c2)) - if (UnsafeFPMath && N1CFP && N0.getOpcode() == ISD::FMUL && + if (DAG.getTarget().Options.UnsafeFPMath && + N1CFP && N0.getOpcode() == ISD::FMUL && N0.getNode()->hasOneUse() && isa<ConstantFPSDNode>(N0.getOperand(1))) return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, N0.getOperand(0), DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, @@ -5383,8 +5424,10 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) { // (fdiv (fneg X), (fneg Y)) -> (fdiv X, Y) - if (char LHSNeg = isNegatibleForFree(N0, LegalOperations)) { - if (char RHSNeg = isNegatibleForFree(N1, LegalOperations)) { + if (char LHSNeg = isNegatibleForFree(N0, LegalOperations, + &DAG.getTarget().Options)) { + if (char RHSNeg = isNegatibleForFree(N1, LegalOperations, + &DAG.getTarget().Options)) { // Both can be negated for free, check to see if at least one is cheaper // negated. if (LHSNeg == 2 || RHSNeg == 2) @@ -5637,7 +5680,7 @@ SDValue DAGCombiner::visitFNEG(SDNode *N) { SDValue N0 = N->getOperand(0); EVT VT = N->getValueType(0); - if (isNegatibleForFree(N0, LegalOperations)) + if (isNegatibleForFree(N0, LegalOperations, &DAG.getTarget().Options)) return GetNegatedExpression(N0, DAG, LegalOperations); // Transform fneg(bitconvert(x)) -> bitconvert(x^sign) to avoid loading @@ -7162,19 +7205,23 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) { if (NVT != SmallVT || NVT.getSizeInBits()*2 != BigVT.getSizeInBits()) return SDValue(); - // Combine: - // (extract_subvec (insert_subvec V1, V2, InsIdx), ExtIdx) - // Into: - // indicies are equal => V1 - // otherwise => (extract_subvec V1, ExtIdx) - // - SDValue InsIdx = N->getOperand(1); - SDValue ExtIdx = V->getOperand(2); - - if (InsIdx == ExtIdx) - return V->getOperand(1); - return DAG.getNode(ISD::EXTRACT_SUBVECTOR, N->getDebugLoc(), NVT, - V->getOperand(0), N->getOperand(1)); + // Only handle cases where both indexes are constants with the same type. + ConstantSDNode *InsIdx = dyn_cast<ConstantSDNode>(N->getOperand(1)); + ConstantSDNode *ExtIdx = dyn_cast<ConstantSDNode>(V->getOperand(2)); + + if (InsIdx && ExtIdx && + InsIdx->getValueType(0).getSizeInBits() <= 64 && + ExtIdx->getValueType(0).getSizeInBits() <= 64) { + // Combine: + // (extract_subvec (insert_subvec V1, V2, InsIdx), ExtIdx) + // Into: + // indices are equal => V1 + // otherwise => (extract_subvec V1, ExtIdx) + if (InsIdx->getZExtValue() == ExtIdx->getZExtValue()) + return V->getOperand(1); + return DAG.getNode(ISD::EXTRACT_SUBVECTOR, N->getDebugLoc(), NVT, + V->getOperand(0), N->getOperand(1)); + } } return SDValue(); diff --git a/lib/CodeGen/SelectionDAG/FastISel.cpp b/lib/CodeGen/SelectionDAG/FastISel.cpp index cff37c2..b4946ec 100644 --- a/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -62,8 +62,11 @@ #include "llvm/ADT/Statistic.h" using namespace llvm; -STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by target-independent selector"); -STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by target-specific selector"); +STATISTIC(NumFastIselSuccessIndependent, "Number of insts selected by " + "target-independent selector"); +STATISTIC(NumFastIselSuccessTarget, "Number of insts selected by " + "target-specific selector"); +STATISTIC(NumFastIselDead, "Number of dead insts removed on failure"); /// startNewBlock - Set the current block to which generated machine /// instructions will be appended, and clear the local CSE map. @@ -307,6 +310,18 @@ void FastISel::recomputeInsertPt() { ++FuncInfo.InsertPt; } +void FastISel::removeDeadCode(MachineBasicBlock::iterator I, + MachineBasicBlock::iterator E) { + assert (I && E && std::distance(I, E) > 0 && "Invalid iterator!"); + while (I != E) { + MachineInstr *Dead = &*I; + ++I; + Dead->eraseFromParent(); + ++NumFastIselDead; + } + recomputeInsertPt(); +} + FastISel::SavePoint FastISel::enterLocalValueArea() { MachineBasicBlock::iterator OldInsertPt = FuncInfo.InsertPt; DebugLoc OldDL = DL; @@ -792,19 +807,33 @@ FastISel::SelectInstruction(const Instruction *I) { DL = I->getDebugLoc(); + MachineBasicBlock::iterator SavedInsertPt = FuncInfo.InsertPt; + // First, try doing target-independent selection. if (SelectOperator(I, I->getOpcode())) { ++NumFastIselSuccessIndependent; DL = DebugLoc(); return true; } + // Remove dead code. However, ignore call instructions since we've flushed + // the local value map and recomputed the insert point. + if (!isa<CallInst>(I)) { + recomputeInsertPt(); + if (SavedInsertPt != FuncInfo.InsertPt) + removeDeadCode(FuncInfo.InsertPt, SavedInsertPt); + } // Next, try calling the target to attempt to handle the instruction. + SavedInsertPt = FuncInfo.InsertPt; if (TargetSelectInstruction(I)) { ++NumFastIselSuccessTarget; DL = DebugLoc(); return true; } + // Check for dead code and remove as necessary. + recomputeInsertPt(); + if (SavedInsertPt != FuncInfo.InsertPt) + removeDeadCode(FuncInfo.InsertPt, SavedInsertPt); DL = DebugLoc(); return false; diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index 2ff66f8..cb6fd53 100644 --- a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -294,7 +294,7 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op, const TargetRegisterClass *DstRC = 0; if (IIOpNum < II->getNumOperands()) DstRC = TII->getRegClass(*II, IIOpNum, TRI); - assert((DstRC || (MCID.isVariadic() && IIOpNum >= MCID.getNumOperands())) && + assert((DstRC || (MI->isVariadic() && IIOpNum >= MCID.getNumOperands())) && "Don't have operand info for this instruction!"); if (DstRC && !MRI->constrainRegClass(VReg, DstRC, MinRCSize)) { unsigned NewVReg = MRI->createVirtualRegister(DstRC); diff --git a/lib/CodeGen/SelectionDAG/LLVMBuild.txt b/lib/CodeGen/SelectionDAG/LLVMBuild.txt index 10a849f..81d2e00 100644 --- a/lib/CodeGen/SelectionDAG/LLVMBuild.txt +++ b/lib/CodeGen/SelectionDAG/LLVMBuild.txt @@ -20,4 +20,3 @@ type = Library name = SelectionDAG parent = CodeGen required_libraries = Analysis CodeGen Core MC Support Target TransformUtils - diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 156cc70..75f5761 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -785,7 +785,6 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { case ISD::FRAME_TO_ARGS_OFFSET: case ISD::EH_SJLJ_SETJMP: case ISD::EH_SJLJ_LONGJMP: - case ISD::EH_SJLJ_DISPATCHSETUP: // These operations lie about being legal: when they claim to be legal, // they should actually be expanded. Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); @@ -2383,6 +2382,9 @@ SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op, return Op; } + case ISD::CTLZ_ZERO_UNDEF: + // This trivially expands to CTLZ. + return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op); case ISD::CTLZ: { // for now, we do this: // x = x | (x >> 1); @@ -2404,6 +2406,9 @@ SDValue SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDValue Op, Op = DAG.getNOT(dl, Op, VT); return DAG.getNode(ISD::CTPOP, dl, VT, Op); } + case ISD::CTTZ_ZERO_UNDEF: + // This trivially expands to CTTZ. + return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op); case ISD::CTTZ: { // for now, we use: { return popcount(~x & (x - 1)); } // unless the target has ctlz but not ctpop, in which case we use: @@ -2518,7 +2523,9 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) { switch (Node->getOpcode()) { case ISD::CTPOP: case ISD::CTLZ: + case ISD::CTLZ_ZERO_UNDEF: case ISD::CTTZ: + case ISD::CTTZ_ZERO_UNDEF: Tmp1 = ExpandBitCount(Node->getOpcode(), Node->getOperand(0), dl); Results.push_back(Tmp1); break; @@ -2538,7 +2545,6 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) { case ISD::PREFETCH: case ISD::VAEND: case ISD::EH_SJLJ_LONGJMP: - case ISD::EH_SJLJ_DISPATCHSETUP: // If the target didn't expand these, there's nothing to do, so just // preserve the chain and be done. Results.push_back(Node->getOperand(0)); @@ -3421,20 +3427,24 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) { SDValue Tmp1, Tmp2, Tmp3; switch (Node->getOpcode()) { case ISD::CTTZ: + case ISD::CTTZ_ZERO_UNDEF: case ISD::CTLZ: + case ISD::CTLZ_ZERO_UNDEF: case ISD::CTPOP: // Zero extend the argument. Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); - // Perform the larger operation. + // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is + // already the correct result. Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); if (Node->getOpcode() == ISD::CTTZ) { - //if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT) + // FIXME: This should set a bit in the zero extended value instead. Tmp2 = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Tmp1, DAG.getConstant(NVT.getSizeInBits(), NVT), ISD::SETEQ); Tmp1 = DAG.getNode(ISD::SELECT, dl, NVT, Tmp2, DAG.getConstant(OVT.getSizeInBits(), NVT), Tmp1); - } else if (Node->getOpcode() == ISD::CTLZ) { + } else if (Node->getOpcode() == ISD::CTLZ || + Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, DAG.getConstant(NVT.getSizeInBits() - diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index fd24238..1c02c4f 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -56,8 +56,10 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { case ISD::Constant: Res = PromoteIntRes_Constant(N); break; case ISD::CONVERT_RNDSAT: Res = PromoteIntRes_CONVERT_RNDSAT(N); break; + case ISD::CTLZ_ZERO_UNDEF: case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break; case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break; + case ISD::CTTZ_ZERO_UNDEF: case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break; case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; @@ -216,7 +218,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { case TargetLowering::TypeLegal: break; case TargetLowering::TypePromoteInteger: - if (NOutVT.bitsEq(NInVT)) + if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector()) // The input promotes to the same size. Convert the promoted value. return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp)); break; @@ -311,7 +313,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) { DebugLoc dl = N->getDebugLoc(); EVT OVT = N->getValueType(0); EVT NVT = Op.getValueType(); - Op = DAG.getNode(ISD::CTLZ, dl, NVT, Op); + Op = DAG.getNode(N->getOpcode(), dl, NVT, Op); // Subtract off the extra leading bits in the bigger type. return DAG.getNode(ISD::SUB, dl, NVT, Op, DAG.getConstant(NVT.getSizeInBits() - @@ -329,13 +331,15 @@ SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) { EVT OVT = N->getValueType(0); EVT NVT = Op.getValueType(); DebugLoc dl = N->getDebugLoc(); - // The count is the same in the promoted type except if the original - // value was zero. This can be handled by setting the bit just off - // the top of the original type. - APInt TopBit(NVT.getSizeInBits(), 0); - TopBit.setBit(OVT.getSizeInBits()); - Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, NVT)); - return DAG.getNode(ISD::CTTZ, dl, NVT, Op); + if (N->getOpcode() == ISD::CTTZ) { + // The count is the same in the promoted type except if the original + // value was zero. This can be handled by setting the bit just off + // the top of the original type. + APInt TopBit(NVT.getSizeInBits(), 0); + TopBit.setBit(OVT.getSizeInBits()); + Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, NVT)); + } + return DAG.getNode(N->getOpcode(), dl, NVT, Op); } SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) { @@ -1097,8 +1101,10 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break; case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; + case ISD::CTLZ_ZERO_UNDEF: case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; + case ISD::CTTZ_ZERO_UNDEF: case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break; case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; @@ -1701,8 +1707,8 @@ void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, SDValue HiNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Hi, DAG.getConstant(0, NVT), ISD::SETNE); - SDValue LoLZ = DAG.getNode(ISD::CTLZ, dl, NVT, Lo); - SDValue HiLZ = DAG.getNode(ISD::CTLZ, dl, NVT, Hi); + SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo); + SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi); Lo = DAG.getNode(ISD::SELECT, dl, NVT, HiNotZero, HiLZ, DAG.getNode(ISD::ADD, dl, NVT, LoLZ, @@ -1731,8 +1737,8 @@ void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N, SDValue LoNotZero = DAG.getSetCC(dl, TLI.getSetCCResultType(NVT), Lo, DAG.getConstant(0, NVT), ISD::SETNE); - SDValue LoLZ = DAG.getNode(ISD::CTTZ, dl, NVT, Lo); - SDValue HiLZ = DAG.getNode(ISD::CTTZ, dl, NVT, Hi); + SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo); + SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi); Lo = DAG.getNode(ISD::SELECT, dl, NVT, LoNotZero, LoLZ, DAG.getNode(ISD::ADD, dl, NVT, HiLZ, diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index 4e02b90..4696c0d 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -185,8 +185,10 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { case ISD::SRL: case ISD::ROTL: case ISD::ROTR: - case ISD::CTTZ: case ISD::CTLZ: + case ISD::CTTZ: + case ISD::CTLZ_ZERO_UNDEF: + case ISD::CTTZ_ZERO_UNDEF: case ISD::CTPOP: case ISD::SELECT: case ISD::VSELECT: diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index ad83565..7ca0d1e 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -441,8 +441,10 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { case ISD::ANY_EXTEND: case ISD::CONVERT_RNDSAT: case ISD::CTLZ: - case ISD::CTPOP: case ISD::CTTZ: + case ISD::CTLZ_ZERO_UNDEF: + case ISD::CTTZ_ZERO_UNDEF: + case ISD::CTPOP: case ISD::FABS: case ISD::FCEIL: case ISD::FCOS: diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp index cd0da37..80162d7 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp @@ -102,17 +102,6 @@ static cl::opt<unsigned> AvgIPC( "sched-avg-ipc", cl::Hidden, cl::init(1), cl::desc("Average inst/cycle whan no target itinerary exists.")); -#ifndef NDEBUG -namespace { - // For sched=list-ilp, Count the number of times each factor comes into play. - enum { FactPressureDiff, FactRegUses, FactStall, FactHeight, FactDepth, - FactStatic, FactOther, NumFactors }; -} -static const char *FactorName[NumFactors] = -{"PressureDiff", "RegUses", "Stall", "Height", "Depth","Static", "Other"}; -static int FactorCount[NumFactors]; -#endif //!NDEBUG - namespace { //===----------------------------------------------------------------------===// /// ScheduleDAGRRList - The actual register reduction list scheduler @@ -157,6 +146,10 @@ private: /// and similar queries. ScheduleDAGTopologicalSort Topo; + // Hack to keep track of the inverse of FindCallSeqStart without more crazy + // DAG crawling. + DenseMap<SUnit*, SUnit*> CallSeqEndForStart; + public: ScheduleDAGRRList(MachineFunction &mf, bool needlatency, SchedulingPriorityQueue *availqueue, @@ -308,11 +301,6 @@ void ScheduleDAGRRList::Schedule() { DEBUG(dbgs() << "********** List Scheduling BB#" << BB->getNumber() << " '" << BB->getName() << "' **********\n"); -#ifndef NDEBUG - for (int i = 0; i < NumFactors; ++i) { - FactorCount[i] = 0; - } -#endif //!NDEBUG CurCycle = 0; IssueCount = 0; @@ -322,6 +310,7 @@ void ScheduleDAGRRList::Schedule() { // to track the virtual resource of a calling sequence. LiveRegDefs.resize(TRI->getNumRegs() + 1, NULL); LiveRegGens.resize(TRI->getNumRegs() + 1, NULL); + CallSeqEndForStart.clear(); // Build the scheduling graph. BuildSchedGraph(NULL); @@ -337,11 +326,6 @@ void ScheduleDAGRRList::Schedule() { // Execute the actual scheduling loop. ListScheduleBottomUp(); -#ifndef NDEBUG - for (int i = 0; i < NumFactors; ++i) { - DEBUG(dbgs() << FactorName[i] << "\t" << FactorCount[i] << "\n"); - } -#endif // !NDEBUG AvailableQueue->releaseState(); } @@ -545,6 +529,8 @@ void ScheduleDAGRRList::ReleasePredecessors(SUnit *SU) { SDNode *N = FindCallSeqStart(Node, NestLevel, MaxNest, TII); SUnit *Def = &SUnits[N->getNodeId()]; + CallSeqEndForStart[Def] = SU; + ++NumLiveRegs; LiveRegDefs[CallResource] = Def; LiveRegGens[CallResource] = SU; @@ -811,7 +797,7 @@ void ScheduleDAGRRList::UnscheduleNodeBottomUp(SUnit *SU) { SUNode->getMachineOpcode() == (unsigned)TII->getCallFrameSetupOpcode()) { ++NumLiveRegs; LiveRegDefs[CallResource] = SU; - LiveRegGens[CallResource] = NULL; + LiveRegGens[CallResource] = CallSeqEndForStart[SU]; } } @@ -832,12 +818,11 @@ void ScheduleDAGRRList::UnscheduleNodeBottomUp(SUnit *SU) { for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end(); I != E; ++I) { if (I->isAssignedRegDep()) { + if (!LiveRegDefs[I->getReg()]) + ++NumLiveRegs; // This becomes the nearest def. Note that an earlier def may still be // pending if this is a two-address node. LiveRegDefs[I->getReg()] = SU; - if (!LiveRegDefs[I->getReg()]) { - ++NumLiveRegs; - } if (LiveRegGens[I->getReg()] == NULL || I->getSUnit()->getHeight() < LiveRegGens[I->getReg()]->getHeight()) LiveRegGens[I->getReg()] = I->getSUnit(); @@ -2296,28 +2281,20 @@ static int BUCompareLatency(SUnit *left, SUnit *right, bool checkPref, // If scheduling either one of the node will cause a pipeline stall, sort // them according to their height. if (LStall) { - if (!RStall) { - DEBUG(++FactorCount[FactStall]); + if (!RStall) return 1; - } - if (LHeight != RHeight) { - DEBUG(++FactorCount[FactStall]); + if (LHeight != RHeight) return LHeight > RHeight ? 1 : -1; - } - } else if (RStall) { - DEBUG(++FactorCount[FactStall]); + } else if (RStall) return -1; - } // If either node is scheduling for latency, sort them by height/depth // and latency. if (!checkPref || (left->SchedulingPref == Sched::ILP || right->SchedulingPref == Sched::ILP)) { if (DisableSchedCycles) { - if (LHeight != RHeight) { - DEBUG(++FactorCount[FactHeight]); + if (LHeight != RHeight) return LHeight > RHeight ? 1 : -1; - } } else { // If neither instruction stalls (!LStall && !RStall) then @@ -2326,17 +2303,14 @@ static int BUCompareLatency(SUnit *left, SUnit *right, bool checkPref, int LDepth = left->getDepth() - LPenalty; int RDepth = right->getDepth() - RPenalty; if (LDepth != RDepth) { - DEBUG(++FactorCount[FactDepth]); DEBUG(dbgs() << " Comparing latency of SU (" << left->NodeNum << ") depth " << LDepth << " vs SU (" << right->NodeNum << ") depth " << RDepth << "\n"); return LDepth < RDepth ? 1 : -1; } } - if (left->Latency != right->Latency) { - DEBUG(++FactorCount[FactOther]); + if (left->Latency != right->Latency) return left->Latency > right->Latency ? 1 : -1; - } } return 0; } @@ -2350,7 +2324,6 @@ static bool BURRSort(SUnit *left, SUnit *right, RegReductionPQBase *SPQ) { bool LHasPhysReg = left->hasPhysRegDefs; bool RHasPhysReg = right->hasPhysRegDefs; if (LHasPhysReg != RHasPhysReg) { - DEBUG(++FactorCount[FactRegUses]); #ifndef NDEBUG const char *PhysRegMsg[] = {" has no physreg", " defines a physreg"}; #endif @@ -2376,10 +2349,8 @@ static bool BURRSort(SUnit *left, SUnit *right, RegReductionPQBase *SPQ) { LPriority = (LPriority > LNumVals) ? (LPriority - LNumVals) : 0; } - if (LPriority != RPriority) { - DEBUG(++FactorCount[FactStatic]); + if (LPriority != RPriority) return LPriority > RPriority; - } // One or both of the nodes are calls and their sethi-ullman numbers are the // same, then keep source order. @@ -2412,18 +2383,14 @@ static bool BURRSort(SUnit *left, SUnit *right, RegReductionPQBase *SPQ) { // This creates more short live intervals. unsigned LDist = closestSucc(left); unsigned RDist = closestSucc(right); - if (LDist != RDist) { - DEBUG(++FactorCount[FactOther]); + if (LDist != RDist) return LDist < RDist; - } // How many registers becomes live when the node is scheduled. unsigned LScratch = calcMaxScratches(left); unsigned RScratch = calcMaxScratches(right); - if (LScratch != RScratch) { - DEBUG(++FactorCount[FactOther]); + if (LScratch != RScratch) return LScratch > RScratch; - } // Comparing latency against a call makes little sense unless the node // is register pressure-neutral. @@ -2438,20 +2405,15 @@ static bool BURRSort(SUnit *left, SUnit *right, RegReductionPQBase *SPQ) { return result > 0; } else { - if (left->getHeight() != right->getHeight()) { - DEBUG(++FactorCount[FactHeight]); + if (left->getHeight() != right->getHeight()) return left->getHeight() > right->getHeight(); - } - if (left->getDepth() != right->getDepth()) { - DEBUG(++FactorCount[FactDepth]); + if (left->getDepth() != right->getDepth()) return left->getDepth() < right->getDepth(); - } } assert(left->NodeQueueId && right->NodeQueueId && "NodeQueueId cannot be zero"); - DEBUG(++FactorCount[FactOther]); return (left->NodeQueueId > right->NodeQueueId); } @@ -2511,13 +2473,11 @@ bool hybrid_ls_rr_sort::operator()(SUnit *left, SUnit *right) const { // Avoid causing spills. If register pressure is high, schedule for // register pressure reduction. if (LHigh && !RHigh) { - DEBUG(++FactorCount[FactPressureDiff]); DEBUG(dbgs() << " pressure SU(" << left->NodeNum << ") > SU(" << right->NodeNum << ")\n"); return true; } else if (!LHigh && RHigh) { - DEBUG(++FactorCount[FactPressureDiff]); DEBUG(dbgs() << " pressure SU(" << right->NodeNum << ") > SU(" << left->NodeNum << ")\n"); return false; @@ -2581,7 +2541,6 @@ bool ilp_ls_rr_sort::operator()(SUnit *left, SUnit *right) const { RPDiff = SPQ->RegPressureDiff(right, RLiveUses); } if (!DisableSchedRegPressure && LPDiff != RPDiff) { - DEBUG(++FactorCount[FactPressureDiff]); DEBUG(dbgs() << "RegPressureDiff SU(" << left->NodeNum << "): " << LPDiff << " != SU(" << right->NodeNum << "): " << RPDiff << "\n"); return LPDiff > RPDiff; @@ -2590,7 +2549,6 @@ bool ilp_ls_rr_sort::operator()(SUnit *left, SUnit *right) const { if (!DisableSchedRegPressure && (LPDiff > 0 || RPDiff > 0)) { bool LReduce = canEnableCoalescing(left); bool RReduce = canEnableCoalescing(right); - DEBUG(if (LReduce != RReduce) ++FactorCount[FactPressureDiff]); if (LReduce && !RReduce) return false; if (RReduce && !LReduce) return true; } @@ -2598,17 +2556,14 @@ bool ilp_ls_rr_sort::operator()(SUnit *left, SUnit *right) const { if (!DisableSchedLiveUses && (LLiveUses != RLiveUses)) { DEBUG(dbgs() << "Live uses SU(" << left->NodeNum << "): " << LLiveUses << " != SU(" << right->NodeNum << "): " << RLiveUses << "\n"); - DEBUG(++FactorCount[FactRegUses]); return LLiveUses < RLiveUses; } if (!DisableSchedStalls) { bool LStall = BUHasStall(left, left->getHeight(), SPQ); bool RStall = BUHasStall(right, right->getHeight(), SPQ); - if (LStall != RStall) { - DEBUG(++FactorCount[FactHeight]); + if (LStall != RStall) return left->getHeight() > right->getHeight(); - } } if (!DisableSchedCriticalPath) { @@ -2617,17 +2572,14 @@ bool ilp_ls_rr_sort::operator()(SUnit *left, SUnit *right) const { DEBUG(dbgs() << "Depth of SU(" << left->NodeNum << "): " << left->getDepth() << " != SU(" << right->NodeNum << "): " << right->getDepth() << "\n"); - DEBUG(++FactorCount[FactDepth]); return left->getDepth() < right->getDepth(); } } if (!DisableSchedHeight && left->getHeight() != right->getHeight()) { int spread = (int)left->getHeight() - (int)right->getHeight(); - if (std::abs(spread) > MaxReorderWindow) { - DEBUG(++FactorCount[FactHeight]); + if (std::abs(spread) > MaxReorderWindow) return left->getHeight() > right->getHeight(); - } } return BURRSort(left, right, SPQ); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 497c286..dd626e2 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -840,9 +840,9 @@ unsigned SelectionDAG::getEVTAlignment(EVT VT) const { } // EntryNode could meaningfully have debug info if we can find it... -SelectionDAG::SelectionDAG(const TargetMachine &tm) +SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) : TM(tm), TLI(*tm.getTargetLowering()), TSI(*tm.getSelectionDAGInfo()), - EntryNode(ISD::EntryToken, DebugLoc(), getVTList(MVT::Other)), + OptLevel(OL), EntryNode(ISD::EntryToken, DebugLoc(), getVTList(MVT::Other)), Root(getEntryNode()), Ordering(0) { AllNodes.push_back(&EntryNode); Ordering = new SDNodeOrdering(); @@ -1856,7 +1856,9 @@ void SelectionDAG::ComputeMaskedBits(SDValue Op, const APInt &Mask, return; } case ISD::CTTZ: + case ISD::CTTZ_ZERO_UNDEF: case ISD::CTLZ: + case ISD::CTLZ_ZERO_UNDEF: case ISD::CTPOP: { unsigned LowBits = Log2_32(BitWidth)+1; KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits); @@ -2334,7 +2336,7 @@ bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { bool SelectionDAG::isKnownNeverNaN(SDValue Op) const { // If we're told that NaNs won't happen, assume they won't. - if (NoNaNsFPMath) + if (getTarget().Options.NoNaNsFPMath) return true; // If the value is a constant, we can obviously see if it is a NaN or not. @@ -2429,8 +2431,10 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, case ISD::CTPOP: return getConstant(Val.countPopulation(), VT); case ISD::CTLZ: + case ISD::CTLZ_ZERO_UNDEF: return getConstant(Val.countLeadingZeros(), VT); case ISD::CTTZ: + case ISD::CTTZ_ZERO_UNDEF: return getConstant(Val.countTrailingZeros(), VT); } } @@ -2607,7 +2611,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, break; case ISD::FNEG: // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 - if (UnsafeFPMath && OpOpcode == ISD::FSUB) + if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB) return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1), Operand.getNode()->getOperand(0)); if (OpOpcode == ISD::FNEG) // --X -> X @@ -2742,7 +2746,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, case ISD::FMUL: case ISD::FDIV: case ISD::FREM: - if (UnsafeFPMath) { + if (getTarget().Options.UnsafeFPMath) { if (Opcode == ISD::FADD) { // 0+x --> x if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) @@ -3065,7 +3069,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT, case ISD::FMUL: case ISD::FDIV: case ISD::FREM: - if (UnsafeFPMath) + if (getTarget().Options.UnsafeFPMath) return N2; break; case ISD::MUL: @@ -4914,6 +4918,20 @@ SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, return N; } +/// UpdadeDebugLocOnMergedSDNode - If the opt level is -O0 then it throws away +/// the line number information on the merged node since it is not possible to +/// preserve the information that operation is associated with multiple lines. +/// This will make the debugger working better at -O0, were there is a higher +/// probability having other instructions associated with that line. +/// +SDNode *SelectionDAG::UpdadeDebugLocOnMergedSDNode(SDNode *N, DebugLoc OLoc) { + DebugLoc NLoc = N->getDebugLoc(); + if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) && (OLoc != NLoc)) { + N->setDebugLoc(DebugLoc()); + } + return N; +} + /// MorphNodeTo - This *mutates* the specified node to have the specified /// return type, opcode, and operands. /// @@ -4935,7 +4953,7 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, FoldingSetNodeID ID; AddNodeIDNode(ID, Opc, VTs, Ops, NumOps); if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP)) - return ON; + return UpdadeDebugLocOnMergedSDNode(ON, N->getDebugLoc()); } if (!RemoveNodeFromCSEMaps(N)) @@ -5139,8 +5157,9 @@ SelectionDAG::getMachineNode(unsigned Opcode, DebugLoc DL, SDVTList VTs, FoldingSetNodeID ID; AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps); IP = 0; - if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) - return cast<MachineSDNode>(E); + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { + return cast<MachineSDNode>(UpdadeDebugLocOnMergedSDNode(E, DL)); + } } // Allocate a new MachineSDNode. @@ -5943,7 +5962,6 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::EH_RETURN: return "EH_RETURN"; case ISD::EH_SJLJ_SETJMP: return "EH_SJLJ_SETJMP"; case ISD::EH_SJLJ_LONGJMP: return "EH_SJLJ_LONGJMP"; - case ISD::EH_SJLJ_DISPATCHSETUP: return "EH_SJLJ_DISPATCHSETUP"; case ISD::ConstantPool: return "ConstantPool"; case ISD::ExternalSymbol: return "ExternalSymbol"; case ISD::BlockAddress: return "BlockAddress"; @@ -6112,10 +6130,12 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::TRAP: return "trap"; // Bit manipulation - case ISD::BSWAP: return "bswap"; - case ISD::CTPOP: return "ctpop"; - case ISD::CTTZ: return "cttz"; - case ISD::CTLZ: return "ctlz"; + case ISD::BSWAP: return "bswap"; + case ISD::CTPOP: return "ctpop"; + case ISD::CTTZ: return "cttz"; + case ISD::CTTZ_ZERO_UNDEF: return "cttz_zero_undef"; + case ISD::CTLZ: return "ctlz"; + case ISD::CTLZ_ZERO_UNDEF: return "ctlz_zero_undef"; // Trampolines case ISD::INIT_TRAMPOLINE: return "init_trampoline"; @@ -6146,6 +6166,11 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::SETLT: return "setlt"; case ISD::SETLE: return "setle"; case ISD::SETNE: return "setne"; + + case ISD::SETTRUE: return "settrue"; + case ISD::SETTRUE2: return "settrue2"; + case ISD::SETFALSE: return "setfalse"; + case ISD::SETFALSE2: return "setfalse2"; } } } @@ -6554,20 +6579,15 @@ unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { const GlobalValue *GV; int64_t GVOffset = 0; if (TLI.isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { - // If GV has specified alignment, then use it. Otherwise, use the preferred - // alignment. - unsigned Align = GV->getAlignment(); - if (!Align) { - if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV)) { - if (GVar->hasInitializer()) { - const TargetData *TD = TLI.getTargetData(); - Align = TD->getPreferredAlignment(GVar); - } - } - if (!Align) - Align = TLI.getTargetData()->getABITypeAlignment(GV->getType()); - } - return MinAlign(Align, GVOffset); + unsigned PtrWidth = TLI.getPointerTy().getSizeInBits(); + APInt AllOnes = APInt::getAllOnesValue(PtrWidth); + APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0); + llvm::ComputeMaskedBits(const_cast<GlobalValue*>(GV), AllOnes, + KnownZero, KnownOne, TLI.getTargetData()); + unsigned AlignBits = KnownZero.countTrailingOnes(); + unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0; + if (Align) + return MinAlign(Align, GVOffset); } // If this is a direct reference to a stack slot, use information about the diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 8d02350..68c9514 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -47,6 +47,7 @@ #include "llvm/Target/TargetFrameLowering.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetIntrinsicInfo.h" +#include "llvm/Target/TargetLibraryInfo.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Support/CommandLine.h" @@ -812,9 +813,11 @@ void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching, } } -void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa) { +void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa, + const TargetLibraryInfo *li) { AA = &aa; GFI = gfi; + LibInfo = li; TD = DAG.getTarget().getTargetData(); LPadToCallSiteMap.clear(); } @@ -1335,6 +1338,8 @@ SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond, Condition = getICmpCondCode(IC->getPredicate()); } else if (const FCmpInst *FC = dyn_cast<FCmpInst>(Cond)) { Condition = getFCmpCondCode(FC->getPredicate()); + if (TM.Options.NoNaNsFPMath) + Condition = getFCmpCodeWithoutNaN(Condition); } else { Condition = ISD::SETEQ; // silence warning. llvm_unreachable("Unknown compare instruction"); @@ -2002,7 +2007,7 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR, } static inline bool areJTsAllowed(const TargetLowering &TLI) { - return !DisableJumpTables && + return !TLI.getTargetMachine().Options.DisableJumpTables && (TLI.isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) || TLI.isOperationLegalOrCustom(ISD::BRIND, MVT::Other)); } @@ -2625,6 +2630,8 @@ void SelectionDAGBuilder::visitFCmp(const User &I) { SDValue Op1 = getValue(I.getOperand(0)); SDValue Op2 = getValue(I.getOperand(1)); ISD::CondCode Condition = getFCmpCondCode(predicate); + if (TM.Options.NoNaNsFPMath) + Condition = getFCmpCodeWithoutNaN(Condition); EVT DestVT = TLI.getValueType(I.getType()); setValue(&I, DAG.getSetCC(getCurDebugLoc(), DestVT, Op1, Op2, Condition)); } @@ -3095,7 +3102,7 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) { unsigned Amt = ElementSize.logBase2(); IdxN = DAG.getNode(ISD::SHL, getCurDebugLoc(), N.getValueType(), IdxN, - DAG.getConstant(Amt, TLI.getPointerTy())); + DAG.getConstant(Amt, IdxN.getValueType())); } else { SDValue Scale = DAG.getConstant(ElementSize, TLI.getPointerTy()); IdxN = DAG.getNode(ISD::MUL, getCurDebugLoc(), @@ -4775,11 +4782,6 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { getRoot(), getValue(I.getArgOperand(0)))); return 0; } - case Intrinsic::eh_sjlj_dispatch_setup: { - DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_DISPATCHSETUP, dl, MVT::Other, - getRoot(), getValue(I.getArgOperand(0)))); - return 0; - } case Intrinsic::x86_mmx_pslli_w: case Intrinsic::x86_mmx_pslli_d: @@ -4946,14 +4948,18 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { return 0; case Intrinsic::cttz: { SDValue Arg = getValue(I.getArgOperand(0)); + ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1)); EVT Ty = Arg.getValueType(); - setValue(&I, DAG.getNode(ISD::CTTZ, dl, Ty, Arg)); + setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF, + dl, Ty, Arg)); return 0; } case Intrinsic::ctlz: { SDValue Arg = getValue(I.getArgOperand(0)); + ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1)); EVT Ty = Arg.getValueType(); - setValue(&I, DAG.getNode(ISD::CTLZ, dl, Ty, Arg)); + setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF, + dl, Ty, Arg)); return 0; } case Intrinsic::ctpop: { @@ -5064,7 +5070,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { } case Intrinsic::trap: { - StringRef TrapFuncName = getTrapFunctionName(); + StringRef TrapFuncName = TM.Options.getTrapFunctionName(); if (TrapFuncName.empty()) { DAG.setRoot(DAG.getNode(ISD::TRAP, dl,MVT::Other, getRoot())); return 0; @@ -5226,7 +5232,7 @@ void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, // If there's a possibility that fast-isel has already selected some amount // of the current basic block, don't emit a tail call. - if (isTailCall && EnableFastISel) + if (isTailCall && TM.Options.EnableFastISel) isTailCall = false; std::pair<SDValue,SDValue> Result = @@ -5510,7 +5516,9 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { // can't be a library call. if (!F->hasLocalLinkage() && F->hasName()) { StringRef Name = F->getName(); - if (Name == "copysign" || Name == "copysignf" || Name == "copysignl") { + if ((LibInfo->has(LibFunc::copysign) && Name == "copysign") || + (LibInfo->has(LibFunc::copysignf) && Name == "copysignf") || + (LibInfo->has(LibFunc::copysignl) && Name == "copysignl")) { if (I.getNumArgOperands() == 2 && // Basic sanity checks. I.getArgOperand(0)->getType()->isFloatingPointTy() && I.getType() == I.getArgOperand(0)->getType() && @@ -5521,7 +5529,9 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { LHS.getValueType(), LHS, RHS)); return; } - } else if (Name == "fabs" || Name == "fabsf" || Name == "fabsl") { + } else if ((LibInfo->has(LibFunc::fabs) && Name == "fabs") || + (LibInfo->has(LibFunc::fabsf) && Name == "fabsf") || + (LibInfo->has(LibFunc::fabsl) && Name == "fabsl")) { if (I.getNumArgOperands() == 1 && // Basic sanity checks. I.getArgOperand(0)->getType()->isFloatingPointTy() && I.getType() == I.getArgOperand(0)->getType()) { @@ -5530,7 +5540,9 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { Tmp.getValueType(), Tmp)); return; } - } else if (Name == "sin" || Name == "sinf" || Name == "sinl") { + } else if ((LibInfo->has(LibFunc::sin) && Name == "sin") || + (LibInfo->has(LibFunc::sinf) && Name == "sinf") || + (LibInfo->has(LibFunc::sinl) && Name == "sinl")) { if (I.getNumArgOperands() == 1 && // Basic sanity checks. I.getArgOperand(0)->getType()->isFloatingPointTy() && I.getType() == I.getArgOperand(0)->getType() && @@ -5540,7 +5552,9 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { Tmp.getValueType(), Tmp)); return; } - } else if (Name == "cos" || Name == "cosf" || Name == "cosl") { + } else if ((LibInfo->has(LibFunc::cos) && Name == "cos") || + (LibInfo->has(LibFunc::cosf) && Name == "cosf") || + (LibInfo->has(LibFunc::cosl) && Name == "cosl")) { if (I.getNumArgOperands() == 1 && // Basic sanity checks. I.getArgOperand(0)->getType()->isFloatingPointTy() && I.getType() == I.getArgOperand(0)->getType() && @@ -5550,7 +5564,9 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { Tmp.getValueType(), Tmp)); return; } - } else if (Name == "sqrt" || Name == "sqrtf" || Name == "sqrtl") { + } else if ((LibInfo->has(LibFunc::sqrt) && Name == "sqrt") || + (LibInfo->has(LibFunc::sqrtf) && Name == "sqrtf") || + (LibInfo->has(LibFunc::sqrtl) && Name == "sqrtl")) { if (I.getNumArgOperands() == 1 && // Basic sanity checks. I.getArgOperand(0)->getType()->isFloatingPointTy() && I.getType() == I.getArgOperand(0)->getType() && @@ -5560,6 +5576,83 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { Tmp.getValueType(), Tmp)); return; } + } else if ((LibInfo->has(LibFunc::floor) && Name == "floor") || + (LibInfo->has(LibFunc::floorf) && Name == "floorf") || + (LibInfo->has(LibFunc::floorl) && Name == "floorl")) { + if (I.getNumArgOperands() == 1 && // Basic sanity checks. + I.getArgOperand(0)->getType()->isFloatingPointTy() && + I.getType() == I.getArgOperand(0)->getType()) { + SDValue Tmp = getValue(I.getArgOperand(0)); + setValue(&I, DAG.getNode(ISD::FFLOOR, getCurDebugLoc(), + Tmp.getValueType(), Tmp)); + return; + } + } else if ((LibInfo->has(LibFunc::nearbyint) && Name == "nearbyint") || + (LibInfo->has(LibFunc::nearbyintf) && Name == "nearbyintf") || + (LibInfo->has(LibFunc::nearbyintl) && Name == "nearbyintl")) { + if (I.getNumArgOperands() == 1 && // Basic sanity checks. + I.getArgOperand(0)->getType()->isFloatingPointTy() && + I.getType() == I.getArgOperand(0)->getType()) { + SDValue Tmp = getValue(I.getArgOperand(0)); + setValue(&I, DAG.getNode(ISD::FNEARBYINT, getCurDebugLoc(), + Tmp.getValueType(), Tmp)); + return; + } + } else if ((LibInfo->has(LibFunc::ceil) && Name == "ceil") || + (LibInfo->has(LibFunc::ceilf) && Name == "ceilf") || + (LibInfo->has(LibFunc::ceill) && Name == "ceill")) { + if (I.getNumArgOperands() == 1 && // Basic sanity checks. + I.getArgOperand(0)->getType()->isFloatingPointTy() && + I.getType() == I.getArgOperand(0)->getType()) { + SDValue Tmp = getValue(I.getArgOperand(0)); + setValue(&I, DAG.getNode(ISD::FCEIL, getCurDebugLoc(), + Tmp.getValueType(), Tmp)); + return; + } + } else if ((LibInfo->has(LibFunc::rint) && Name == "rint") || + (LibInfo->has(LibFunc::rintf) && Name == "rintf") || + (LibInfo->has(LibFunc::rintl) && Name == "rintl")) { + if (I.getNumArgOperands() == 1 && // Basic sanity checks. + I.getArgOperand(0)->getType()->isFloatingPointTy() && + I.getType() == I.getArgOperand(0)->getType()) { + SDValue Tmp = getValue(I.getArgOperand(0)); + setValue(&I, DAG.getNode(ISD::FRINT, getCurDebugLoc(), + Tmp.getValueType(), Tmp)); + return; + } + } else if ((LibInfo->has(LibFunc::trunc) && Name == "trunc") || + (LibInfo->has(LibFunc::truncf) && Name == "truncf") || + (LibInfo->has(LibFunc::truncl) && Name == "truncl")) { + if (I.getNumArgOperands() == 1 && // Basic sanity checks. + I.getArgOperand(0)->getType()->isFloatingPointTy() && + I.getType() == I.getArgOperand(0)->getType()) { + SDValue Tmp = getValue(I.getArgOperand(0)); + setValue(&I, DAG.getNode(ISD::FTRUNC, getCurDebugLoc(), + Tmp.getValueType(), Tmp)); + return; + } + } else if ((LibInfo->has(LibFunc::log2) && Name == "log2") || + (LibInfo->has(LibFunc::log2f) && Name == "log2f") || + (LibInfo->has(LibFunc::log2l) && Name == "log2l")) { + if (I.getNumArgOperands() == 1 && // Basic sanity checks. + I.getArgOperand(0)->getType()->isFloatingPointTy() && + I.getType() == I.getArgOperand(0)->getType()) { + SDValue Tmp = getValue(I.getArgOperand(0)); + setValue(&I, DAG.getNode(ISD::FLOG2, getCurDebugLoc(), + Tmp.getValueType(), Tmp)); + return; + } + } else if ((LibInfo->has(LibFunc::exp2) && Name == "exp2") || + (LibInfo->has(LibFunc::exp2f) && Name == "exp2f") || + (LibInfo->has(LibFunc::exp2l) && Name == "exp2l")) { + if (I.getNumArgOperands() == 1 && // Basic sanity checks. + I.getArgOperand(0)->getType()->isFloatingPointTy() && + I.getType() == I.getArgOperand(0)->getType()) { + SDValue Tmp = getValue(I.getArgOperand(0)); + setValue(&I, DAG.getNode(ISD::FEXP2, getCurDebugLoc(), + Tmp.getValueType(), Tmp)); + return; + } } else if (Name == "memcmp") { if (visitMemCmpCall(I)) return; @@ -6516,10 +6609,10 @@ SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V, unsigned Reg) { /// isOnlyUsedInEntryBlock - If the specified argument is only used in the /// entry block, return true. This includes arguments used by switches, since /// the switch may expand into multiple basic blocks. -static bool isOnlyUsedInEntryBlock(const Argument *A) { +static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) { // With FastISel active, we may be splitting blocks, so force creation // of virtual registers for all non-dead arguments. - if (EnableFastISel) + if (FastISel) return A->use_empty(); const BasicBlock *Entry = A->getParent()->begin(); @@ -6709,7 +6802,7 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) { SDB->getCurDebugLoc()); SDB->setValue(I, Res); - if (!EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) { + if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) { if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(Res.getOperand(0).getNode())) if (FrameIndexSDNode *FI = @@ -6719,7 +6812,7 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) { // If this argument is live outside of the entry block, insert a copy from // wherever we got it to the vreg that other BB's will reference it as. - if (!EnableFastISel && Res.getOpcode() == ISD::CopyFromReg) { + if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::CopyFromReg) { // If we can, though, try to skip creating an unnecessary vreg. // FIXME: This isn't very clean... it would be nice to make this more // general. It's also subtly incompatible with the hacks FastISel @@ -6730,7 +6823,7 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) { continue; } } - if (!isOnlyUsedInEntryBlock(I)) { + if (!isOnlyUsedInEntryBlock(I, TM.Options.EnableFastISel)) { FuncInfo->InitializeRegForValue(I); SDB->CopyToExportRegsIfNeeded(I); } diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 0a21ca3..5147b6c 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -67,6 +67,7 @@ class SIToFPInst; class StoreInst; class SwitchInst; class TargetData; +class TargetLibraryInfo; class TargetLowering; class TruncInst; class UIToFPInst; @@ -294,6 +295,7 @@ public: SelectionDAG &DAG; const TargetData *TD; AliasAnalysis *AA; + const TargetLibraryInfo *LibInfo; /// SwitchCases - Vector of CaseBlock structures used to communicate /// SwitchInst code generation information. @@ -338,7 +340,8 @@ public: HasTailCall(false), Context(dag.getContext()) { } - void init(GCFunctionInfo *gfi, AliasAnalysis &aa); + void init(GCFunctionInfo *gfi, AliasAnalysis &aa, + const TargetLibraryInfo *li); /// clear - Clear out the current SelectionDAG and the associated /// state and prepare this SelectionDAGBuilder object to be used diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index 8cecc17..3c95059 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -41,6 +41,7 @@ #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetIntrinsicInfo.h" #include "llvm/Target/TargetInstrInfo.h" +#include "llvm/Target/TargetLibraryInfo.h" #include "llvm/Target/TargetLowering.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" @@ -61,6 +62,81 @@ STATISTIC(NumFastIselBlocks, "Number of blocks selected entirely by fast isel"); STATISTIC(NumDAGBlocks, "Number of blocks selected using DAG"); STATISTIC(NumDAGIselRetries,"Number of times dag isel has to try another path"); +#ifndef NDEBUG +static cl::opt<bool> +EnableFastISelVerbose2("fast-isel-verbose2", cl::Hidden, + cl::desc("Enable extra verbose messages in the \"fast\" " + "instruction selector")); + // Terminators +STATISTIC(NumFastIselFailRet,"Fast isel fails on Ret"); +STATISTIC(NumFastIselFailBr,"Fast isel fails on Br"); +STATISTIC(NumFastIselFailSwitch,"Fast isel fails on Switch"); +STATISTIC(NumFastIselFailIndirectBr,"Fast isel fails on IndirectBr"); +STATISTIC(NumFastIselFailInvoke,"Fast isel fails on Invoke"); +STATISTIC(NumFastIselFailResume,"Fast isel fails on Resume"); +STATISTIC(NumFastIselFailUnwind,"Fast isel fails on Unwind"); +STATISTIC(NumFastIselFailUnreachable,"Fast isel fails on Unreachable"); + + // Standard binary operators... +STATISTIC(NumFastIselFailAdd,"Fast isel fails on Add"); +STATISTIC(NumFastIselFailFAdd,"Fast isel fails on FAdd"); +STATISTIC(NumFastIselFailSub,"Fast isel fails on Sub"); +STATISTIC(NumFastIselFailFSub,"Fast isel fails on FSub"); +STATISTIC(NumFastIselFailMul,"Fast isel fails on Mul"); +STATISTIC(NumFastIselFailFMul,"Fast isel fails on FMul"); +STATISTIC(NumFastIselFailUDiv,"Fast isel fails on UDiv"); +STATISTIC(NumFastIselFailSDiv,"Fast isel fails on SDiv"); +STATISTIC(NumFastIselFailFDiv,"Fast isel fails on FDiv"); +STATISTIC(NumFastIselFailURem,"Fast isel fails on URem"); +STATISTIC(NumFastIselFailSRem,"Fast isel fails on SRem"); +STATISTIC(NumFastIselFailFRem,"Fast isel fails on FRem"); + + // Logical operators... +STATISTIC(NumFastIselFailAnd,"Fast isel fails on And"); +STATISTIC(NumFastIselFailOr,"Fast isel fails on Or"); +STATISTIC(NumFastIselFailXor,"Fast isel fails on Xor"); + + // Memory instructions... +STATISTIC(NumFastIselFailAlloca,"Fast isel fails on Alloca"); +STATISTIC(NumFastIselFailLoad,"Fast isel fails on Load"); +STATISTIC(NumFastIselFailStore,"Fast isel fails on Store"); +STATISTIC(NumFastIselFailAtomicCmpXchg,"Fast isel fails on AtomicCmpXchg"); +STATISTIC(NumFastIselFailAtomicRMW,"Fast isel fails on AtomicRWM"); +STATISTIC(NumFastIselFailFence,"Fast isel fails on Frence"); +STATISTIC(NumFastIselFailGetElementPtr,"Fast isel fails on GetElementPtr"); + + // Convert instructions... +STATISTIC(NumFastIselFailTrunc,"Fast isel fails on Trunc"); +STATISTIC(NumFastIselFailZExt,"Fast isel fails on ZExt"); +STATISTIC(NumFastIselFailSExt,"Fast isel fails on SExt"); +STATISTIC(NumFastIselFailFPTrunc,"Fast isel fails on FPTrunc"); +STATISTIC(NumFastIselFailFPExt,"Fast isel fails on FPExt"); +STATISTIC(NumFastIselFailFPToUI,"Fast isel fails on FPToUI"); +STATISTIC(NumFastIselFailFPToSI,"Fast isel fails on FPToSI"); +STATISTIC(NumFastIselFailUIToFP,"Fast isel fails on UIToFP"); +STATISTIC(NumFastIselFailSIToFP,"Fast isel fails on SIToFP"); +STATISTIC(NumFastIselFailIntToPtr,"Fast isel fails on IntToPtr"); +STATISTIC(NumFastIselFailPtrToInt,"Fast isel fails on PtrToInt"); +STATISTIC(NumFastIselFailBitCast,"Fast isel fails on BitCast"); + + // Other instructions... +STATISTIC(NumFastIselFailICmp,"Fast isel fails on ICmp"); +STATISTIC(NumFastIselFailFCmp,"Fast isel fails on FCmp"); +STATISTIC(NumFastIselFailPHI,"Fast isel fails on PHI"); +STATISTIC(NumFastIselFailSelect,"Fast isel fails on Select"); +STATISTIC(NumFastIselFailCall,"Fast isel fails on Call"); +STATISTIC(NumFastIselFailShl,"Fast isel fails on Shl"); +STATISTIC(NumFastIselFailLShr,"Fast isel fails on LShr"); +STATISTIC(NumFastIselFailAShr,"Fast isel fails on AShr"); +STATISTIC(NumFastIselFailVAArg,"Fast isel fails on VAArg"); +STATISTIC(NumFastIselFailExtractElement,"Fast isel fails on ExtractElement"); +STATISTIC(NumFastIselFailInsertElement,"Fast isel fails on InsertElement"); +STATISTIC(NumFastIselFailShuffleVector,"Fast isel fails on ShuffleVector"); +STATISTIC(NumFastIselFailExtractValue,"Fast isel fails on ExtractValue"); +STATISTIC(NumFastIselFailInsertValue,"Fast isel fails on InsertValue"); +STATISTIC(NumFastIselFailLandingPad,"Fast isel fails on LandingPad"); +#endif + static cl::opt<bool> EnableFastISelVerbose("fast-isel-verbose", cl::Hidden, cl::desc("Enable verbose messages in the \"fast\" " @@ -177,7 +253,7 @@ TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, void TargetLowering::AdjustInstrPostInstrSelection(MachineInstr *MI, SDNode *Node) const { - assert(!MI->getDesc().hasPostISelHook() && + assert(!MI->hasPostISelHook() && "If a target marks an instruction with 'hasPostISelHook', " "it must implement TargetLowering::AdjustInstrPostInstrSelection!"); } @@ -190,7 +266,7 @@ SelectionDAGISel::SelectionDAGISel(const TargetMachine &tm, CodeGenOpt::Level OL) : MachineFunctionPass(ID), TM(tm), TLI(*tm.getTargetLowering()), FuncInfo(new FunctionLoweringInfo(TLI)), - CurDAG(new SelectionDAG(tm)), + CurDAG(new SelectionDAG(tm, OL)), SDB(new SelectionDAGBuilder(*CurDAG, *FuncInfo, OL)), GFI(), OptLevel(OL), @@ -198,6 +274,7 @@ SelectionDAGISel::SelectionDAGISel(const TargetMachine &tm, initializeGCModuleInfoPass(*PassRegistry::getPassRegistry()); initializeAliasAnalysisAnalysisGroup(*PassRegistry::getPassRegistry()); initializeBranchProbabilityInfoPass(*PassRegistry::getPassRegistry()); + initializeTargetLibraryInfoPass(*PassRegistry::getPassRegistry()); } SelectionDAGISel::~SelectionDAGISel() { @@ -211,6 +288,7 @@ void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const { AU.addPreserved<AliasAnalysis>(); AU.addRequired<GCModuleInfo>(); AU.addPreserved<GCModuleInfo>(); + AU.addRequired<TargetLibraryInfo>(); if (UseMBPI && OptLevel != CodeGenOpt::None) AU.addRequired<BranchProbabilityInfo>(); MachineFunctionPass::getAnalysisUsage(AU); @@ -256,9 +334,9 @@ static void SplitCriticalSideEffectEdges(Function &Fn, Pass *SDISel) { bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { // Do some sanity-checking on the command-line options. - assert((!EnableFastISelVerbose || EnableFastISel) && + assert((!EnableFastISelVerbose || TM.Options.EnableFastISel) && "-fast-isel-verbose requires -fast-isel"); - assert((!EnableFastISelAbort || EnableFastISel) && + assert((!EnableFastISelAbort || TM.Options.EnableFastISel) && "-fast-isel-abort requires -fast-isel"); const Function &Fn = *mf.getFunction(); @@ -268,6 +346,7 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { MF = &mf; RegInfo = &MF->getRegInfo(); AA = &getAnalysis<AliasAnalysis>(); + LibInfo = &getAnalysis<TargetLibraryInfo>(); GFI = Fn.hasGC() ? &getAnalysis<GCModuleInfo>().getFunctionInfo(Fn) : 0; DEBUG(dbgs() << "\n\n\n=== " << Fn.getName() << "\n"); @@ -282,7 +361,7 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { else FuncInfo->BPI = 0; - SDB->init(GFI, *AA); + SDB->init(GFI, *AA, LibInfo); SelectAllBasicBlocks(Fn); @@ -346,7 +425,8 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { TII.get(TargetOpcode::DBG_VALUE)) .addReg(CopyUseMI->getOperand(0).getReg(), RegState::Debug) .addImm(Offset).addMetadata(Variable); - EntryMBB->insertAfter(CopyUseMI, NewMI); + MachineBasicBlock::iterator Pos = CopyUseMI; + EntryMBB->insertAfter(Pos, NewMI); } } } @@ -820,10 +900,88 @@ static bool isFoldedOrDeadInstruction(const Instruction *I, !FuncInfo->isExportedInst(I); // Exported instrs must be computed. } +#ifndef NDEBUG +static void collectFailStats(const Instruction *I) { + switch (I->getOpcode()) { + default: assert (0 && "<Invalid operator> "); + + // Terminators + case Instruction::Ret: NumFastIselFailRet++; return; + case Instruction::Br: NumFastIselFailBr++; return; + case Instruction::Switch: NumFastIselFailSwitch++; return; + case Instruction::IndirectBr: NumFastIselFailIndirectBr++; return; + case Instruction::Invoke: NumFastIselFailInvoke++; return; + case Instruction::Resume: NumFastIselFailResume++; return; + case Instruction::Unwind: NumFastIselFailUnwind++; return; + case Instruction::Unreachable: NumFastIselFailUnreachable++; return; + + // Standard binary operators... + case Instruction::Add: NumFastIselFailAdd++; return; + case Instruction::FAdd: NumFastIselFailFAdd++; return; + case Instruction::Sub: NumFastIselFailSub++; return; + case Instruction::FSub: NumFastIselFailFSub++; return; + case Instruction::Mul: NumFastIselFailMul++; return; + case Instruction::FMul: NumFastIselFailFMul++; return; + case Instruction::UDiv: NumFastIselFailUDiv++; return; + case Instruction::SDiv: NumFastIselFailSDiv++; return; + case Instruction::FDiv: NumFastIselFailFDiv++; return; + case Instruction::URem: NumFastIselFailURem++; return; + case Instruction::SRem: NumFastIselFailSRem++; return; + case Instruction::FRem: NumFastIselFailFRem++; return; + + // Logical operators... + case Instruction::And: NumFastIselFailAnd++; return; + case Instruction::Or: NumFastIselFailOr++; return; + case Instruction::Xor: NumFastIselFailXor++; return; + + // Memory instructions... + case Instruction::Alloca: NumFastIselFailAlloca++; return; + case Instruction::Load: NumFastIselFailLoad++; return; + case Instruction::Store: NumFastIselFailStore++; return; + case Instruction::AtomicCmpXchg: NumFastIselFailAtomicCmpXchg++; return; + case Instruction::AtomicRMW: NumFastIselFailAtomicRMW++; return; + case Instruction::Fence: NumFastIselFailFence++; return; + case Instruction::GetElementPtr: NumFastIselFailGetElementPtr++; return; + + // Convert instructions... + case Instruction::Trunc: NumFastIselFailTrunc++; return; + case Instruction::ZExt: NumFastIselFailZExt++; return; + case Instruction::SExt: NumFastIselFailSExt++; return; + case Instruction::FPTrunc: NumFastIselFailFPTrunc++; return; + case Instruction::FPExt: NumFastIselFailFPExt++; return; + case Instruction::FPToUI: NumFastIselFailFPToUI++; return; + case Instruction::FPToSI: NumFastIselFailFPToSI++; return; + case Instruction::UIToFP: NumFastIselFailUIToFP++; return; + case Instruction::SIToFP: NumFastIselFailSIToFP++; return; + case Instruction::IntToPtr: NumFastIselFailIntToPtr++; return; + case Instruction::PtrToInt: NumFastIselFailPtrToInt++; return; + case Instruction::BitCast: NumFastIselFailBitCast++; return; + + // Other instructions... + case Instruction::ICmp: NumFastIselFailICmp++; return; + case Instruction::FCmp: NumFastIselFailFCmp++; return; + case Instruction::PHI: NumFastIselFailPHI++; return; + case Instruction::Select: NumFastIselFailSelect++; return; + case Instruction::Call: NumFastIselFailCall++; return; + case Instruction::Shl: NumFastIselFailShl++; return; + case Instruction::LShr: NumFastIselFailLShr++; return; + case Instruction::AShr: NumFastIselFailAShr++; return; + case Instruction::VAArg: NumFastIselFailVAArg++; return; + case Instruction::ExtractElement: NumFastIselFailExtractElement++; return; + case Instruction::InsertElement: NumFastIselFailInsertElement++; return; + case Instruction::ShuffleVector: NumFastIselFailShuffleVector++; return; + case Instruction::ExtractValue: NumFastIselFailExtractValue++; return; + case Instruction::InsertValue: NumFastIselFailInsertValue++; return; + case Instruction::LandingPad: NumFastIselFailLandingPad++; return; + } + return; +} +#endif + void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { // Initialize the Fast-ISel state, if needed. FastISel *FastIS = 0; - if (EnableFastISel) + if (TM.Options.EnableFastISel) FastIS = TLI.createFastISel(*FuncInfo); // Iterate over all basic blocks in the function. @@ -931,6 +1089,11 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { continue; } +#ifndef NDEBUG + if (EnableFastISelVerbose2) + collectFailStats(Inst); +#endif + // Then handle certain instructions as single-LLVM-Instruction blocks. if (isa<CallInst>(Inst)) { diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp index d7bad43..9ced1ac 100644 --- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -577,16 +577,26 @@ TargetLowering::TargetLowering(const TargetMachine &tm, setOperationAction(ISD::ConstantFP, MVT::f80, Expand); // These library functions default to expand. - setOperationAction(ISD::FLOG , MVT::f64, Expand); - setOperationAction(ISD::FLOG2, MVT::f64, Expand); - setOperationAction(ISD::FLOG10,MVT::f64, Expand); - setOperationAction(ISD::FEXP , MVT::f64, Expand); - setOperationAction(ISD::FEXP2, MVT::f64, Expand); - setOperationAction(ISD::FLOG , MVT::f32, Expand); - setOperationAction(ISD::FLOG2, MVT::f32, Expand); - setOperationAction(ISD::FLOG10,MVT::f32, Expand); - setOperationAction(ISD::FEXP , MVT::f32, Expand); - setOperationAction(ISD::FEXP2, MVT::f32, Expand); + setOperationAction(ISD::FLOG , MVT::f64, Expand); + setOperationAction(ISD::FLOG2, MVT::f64, Expand); + setOperationAction(ISD::FLOG10, MVT::f64, Expand); + setOperationAction(ISD::FEXP , MVT::f64, Expand); + setOperationAction(ISD::FEXP2, MVT::f64, Expand); + setOperationAction(ISD::FFLOOR, MVT::f64, Expand); + setOperationAction(ISD::FNEARBYINT, MVT::f64, Expand); + setOperationAction(ISD::FCEIL, MVT::f64, Expand); + setOperationAction(ISD::FRINT, MVT::f64, Expand); + setOperationAction(ISD::FTRUNC, MVT::f64, Expand); + setOperationAction(ISD::FLOG , MVT::f32, Expand); + setOperationAction(ISD::FLOG2, MVT::f32, Expand); + setOperationAction(ISD::FLOG10, MVT::f32, Expand); + setOperationAction(ISD::FEXP , MVT::f32, Expand); + setOperationAction(ISD::FEXP2, MVT::f32, Expand); + setOperationAction(ISD::FFLOOR, MVT::f32, Expand); + setOperationAction(ISD::FNEARBYINT, MVT::f32, Expand); + setOperationAction(ISD::FCEIL, MVT::f32, Expand); + setOperationAction(ISD::FRINT, MVT::f32, Expand); + setOperationAction(ISD::FTRUNC, MVT::f32, Expand); // Default ISD::TRAP to expand (which turns it into abort). setOperationAction(ISD::TRAP, MVT::Other, Expand); @@ -1473,9 +1483,8 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op, if (InOp.getNode()->getOpcode() == ISD::ANY_EXTEND) { SDValue InnerOp = InOp.getNode()->getOperand(0); EVT InnerVT = InnerOp.getValueType(); - if ((APInt::getHighBitsSet(BitWidth, - BitWidth - InnerVT.getSizeInBits()) & - DemandedMask) == 0 && + unsigned InnerBits = InnerVT.getSizeInBits(); + if (ShAmt < InnerBits && NewMask.lshr(InnerBits) == 0 && isTypeDesirableForOp(ISD::SHL, InnerVT)) { EVT ShTy = getShiftAmountTy(InnerVT); if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits())) @@ -1545,7 +1554,7 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op, // always convert this into a logical shr, even if the shift amount is // variable. The low bit of the shift cannot be an input sign bit unless // the shift amount is >= the size of the datatype, which is undefined. - if (DemandedMask == 1) + if (NewMask == 1) return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, Op.getValueType(), Op.getOperand(0), Op.getOperand(1))); @@ -1783,7 +1792,8 @@ bool TargetLowering::SimplifyDemandedBits(SDValue Op, case ISD::BITCAST: // If this is an FP->Int bitcast and if the sign bit is the only // thing demanded, turn this into a FGETSIGN. - if (!Op.getValueType().isVector() && + if (!TLO.LegalOperations() && + !Op.getValueType().isVector() && !Op.getOperand(0).getValueType().isVector() && NewMask == APInt::getSignBit(Op.getValueType().getSizeInBits()) && Op.getOperand(0).getValueType().isFloatingPoint()) { diff --git a/lib/CodeGen/ShrinkWrapping.cpp b/lib/CodeGen/ShrinkWrapping.cpp index 13f269e..70fcf55 100644 --- a/lib/CodeGen/ShrinkWrapping.cpp +++ b/lib/CodeGen/ShrinkWrapping.cpp @@ -124,7 +124,7 @@ MachineLoop* PEI::getTopLevelLoopParent(MachineLoop *LP) { } bool PEI::isReturnBlock(MachineBasicBlock* MBB) { - return (MBB && !MBB->empty() && MBB->back().getDesc().isReturn()); + return (MBB && !MBB->empty() && MBB->back().isReturn()); } // Initialize shrink wrapping DFA sets, called before iterations. diff --git a/lib/CodeGen/SjLjEHPrepare.cpp b/lib/CodeGen/SjLjEHPrepare.cpp index c865192..8e2f74f 100644 --- a/lib/CodeGen/SjLjEHPrepare.cpp +++ b/lib/CodeGen/SjLjEHPrepare.cpp @@ -69,6 +69,8 @@ namespace { private: bool setupEntryBlockAndCallSites(Function &F); + void substituteLPadValues(LandingPadInst *LPI, Value *ExnVal, + Value *SelVal); Value *setupFunctionContext(Function &F, ArrayRef<LandingPadInst*> LPads); void lowerIncomingArguments(Function &F); void lowerAcrossUnwindEdges(Function &F, ArrayRef<InvokeInst*> Invokes); @@ -138,6 +140,38 @@ static void MarkBlocksLiveIn(BasicBlock *BB, MarkBlocksLiveIn(*PI, LiveBBs); } +/// substituteLPadValues - Substitute the values returned by the landingpad +/// instruction with those returned by the personality function. +void SjLjEHPass::substituteLPadValues(LandingPadInst *LPI, Value *ExnVal, + Value *SelVal) { + SmallVector<Value*, 8> UseWorkList(LPI->use_begin(), LPI->use_end()); + while (!UseWorkList.empty()) { + Value *Val = UseWorkList.pop_back_val(); + ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Val); + if (!EVI) continue; + if (EVI->getNumIndices() != 1) continue; + if (*EVI->idx_begin() == 0) + EVI->replaceAllUsesWith(ExnVal); + else if (*EVI->idx_begin() == 1) + EVI->replaceAllUsesWith(SelVal); + if (EVI->getNumUses() == 0) + EVI->eraseFromParent(); + } + + if (LPI->getNumUses() == 0) return; + + // There are still some uses of LPI. Construct an aggregate with the exception + // values and replace the LPI with that aggregate. + Type *LPadType = LPI->getType(); + Value *LPadVal = UndefValue::get(LPadType); + IRBuilder<> + Builder(llvm::next(BasicBlock::iterator(cast<Instruction>(SelVal)))); + LPadVal = Builder.CreateInsertValue(LPadVal, ExnVal, 0, "lpad.val"); + LPadVal = Builder.CreateInsertValue(LPadVal, SelVal, 1, "lpad.val"); + + LPI->replaceAllUsesWith(LPadVal); +} + /// setupFunctionContext - Allocate the function context on the stack and fill /// it with all of the data that we know at this point. Value *SjLjEHPass:: @@ -189,12 +223,7 @@ setupFunctionContext(Function &F, ArrayRef<LandingPadInst*> LPads) { ExnVal = Builder.CreateIntToPtr(ExnVal, Type::getInt8PtrTy(F.getContext())); Value *SelVal = Builder.CreateLoad(SelectorAddr, true, "exn_selector_val"); - Type *LPadType = LPI->getType(); - Value *LPadVal = UndefValue::get(LPadType); - LPadVal = Builder.CreateInsertValue(LPadVal, ExnVal, 0, "lpad.val"); - LPadVal = Builder.CreateInsertValue(LPadVal, SelVal, 1, "lpad.val"); - - LPI->replaceAllUsesWith(LPadVal); + substituteLPadValues(LPI, ExnVal, SelVal); } // Personality function diff --git a/lib/CodeGen/SplitKit.cpp b/lib/CodeGen/SplitKit.cpp index 751d604..c086073 100644 --- a/lib/CodeGen/SplitKit.cpp +++ b/lib/CodeGen/SplitKit.cpp @@ -80,7 +80,7 @@ SlotIndex SplitAnalysis::computeLastSplitPoint(unsigned Num) { for (MachineBasicBlock::const_iterator I = MBB->end(), E = MBB->begin(); I != E;) { --I; - if (I->getDesc().isCall()) { + if (I->isCall()) { LSP.second = LIS.getInstructionIndex(I); break; } diff --git a/lib/CodeGen/Splitter.cpp b/lib/CodeGen/Splitter.cpp deleted file mode 100644 index 16cf9b8..0000000 --- a/lib/CodeGen/Splitter.cpp +++ /dev/null @@ -1,827 +0,0 @@ -//===-- llvm/CodeGen/Splitter.cpp - Splitter -----------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#define DEBUG_TYPE "loopsplitter" - -#include "Splitter.h" - -#include "llvm/Module.h" -#include "llvm/CodeGen/CalcSpillWeights.h" -#include "llvm/CodeGen/LiveIntervalAnalysis.h" -#include "llvm/CodeGen/LiveStackAnalysis.h" -#include "llvm/CodeGen/MachineDominators.h" -#include "llvm/CodeGen/MachineInstrBuilder.h" -#include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/Passes.h" -#include "llvm/CodeGen/SlotIndexes.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetMachine.h" -#include "llvm/Target/TargetInstrInfo.h" - -using namespace llvm; - -char LoopSplitter::ID = 0; -INITIALIZE_PASS_BEGIN(LoopSplitter, "loop-splitting", - "Split virtual regists across loop boundaries.", false, false) -INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) -INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) -INITIALIZE_PASS_DEPENDENCY(SlotIndexes) -INITIALIZE_PASS_DEPENDENCY(LiveIntervals) -INITIALIZE_PASS_END(LoopSplitter, "loop-splitting", - "Split virtual regists across loop boundaries.", false, false) - -namespace llvm { - - class StartSlotComparator { - public: - StartSlotComparator(LiveIntervals &lis) : lis(lis) {} - bool operator()(const MachineBasicBlock *mbb1, - const MachineBasicBlock *mbb2) const { - return lis.getMBBStartIdx(mbb1) < lis.getMBBStartIdx(mbb2); - } - private: - LiveIntervals &lis; - }; - - class LoopSplit { - public: - LoopSplit(LoopSplitter &ls, LiveInterval &li, MachineLoop &loop) - : ls(ls), li(li), loop(loop), valid(true), inSplit(false), newLI(0) { - assert(TargetRegisterInfo::isVirtualRegister(li.reg) && - "Cannot split physical registers."); - } - - LiveInterval& getLI() const { return li; } - - MachineLoop& getLoop() const { return loop; } - - bool isValid() const { return valid; } - - bool isWorthwhile() const { return valid && (inSplit || !outSplits.empty()); } - - void invalidate() { valid = false; } - - void splitIncoming() { inSplit = true; } - - void splitOutgoing(MachineLoop::Edge &edge) { outSplits.insert(edge); } - - void addLoopInstr(MachineInstr *i) { loopInstrs.push_back(i); } - - void apply() { - assert(valid && "Attempt to apply invalid split."); - applyIncoming(); - applyOutgoing(); - copyRanges(); - renameInside(); - } - - private: - LoopSplitter &ls; - LiveInterval &li; - MachineLoop &loop; - bool valid, inSplit; - std::set<MachineLoop::Edge> outSplits; - std::vector<MachineInstr*> loopInstrs; - - LiveInterval *newLI; - std::map<VNInfo*, VNInfo*> vniMap; - - LiveInterval* getNewLI() { - if (newLI == 0) { - const TargetRegisterClass *trc = ls.mri->getRegClass(li.reg); - unsigned vreg = ls.mri->createVirtualRegister(trc); - newLI = &ls.lis->getOrCreateInterval(vreg); - } - return newLI; - } - - VNInfo* getNewVNI(VNInfo *oldVNI) { - VNInfo *newVNI = vniMap[oldVNI]; - - if (newVNI == 0) { - newVNI = getNewLI()->createValueCopy(oldVNI, - ls.lis->getVNInfoAllocator()); - vniMap[oldVNI] = newVNI; - } - - return newVNI; - } - - void applyIncoming() { - if (!inSplit) { - return; - } - - MachineBasicBlock *preHeader = loop.getLoopPreheader(); - if (preHeader == 0) { - assert(ls.canInsertPreHeader(loop) && - "Can't insert required preheader."); - preHeader = &ls.insertPreHeader(loop); - } - - LiveRange *preHeaderRange = - ls.lis->findExitingRange(li, preHeader); - assert(preHeaderRange != 0 && "Range not live into preheader."); - - // Insert the new copy. - MachineInstr *copy = BuildMI(*preHeader, - preHeader->getFirstTerminator(), - DebugLoc(), - ls.tii->get(TargetOpcode::COPY)) - .addReg(getNewLI()->reg, RegState::Define) - .addReg(li.reg, RegState::Kill); - - ls.lis->InsertMachineInstrInMaps(copy); - - SlotIndex copyDefIdx = ls.lis->getInstructionIndex(copy).getRegSlot(); - - VNInfo *newVal = getNewVNI(preHeaderRange->valno); - newVal->def = copyDefIdx; - newVal->setCopy(copy); - li.removeRange(copyDefIdx, ls.lis->getMBBEndIdx(preHeader), true); - - getNewLI()->addRange(LiveRange(copyDefIdx, - ls.lis->getMBBEndIdx(preHeader), - newVal)); - } - - void applyOutgoing() { - - for (std::set<MachineLoop::Edge>::iterator osItr = outSplits.begin(), - osEnd = outSplits.end(); - osItr != osEnd; ++osItr) { - MachineLoop::Edge edge = *osItr; - MachineBasicBlock *outBlock = edge.second; - if (ls.isCriticalEdge(edge)) { - assert(ls.canSplitEdge(edge) && "Unsplitable critical edge."); - outBlock = &ls.splitEdge(edge, loop); - } - LiveRange *outRange = ls.lis->findEnteringRange(li, outBlock); - assert(outRange != 0 && "No exiting range?"); - - MachineInstr *copy = BuildMI(*outBlock, outBlock->begin(), - DebugLoc(), - ls.tii->get(TargetOpcode::COPY)) - .addReg(li.reg, RegState::Define) - .addReg(getNewLI()->reg, RegState::Kill); - - ls.lis->InsertMachineInstrInMaps(copy); - - SlotIndex copyDefIdx = ls.lis->getInstructionIndex(copy).getRegSlot(); - - // Blow away output range definition. - outRange->valno->def = ls.lis->getInvalidIndex(); - li.removeRange(ls.lis->getMBBStartIdx(outBlock), copyDefIdx); - - SlotIndex newDefIdx = ls.lis->getMBBStartIdx(outBlock); - assert(ls.lis->getInstructionFromIndex(newDefIdx) == 0 && - "PHI def index points at actual instruction."); - VNInfo *newVal = - getNewLI()->getNextValue(newDefIdx, 0, ls.lis->getVNInfoAllocator()); - - getNewLI()->addRange(LiveRange(ls.lis->getMBBStartIdx(outBlock), - copyDefIdx, newVal)); - - } - } - - void copyRange(LiveRange &lr) { - std::pair<bool, LoopSplitter::SlotPair> lsr = - ls.getLoopSubRange(lr, loop); - - if (!lsr.first) - return; - - LiveRange loopRange(lsr.second.first, lsr.second.second, - getNewVNI(lr.valno)); - - li.removeRange(loopRange.start, loopRange.end, true); - - getNewLI()->addRange(loopRange); - } - - void copyRanges() { - for (std::vector<MachineInstr*>::iterator iItr = loopInstrs.begin(), - iEnd = loopInstrs.end(); - iItr != iEnd; ++iItr) { - MachineInstr &instr = **iItr; - SlotIndex instrIdx = ls.lis->getInstructionIndex(&instr); - if (instr.modifiesRegister(li.reg, 0)) { - LiveRange *defRange = - li.getLiveRangeContaining(instrIdx.getRegSlot()); - if (defRange != 0) // May have caught this already. - copyRange(*defRange); - } - if (instr.readsRegister(li.reg, 0)) { - LiveRange *useRange = - li.getLiveRangeContaining(instrIdx.getRegSlot(true)); - if (useRange != 0) { // May have caught this already. - copyRange(*useRange); - } - } - } - - for (MachineLoop::block_iterator bbItr = loop.block_begin(), - bbEnd = loop.block_end(); - bbItr != bbEnd; ++bbItr) { - MachineBasicBlock &loopBlock = **bbItr; - LiveRange *enteringRange = - ls.lis->findEnteringRange(li, &loopBlock); - if (enteringRange != 0) { - copyRange(*enteringRange); - } - } - } - - void renameInside() { - for (std::vector<MachineInstr*>::iterator iItr = loopInstrs.begin(), - iEnd = loopInstrs.end(); - iItr != iEnd; ++iItr) { - MachineInstr &instr = **iItr; - for (unsigned i = 0; i < instr.getNumOperands(); ++i) { - MachineOperand &mop = instr.getOperand(i); - if (mop.isReg() && mop.getReg() == li.reg) { - mop.setReg(getNewLI()->reg); - } - } - } - } - - }; - - void LoopSplitter::getAnalysisUsage(AnalysisUsage &au) const { - au.addRequired<MachineDominatorTree>(); - au.addPreserved<MachineDominatorTree>(); - au.addRequired<MachineLoopInfo>(); - au.addPreserved<MachineLoopInfo>(); - au.addPreservedID(RegisterCoalescerPassID); - au.addPreserved<CalculateSpillWeights>(); - au.addPreserved<LiveStacks>(); - au.addRequired<SlotIndexes>(); - au.addPreserved<SlotIndexes>(); - au.addRequired<LiveIntervals>(); - au.addPreserved<LiveIntervals>(); - MachineFunctionPass::getAnalysisUsage(au); - } - - bool LoopSplitter::runOnMachineFunction(MachineFunction &fn) { - - mf = &fn; - mri = &mf->getRegInfo(); - tii = mf->getTarget().getInstrInfo(); - tri = mf->getTarget().getRegisterInfo(); - sis = &getAnalysis<SlotIndexes>(); - lis = &getAnalysis<LiveIntervals>(); - mli = &getAnalysis<MachineLoopInfo>(); - mdt = &getAnalysis<MachineDominatorTree>(); - - fqn = mf->getFunction()->getParent()->getModuleIdentifier() + "." + - mf->getFunction()->getName().str(); - - dbgs() << "Splitting " << mf->getFunction()->getName() << "."; - - dumpOddTerminators(); - -// dbgs() << "----------------------------------------\n"; -// lis->dump(); -// dbgs() << "----------------------------------------\n"; - -// std::deque<MachineLoop*> loops; -// std::copy(mli->begin(), mli->end(), std::back_inserter(loops)); -// dbgs() << "Loops:\n"; -// while (!loops.empty()) { -// MachineLoop &loop = *loops.front(); -// loops.pop_front(); -// std::copy(loop.begin(), loop.end(), std::back_inserter(loops)); - -// dumpLoopInfo(loop); -// } - - //lis->dump(); - //exit(0); - - // Setup initial intervals. - for (LiveIntervals::iterator liItr = lis->begin(), liEnd = lis->end(); - liItr != liEnd; ++liItr) { - LiveInterval *li = liItr->second; - - if (TargetRegisterInfo::isVirtualRegister(li->reg) && - !lis->intervalIsInOneMBB(*li)) { - intervals.push_back(li); - } - } - - processIntervals(); - - intervals.clear(); - -// dbgs() << "----------------------------------------\n"; -// lis->dump(); -// dbgs() << "----------------------------------------\n"; - - dumpOddTerminators(); - - //exit(1); - - return false; - } - - void LoopSplitter::releaseMemory() { - fqn.clear(); - intervals.clear(); - loopRangeMap.clear(); - } - - void LoopSplitter::dumpOddTerminators() { - for (MachineFunction::iterator bbItr = mf->begin(), bbEnd = mf->end(); - bbItr != bbEnd; ++bbItr) { - MachineBasicBlock *mbb = &*bbItr; - MachineBasicBlock *a = 0, *b = 0; - SmallVector<MachineOperand, 4> c; - if (tii->AnalyzeBranch(*mbb, a, b, c)) { - dbgs() << "MBB#" << mbb->getNumber() << " has multiway terminator.\n"; - dbgs() << " Terminators:\n"; - for (MachineBasicBlock::iterator iItr = mbb->begin(), iEnd = mbb->end(); - iItr != iEnd; ++iItr) { - MachineInstr *instr= &*iItr; - dbgs() << " " << *instr << ""; - } - dbgs() << "\n Listed successors: [ "; - for (MachineBasicBlock::succ_iterator sItr = mbb->succ_begin(), sEnd = mbb->succ_end(); - sItr != sEnd; ++sItr) { - MachineBasicBlock *succMBB = *sItr; - dbgs() << succMBB->getNumber() << " "; - } - dbgs() << "]\n\n"; - } - } - } - - void LoopSplitter::dumpLoopInfo(MachineLoop &loop) { - MachineBasicBlock &headerBlock = *loop.getHeader(); - typedef SmallVector<MachineLoop::Edge, 8> ExitEdgesList; - ExitEdgesList exitEdges; - loop.getExitEdges(exitEdges); - - dbgs() << " Header: BB#" << headerBlock.getNumber() << ", Contains: [ "; - for (std::vector<MachineBasicBlock*>::const_iterator - subBlockItr = loop.getBlocks().begin(), - subBlockEnd = loop.getBlocks().end(); - subBlockItr != subBlockEnd; ++subBlockItr) { - MachineBasicBlock &subBlock = **subBlockItr; - dbgs() << "BB#" << subBlock.getNumber() << " "; - } - dbgs() << "], Exit edges: [ "; - for (ExitEdgesList::iterator exitEdgeItr = exitEdges.begin(), - exitEdgeEnd = exitEdges.end(); - exitEdgeItr != exitEdgeEnd; ++exitEdgeItr) { - MachineLoop::Edge &exitEdge = *exitEdgeItr; - dbgs() << "(MBB#" << exitEdge.first->getNumber() - << ", MBB#" << exitEdge.second->getNumber() << ") "; - } - dbgs() << "], Sub-Loop Headers: [ "; - for (MachineLoop::iterator subLoopItr = loop.begin(), - subLoopEnd = loop.end(); - subLoopItr != subLoopEnd; ++subLoopItr) { - MachineLoop &subLoop = **subLoopItr; - MachineBasicBlock &subLoopBlock = *subLoop.getHeader(); - dbgs() << "BB#" << subLoopBlock.getNumber() << " "; - } - dbgs() << "]\n"; - } - - void LoopSplitter::updateTerminators(MachineBasicBlock &mbb) { - mbb.updateTerminator(); - - for (MachineBasicBlock::iterator miItr = mbb.begin(), miEnd = mbb.end(); - miItr != miEnd; ++miItr) { - if (lis->isNotInMIMap(miItr)) { - lis->InsertMachineInstrInMaps(miItr); - } - } - } - - bool LoopSplitter::canInsertPreHeader(MachineLoop &loop) { - MachineBasicBlock *header = loop.getHeader(); - MachineBasicBlock *a = 0, *b = 0; - SmallVector<MachineOperand, 4> c; - - for (MachineBasicBlock::pred_iterator pbItr = header->pred_begin(), - pbEnd = header->pred_end(); - pbItr != pbEnd; ++pbItr) { - MachineBasicBlock *predBlock = *pbItr; - if (!!tii->AnalyzeBranch(*predBlock, a, b, c)) { - return false; - } - } - - MachineFunction::iterator headerItr(header); - if (headerItr == mf->begin()) - return true; - MachineBasicBlock *headerLayoutPred = llvm::prior(headerItr); - assert(headerLayoutPred != 0 && "Header should have layout pred."); - - return (!tii->AnalyzeBranch(*headerLayoutPred, a, b, c)); - } - - MachineBasicBlock& LoopSplitter::insertPreHeader(MachineLoop &loop) { - assert(loop.getLoopPreheader() == 0 && "Loop already has preheader."); - - MachineBasicBlock &header = *loop.getHeader(); - - // Save the preds - we'll need to update them once we insert the preheader. - typedef std::set<MachineBasicBlock*> HeaderPreds; - HeaderPreds headerPreds; - - for (MachineBasicBlock::pred_iterator predItr = header.pred_begin(), - predEnd = header.pred_end(); - predItr != predEnd; ++predItr) { - if (!loop.contains(*predItr)) - headerPreds.insert(*predItr); - } - - assert(!headerPreds.empty() && "No predecessors for header?"); - - //dbgs() << fqn << " MBB#" << header.getNumber() << " inserting preheader..."; - - MachineBasicBlock *preHeader = - mf->CreateMachineBasicBlock(header.getBasicBlock()); - - assert(preHeader != 0 && "Failed to create pre-header."); - - mf->insert(header, preHeader); - - for (HeaderPreds::iterator hpItr = headerPreds.begin(), - hpEnd = headerPreds.end(); - hpItr != hpEnd; ++hpItr) { - assert(*hpItr != 0 && "How'd a null predecessor get into this set?"); - MachineBasicBlock &hp = **hpItr; - hp.ReplaceUsesOfBlockWith(&header, preHeader); - } - preHeader->addSuccessor(&header); - - MachineBasicBlock *oldLayoutPred = - llvm::prior(MachineFunction::iterator(preHeader)); - if (oldLayoutPred != 0) { - updateTerminators(*oldLayoutPred); - } - - lis->InsertMBBInMaps(preHeader); - - if (MachineLoop *parentLoop = loop.getParentLoop()) { - assert(parentLoop->getHeader() != loop.getHeader() && - "Parent loop has same header?"); - parentLoop->addBasicBlockToLoop(preHeader, mli->getBase()); - - // Invalidate all parent loop ranges. - while (parentLoop != 0) { - loopRangeMap.erase(parentLoop); - parentLoop = parentLoop->getParentLoop(); - } - } - - for (LiveIntervals::iterator liItr = lis->begin(), - liEnd = lis->end(); - liItr != liEnd; ++liItr) { - LiveInterval &li = *liItr->second; - - // Is this safe for physregs? - // TargetRegisterInfo::isPhysicalRegister(li.reg) || - if (!lis->isLiveInToMBB(li, &header)) - continue; - - if (lis->isLiveInToMBB(li, preHeader)) { - assert(lis->isLiveOutOfMBB(li, preHeader) && - "Range terminates in newly added preheader?"); - continue; - } - - bool insertRange = false; - - for (MachineBasicBlock::pred_iterator predItr = preHeader->pred_begin(), - predEnd = preHeader->pred_end(); - predItr != predEnd; ++predItr) { - MachineBasicBlock *predMBB = *predItr; - if (lis->isLiveOutOfMBB(li, predMBB)) { - insertRange = true; - break; - } - } - - if (!insertRange) - continue; - - SlotIndex newDefIdx = lis->getMBBStartIdx(preHeader); - assert(lis->getInstructionFromIndex(newDefIdx) == 0 && - "PHI def index points at actual instruction."); - VNInfo *newVal = li.getNextValue(newDefIdx, 0, lis->getVNInfoAllocator()); - li.addRange(LiveRange(lis->getMBBStartIdx(preHeader), - lis->getMBBEndIdx(preHeader), - newVal)); - } - - - //dbgs() << "Dumping SlotIndexes:\n"; - //sis->dump(); - - //dbgs() << "done. (Added MBB#" << preHeader->getNumber() << ")\n"; - - return *preHeader; - } - - bool LoopSplitter::isCriticalEdge(MachineLoop::Edge &edge) { - assert(edge.first->succ_size() > 1 && "Non-sensical edge."); - if (edge.second->pred_size() > 1) - return true; - return false; - } - - bool LoopSplitter::canSplitEdge(MachineLoop::Edge &edge) { - MachineFunction::iterator outBlockItr(edge.second); - if (outBlockItr == mf->begin()) - return true; - MachineBasicBlock *outBlockLayoutPred = llvm::prior(outBlockItr); - assert(outBlockLayoutPred != 0 && "Should have a layout pred if out!=begin."); - MachineBasicBlock *a = 0, *b = 0; - SmallVector<MachineOperand, 4> c; - return (!tii->AnalyzeBranch(*outBlockLayoutPred, a, b, c) && - !tii->AnalyzeBranch(*edge.first, a, b, c)); - } - - MachineBasicBlock& LoopSplitter::splitEdge(MachineLoop::Edge &edge, - MachineLoop &loop) { - - MachineBasicBlock &inBlock = *edge.first; - MachineBasicBlock &outBlock = *edge.second; - - assert((inBlock.succ_size() > 1) && (outBlock.pred_size() > 1) && - "Splitting non-critical edge?"); - - //dbgs() << fqn << " Splitting edge (MBB#" << inBlock.getNumber() - // << " -> MBB#" << outBlock.getNumber() << ")..."; - - MachineBasicBlock *splitBlock = - mf->CreateMachineBasicBlock(); - - assert(splitBlock != 0 && "Failed to create split block."); - - mf->insert(&outBlock, splitBlock); - - inBlock.ReplaceUsesOfBlockWith(&outBlock, splitBlock); - splitBlock->addSuccessor(&outBlock); - - MachineBasicBlock *oldLayoutPred = - llvm::prior(MachineFunction::iterator(splitBlock)); - if (oldLayoutPred != 0) { - updateTerminators(*oldLayoutPred); - } - - lis->InsertMBBInMaps(splitBlock); - - loopRangeMap.erase(&loop); - - MachineLoop *splitParentLoop = loop.getParentLoop(); - while (splitParentLoop != 0 && - !splitParentLoop->contains(&outBlock)) { - splitParentLoop = splitParentLoop->getParentLoop(); - } - - if (splitParentLoop != 0) { - assert(splitParentLoop->contains(&loop) && - "Split-block parent doesn't contain original loop?"); - splitParentLoop->addBasicBlockToLoop(splitBlock, mli->getBase()); - - // Invalidate all parent loop ranges. - while (splitParentLoop != 0) { - loopRangeMap.erase(splitParentLoop); - splitParentLoop = splitParentLoop->getParentLoop(); - } - } - - - for (LiveIntervals::iterator liItr = lis->begin(), - liEnd = lis->end(); - liItr != liEnd; ++liItr) { - LiveInterval &li = *liItr->second; - bool intersects = lis->isLiveOutOfMBB(li, &inBlock) && - lis->isLiveInToMBB(li, &outBlock); - if (lis->isLiveInToMBB(li, splitBlock)) { - if (!intersects) { - li.removeRange(lis->getMBBStartIdx(splitBlock), - lis->getMBBEndIdx(splitBlock), true); - } - } else if (intersects) { - SlotIndex newDefIdx = lis->getMBBStartIdx(splitBlock); - assert(lis->getInstructionFromIndex(newDefIdx) == 0 && - "PHI def index points at actual instruction."); - VNInfo *newVal = li.getNextValue(newDefIdx, 0, - lis->getVNInfoAllocator()); - li.addRange(LiveRange(lis->getMBBStartIdx(splitBlock), - lis->getMBBEndIdx(splitBlock), - newVal)); - } - } - - //dbgs() << "done. (Added MBB#" << splitBlock->getNumber() << ")\n"; - - return *splitBlock; - } - - LoopSplitter::LoopRanges& LoopSplitter::getLoopRanges(MachineLoop &loop) { - typedef std::set<MachineBasicBlock*, StartSlotComparator> LoopMBBSet; - LoopRangeMap::iterator lrItr = loopRangeMap.find(&loop); - if (lrItr == loopRangeMap.end()) { - LoopMBBSet loopMBBs((StartSlotComparator(*lis))); - std::copy(loop.block_begin(), loop.block_end(), - std::inserter(loopMBBs, loopMBBs.begin())); - - assert(!loopMBBs.empty() && "No blocks in loop?"); - - LoopRanges &loopRanges = loopRangeMap[&loop]; - assert(loopRanges.empty() && "Loop encountered but not processed?"); - SlotIndex oldEnd = lis->getMBBEndIdx(*loopMBBs.begin()); - loopRanges.push_back( - std::make_pair(lis->getMBBStartIdx(*loopMBBs.begin()), - lis->getInvalidIndex())); - for (LoopMBBSet::iterator curBlockItr = llvm::next(loopMBBs.begin()), - curBlockEnd = loopMBBs.end(); - curBlockItr != curBlockEnd; ++curBlockItr) { - SlotIndex newStart = lis->getMBBStartIdx(*curBlockItr); - if (newStart != oldEnd) { - loopRanges.back().second = oldEnd; - loopRanges.push_back(std::make_pair(newStart, - lis->getInvalidIndex())); - } - oldEnd = lis->getMBBEndIdx(*curBlockItr); - } - - loopRanges.back().second = - lis->getMBBEndIdx(*llvm::prior(loopMBBs.end())); - - return loopRanges; - } - return lrItr->second; - } - - std::pair<bool, LoopSplitter::SlotPair> LoopSplitter::getLoopSubRange( - const LiveRange &lr, - MachineLoop &loop) { - LoopRanges &loopRanges = getLoopRanges(loop); - LoopRanges::iterator lrItr = loopRanges.begin(), - lrEnd = loopRanges.end(); - while (lrItr != lrEnd && lr.start >= lrItr->second) { - ++lrItr; - } - - if (lrItr == lrEnd) { - SlotIndex invalid = lis->getInvalidIndex(); - return std::make_pair(false, SlotPair(invalid, invalid)); - } - - SlotIndex srStart(lr.start < lrItr->first ? lrItr->first : lr.start); - SlotIndex srEnd(lr.end > lrItr->second ? lrItr->second : lr.end); - - return std::make_pair(true, SlotPair(srStart, srEnd)); - } - - void LoopSplitter::dumpLoopRanges(MachineLoop &loop) { - LoopRanges &loopRanges = getLoopRanges(loop); - dbgs() << "For loop MBB#" << loop.getHeader()->getNumber() << ", subranges are: [ "; - for (LoopRanges::iterator lrItr = loopRanges.begin(), lrEnd = loopRanges.end(); - lrItr != lrEnd; ++lrItr) { - dbgs() << "[" << lrItr->first << ", " << lrItr->second << ") "; - } - dbgs() << "]\n"; - } - - void LoopSplitter::processHeader(LoopSplit &split) { - MachineBasicBlock &header = *split.getLoop().getHeader(); - //dbgs() << " Processing loop header BB#" << header.getNumber() << "\n"; - - if (!lis->isLiveInToMBB(split.getLI(), &header)) - return; // Not live in, but nothing wrong so far. - - MachineBasicBlock *preHeader = split.getLoop().getLoopPreheader(); - if (!preHeader) { - - if (!canInsertPreHeader(split.getLoop())) { - split.invalidate(); - return; // Couldn't insert a pre-header. Bail on this interval. - } - - for (MachineBasicBlock::pred_iterator predItr = header.pred_begin(), - predEnd = header.pred_end(); - predItr != predEnd; ++predItr) { - if (lis->isLiveOutOfMBB(split.getLI(), *predItr)) { - split.splitIncoming(); - break; - } - } - } else if (lis->isLiveOutOfMBB(split.getLI(), preHeader)) { - split.splitIncoming(); - } - } - - void LoopSplitter::processLoopExits(LoopSplit &split) { - typedef SmallVector<MachineLoop::Edge, 8> ExitEdgesList; - ExitEdgesList exitEdges; - split.getLoop().getExitEdges(exitEdges); - - //dbgs() << " Processing loop exits:\n"; - - for (ExitEdgesList::iterator exitEdgeItr = exitEdges.begin(), - exitEdgeEnd = exitEdges.end(); - exitEdgeItr != exitEdgeEnd; ++exitEdgeItr) { - MachineLoop::Edge exitEdge = *exitEdgeItr; - - LiveRange *outRange = - split.getLI().getLiveRangeContaining(lis->getMBBStartIdx(exitEdge.second)); - - if (outRange != 0) { - if (isCriticalEdge(exitEdge) && !canSplitEdge(exitEdge)) { - split.invalidate(); - return; - } - - split.splitOutgoing(exitEdge); - } - } - } - - void LoopSplitter::processLoopUses(LoopSplit &split) { - std::set<MachineInstr*> processed; - - for (MachineRegisterInfo::reg_iterator - rItr = mri->reg_begin(split.getLI().reg), - rEnd = mri->reg_end(); - rItr != rEnd; ++rItr) { - MachineInstr &instr = *rItr; - if (split.getLoop().contains(&instr) && processed.count(&instr) == 0) { - split.addLoopInstr(&instr); - processed.insert(&instr); - } - } - - //dbgs() << " Rewriting reg" << li.reg << " to reg" << newLI->reg - // << " in blocks [ "; - //dbgs() << "]\n"; - } - - bool LoopSplitter::splitOverLoop(LiveInterval &li, MachineLoop &loop) { - assert(TargetRegisterInfo::isVirtualRegister(li.reg) && - "Attempt to split physical register."); - - LoopSplit split(*this, li, loop); - processHeader(split); - if (split.isValid()) - processLoopExits(split); - if (split.isValid()) - processLoopUses(split); - if (split.isValid() /* && split.isWorthwhile() */) { - split.apply(); - DEBUG(dbgs() << "Success.\n"); - return true; - } - DEBUG(dbgs() << "Failed.\n"); - return false; - } - - void LoopSplitter::processInterval(LiveInterval &li) { - std::deque<MachineLoop*> loops; - std::copy(mli->begin(), mli->end(), std::back_inserter(loops)); - - while (!loops.empty()) { - MachineLoop &loop = *loops.front(); - loops.pop_front(); - DEBUG( - dbgs() << fqn << " reg" << li.reg << " " << li.weight << " BB#" - << loop.getHeader()->getNumber() << " "; - ); - if (!splitOverLoop(li, loop)) { - // Couldn't split over outer loop, schedule sub-loops to be checked. - std::copy(loop.begin(), loop.end(), std::back_inserter(loops)); - } - } - } - - void LoopSplitter::processIntervals() { - while (!intervals.empty()) { - LiveInterval &li = *intervals.front(); - intervals.pop_front(); - - assert(!lis->intervalIsInOneMBB(li) && - "Single interval in process worklist."); - - processInterval(li); - } - } - -} diff --git a/lib/CodeGen/Splitter.h b/lib/CodeGen/Splitter.h deleted file mode 100644 index 9fb1b8b..0000000 --- a/lib/CodeGen/Splitter.h +++ /dev/null @@ -1,101 +0,0 @@ -//===-- llvm/CodeGen/Splitter.h - Splitter -*- C++ -*----------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CODEGEN_SPLITTER_H -#define LLVM_CODEGEN_SPLITTER_H - -#include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineLoopInfo.h" -#include "llvm/CodeGen/SlotIndexes.h" - -#include <deque> -#include <map> -#include <string> -#include <vector> - -namespace llvm { - - class LiveInterval; - class LiveIntervals; - struct LiveRange; - class LoopSplit; - class MachineDominatorTree; - class MachineRegisterInfo; - class SlotIndexes; - class TargetInstrInfo; - class VNInfo; - - class LoopSplitter : public MachineFunctionPass { - friend class LoopSplit; - public: - static char ID; - - LoopSplitter() : MachineFunctionPass(ID) { - initializeLoopSplitterPass(*PassRegistry::getPassRegistry()); - } - - virtual void getAnalysisUsage(AnalysisUsage &au) const; - - virtual bool runOnMachineFunction(MachineFunction &fn); - - virtual void releaseMemory(); - - - private: - - MachineFunction *mf; - LiveIntervals *lis; - MachineLoopInfo *mli; - MachineRegisterInfo *mri; - MachineDominatorTree *mdt; - SlotIndexes *sis; - const TargetInstrInfo *tii; - const TargetRegisterInfo *tri; - - std::string fqn; - std::deque<LiveInterval*> intervals; - - typedef std::pair<SlotIndex, SlotIndex> SlotPair; - typedef std::vector<SlotPair> LoopRanges; - typedef std::map<MachineLoop*, LoopRanges> LoopRangeMap; - LoopRangeMap loopRangeMap; - - void dumpLoopInfo(MachineLoop &loop); - - void dumpOddTerminators(); - - void updateTerminators(MachineBasicBlock &mbb); - - bool canInsertPreHeader(MachineLoop &loop); - MachineBasicBlock& insertPreHeader(MachineLoop &loop); - - bool isCriticalEdge(MachineLoop::Edge &edge); - bool canSplitEdge(MachineLoop::Edge &edge); - MachineBasicBlock& splitEdge(MachineLoop::Edge &edge, MachineLoop &loop); - - LoopRanges& getLoopRanges(MachineLoop &loop); - std::pair<bool, SlotPair> getLoopSubRange(const LiveRange &lr, - MachineLoop &loop); - - void dumpLoopRanges(MachineLoop &loop); - - void processHeader(LoopSplit &split); - void processLoopExits(LoopSplit &split); - void processLoopUses(LoopSplit &split); - - bool splitOverLoop(LiveInterval &li, MachineLoop &loop); - - void processInterval(LiveInterval &li); - - void processIntervals(); - }; - -} - -#endif diff --git a/lib/CodeGen/TailDuplication.cpp b/lib/CodeGen/TailDuplication.cpp index 3a6211a..031377b 100644 --- a/lib/CodeGen/TailDuplication.cpp +++ b/lib/CodeGen/TailDuplication.cpp @@ -553,7 +553,7 @@ TailDuplicatePass::shouldTailDuplicate(const MachineFunction &MF, bool HasIndirectbr = false; if (!TailBB.empty()) - HasIndirectbr = TailBB.back().getDesc().isIndirectBranch(); + HasIndirectbr = TailBB.back().isIndirectBranch(); if (HasIndirectbr && PreRegAlloc) MaxDuplicateCount = 20; @@ -561,22 +561,21 @@ TailDuplicatePass::shouldTailDuplicate(const MachineFunction &MF, // Check the instructions in the block to determine whether tail-duplication // is invalid or unlikely to be profitable. unsigned InstrCount = 0; - for (MachineBasicBlock::const_iterator I = TailBB.begin(); I != TailBB.end(); - ++I) { + for (MachineBasicBlock::iterator I = TailBB.begin(); I != TailBB.end(); ++I) { // Non-duplicable things shouldn't be tail-duplicated. - if (I->getDesc().isNotDuplicable()) + if (I->isNotDuplicable()) return false; // Do not duplicate 'return' instructions if this is a pre-regalloc run. // A return may expand into a lot more instructions (e.g. reload of callee // saved registers) after PEI. - if (PreRegAlloc && I->getDesc().isReturn()) + if (PreRegAlloc && I->isReturn()) return false; // Avoid duplicating calls before register allocation. Calls presents a // barrier to register allocation so duplicating them may end up increasing // spills. - if (PreRegAlloc && I->getDesc().isCall()) + if (PreRegAlloc && I->isCall()) return false; if (!I->isPHI() && !I->isDebugValue()) @@ -611,7 +610,7 @@ TailDuplicatePass::isSimpleBB(MachineBasicBlock *TailBB) { ++I; if (I == E) return true; - return I->getDesc().isUnconditionalBranch(); + return I->isUnconditionalBranch(); } static bool diff --git a/lib/CodeGen/TargetFrameLoweringImpl.cpp b/lib/CodeGen/TargetFrameLoweringImpl.cpp new file mode 100644 index 0000000..cadb878 --- /dev/null +++ b/lib/CodeGen/TargetFrameLoweringImpl.cpp @@ -0,0 +1,45 @@ +//===----- TargetFrameLoweringImpl.cpp - Implement target frame interface --==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// Implements the layout of a stack frame on the target machine. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/Target/TargetFrameLowering.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Target/TargetRegisterInfo.h" + +#include <cstdlib> +using namespace llvm; + +TargetFrameLowering::~TargetFrameLowering() { +} + +/// getFrameIndexOffset - Returns the displacement from the frame register to +/// the stack frame of the specified index. This is the default implementation +/// which is overridden for some targets. +int TargetFrameLowering::getFrameIndexOffset(const MachineFunction &MF, + int FI) const { + const MachineFrameInfo *MFI = MF.getFrameInfo(); + return MFI->getObjectOffset(FI) + MFI->getStackSize() - + getOffsetOfLocalArea() + MFI->getOffsetAdjustment(); +} + +int TargetFrameLowering::getFrameIndexReference(const MachineFunction &MF, + int FI, unsigned &FrameReg) const { + const TargetRegisterInfo *RI = MF.getTarget().getRegisterInfo(); + + // By default, assume all frame indices are referenced via whatever + // getFrameRegister() says. The target can override this if it's doing + // something different. + FrameReg = RI->getFrameRegister(MF); + return getFrameIndexOffset(MF, FI); +} diff --git a/lib/CodeGen/TargetInstrInfoImpl.cpp b/lib/CodeGen/TargetInstrInfoImpl.cpp index 08e2b16..7ed9455 100644 --- a/lib/CodeGen/TargetInstrInfoImpl.cpp +++ b/lib/CodeGen/TargetInstrInfoImpl.cpp @@ -24,6 +24,7 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/ScoreboardHazardRecognizer.h" #include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/MC/MCInstrItineraries.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" @@ -121,6 +122,9 @@ MachineInstr *TargetInstrInfoImpl::commuteInstruction(MachineInstr *MI, bool TargetInstrInfoImpl::findCommutedOpIndices(MachineInstr *MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const { + assert(!MI->isBundle() && + "TargetInstrInfoImpl::findCommutedOpIndices() can't handle bundles"); + const MCInstrDesc &MCID = MI->getDesc(); if (!MCID.isCommutable()) return false; @@ -136,11 +140,28 @@ bool TargetInstrInfoImpl::findCommutedOpIndices(MachineInstr *MI, } +bool +TargetInstrInfoImpl::isUnpredicatedTerminator(const MachineInstr *MI) const { + if (!MI->isTerminator()) return false; + + // Conditional branch is a special case. + if (MI->isBranch() && !MI->isBarrier()) + return true; + if (!MI->isPredicable()) + return true; + return !isPredicated(MI); +} + + bool TargetInstrInfoImpl::PredicateInstruction(MachineInstr *MI, const SmallVectorImpl<MachineOperand> &Pred) const { bool MadeChange = false; + + assert(!MI->isBundle() && + "TargetInstrInfoImpl::PredicateInstruction() can't handle bundles"); + const MCInstrDesc &MCID = MI->getDesc(); - if (!MCID.isPredicable()) + if (!MI->isPredicable()) return false; for (unsigned j = 0, i = 0, e = MI->getNumOperands(); i != e; ++i) { @@ -218,7 +239,7 @@ TargetInstrInfoImpl::produceSameValue(const MachineInstr *MI0, MachineInstr *TargetInstrInfoImpl::duplicate(MachineInstr *Orig, MachineFunction &MF) const { - assert(!Orig->getDesc().isNotDuplicable() && + assert(!Orig->isNotDuplicable() && "Instruction cannot be duplicated"); return MF.CloneMachineInstr(Orig); } @@ -288,10 +309,10 @@ TargetInstrInfo::foldMemoryOperand(MachineBasicBlock::iterator MI, if (MachineInstr *NewMI = foldMemoryOperandImpl(MF, MI, Ops, FI)) { // Add a memory operand, foldMemoryOperandImpl doesn't do that. assert((!(Flags & MachineMemOperand::MOStore) || - NewMI->getDesc().mayStore()) && + NewMI->mayStore()) && "Folded a def to a non-store!"); assert((!(Flags & MachineMemOperand::MOLoad) || - NewMI->getDesc().mayLoad()) && + NewMI->mayLoad()) && "Folded a use to a non-load!"); const MachineFrameInfo &MFI = *MF.getFrameInfo(); assert(MFI.getObjectOffset(FI) != -1); @@ -331,7 +352,7 @@ MachineInstr* TargetInstrInfo::foldMemoryOperand(MachineBasicBlock::iterator MI, const SmallVectorImpl<unsigned> &Ops, MachineInstr* LoadMI) const { - assert(LoadMI->getDesc().canFoldAsLoad() && "LoadMI isn't foldable!"); + assert(LoadMI->canFoldAsLoad() && "LoadMI isn't foldable!"); #ifndef NDEBUG for (unsigned i = 0, e = Ops.size(); i != e; ++i) assert(MI->getOperand(Ops[i]).isUse() && "Folding load into def!"); @@ -382,10 +403,8 @@ isReallyTriviallyReMaterializableGeneric(const MachineInstr *MI, MF.getFrameInfo()->isImmutableObjectIndex(FrameIdx)) return true; - const MCInstrDesc &MCID = MI->getDesc(); - // Avoid instructions obviously unsafe for remat. - if (MCID.isNotDuplicable() || MCID.mayStore() || + if (MI->isNotDuplicable() || MI->mayStore() || MI->hasUnmodeledSideEffects()) return false; @@ -395,7 +414,7 @@ isReallyTriviallyReMaterializableGeneric(const MachineInstr *MI, return false; // Avoid instructions which load from potentially varying memory. - if (MCID.mayLoad() && !MI->isInvariantLoad(AA)) + if (MI->mayLoad() && !MI->isInvariantLoad(AA)) return false; // If any of the registers accessed are non-constant, conservatively assume @@ -456,7 +475,7 @@ bool TargetInstrInfoImpl::isSchedulingBoundary(const MachineInstr *MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const{ // Terminators and labels can't be scheduled around. - if (MI->getDesc().isTerminator() || MI->isLabel()) + if (MI->isTerminator() || MI->isLabel()) return true; // Don't attempt to schedule around any instruction that defines @@ -492,3 +511,32 @@ CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II, return (ScheduleHazardRecognizer *) new ScoreboardHazardRecognizer(II, DAG, "post-RA-sched"); } + +int +TargetInstrInfo::getOperandLatency(const InstrItineraryData *ItinData, + SDNode *DefNode, unsigned DefIdx, + SDNode *UseNode, unsigned UseIdx) const { + if (!ItinData || ItinData->isEmpty()) + return -1; + + if (!DefNode->isMachineOpcode()) + return -1; + + unsigned DefClass = get(DefNode->getMachineOpcode()).getSchedClass(); + if (!UseNode->isMachineOpcode()) + return ItinData->getOperandCycle(DefClass, DefIdx); + unsigned UseClass = get(UseNode->getMachineOpcode()).getSchedClass(); + return ItinData->getOperandLatency(DefClass, DefIdx, UseClass, UseIdx); +} + +int TargetInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, + SDNode *N) const { + if (!ItinData || ItinData->isEmpty()) + return 1; + + if (!N->isMachineOpcode()) + return 1; + + return ItinData->getStageLatency(get(N->getMachineOpcode()).getSchedClass()); +} + diff --git a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index c43e5b6..7fe164a 100644 --- a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -536,9 +536,7 @@ getCFIPersonalitySymbol(const GlobalValue *GV, Mangler *Mang, // Add information about the stub reference to MachOMMI so that the stub // gets emitted by the asmprinter. MCSymbol *SSym = getContext().GetOrCreateSymbol(Name.str()); - MachineModuleInfoImpl::StubValueTy &StubSym = - GV->hasHiddenVisibility() ? MachOMMI.getHiddenGVStubEntry(SSym) : - MachOMMI.getGVStubEntry(SSym); + MachineModuleInfoImpl::StubValueTy &StubSym = MachOMMI.getGVStubEntry(SSym); if (StubSym.getPointer() == 0) { MCSymbol *Sym = Mang->getSymbol(GV); StubSym = MachineModuleInfoImpl::StubValueTy(Sym, !GV->hasLocalLinkage()); diff --git a/lib/CodeGen/TargetOptionsImpl.cpp b/lib/CodeGen/TargetOptionsImpl.cpp new file mode 100644 index 0000000..0f59d01 --- /dev/null +++ b/lib/CodeGen/TargetOptionsImpl.cpp @@ -0,0 +1,52 @@ +//===-- TargetOptionsImpl.cpp - Options that apply to all targets ----------==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the methods in the TargetOptions. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineFrameInfo.h" +#include "llvm/Target/TargetOptions.h" +using namespace llvm; + +/// DisableFramePointerElim - This returns true if frame pointer elimination +/// optimization should be disabled for the given machine function. +bool TargetOptions::DisableFramePointerElim(const MachineFunction &MF) const { + // Check to see if we should eliminate non-leaf frame pointers and then + // check to see if we should eliminate all frame pointers. + if (NoFramePointerElimNonLeaf && !NoFramePointerElim) { + const MachineFrameInfo *MFI = MF.getFrameInfo(); + return MFI->hasCalls(); + } + + return NoFramePointerElim; +} + +/// LessPreciseFPMAD - This flag return true when -enable-fp-mad option +/// is specified on the command line. When this flag is off(default), the +/// code generator is not allowed to generate mad (multiply add) if the +/// result is "less precise" than doing those operations individually. +bool TargetOptions::LessPreciseFPMAD() const { + return UnsafeFPMath || LessPreciseFPMADOption; +} + +/// HonorSignDependentRoundingFPMath - Return true if the codegen must assume +/// that the rounding mode of the FPU can change from its default. +bool TargetOptions::HonorSignDependentRoundingFPMath() const { + return !UnsafeFPMath && HonorSignDependentRoundingFPMathOption; +} + +/// getTrapFunctionName - If this returns a non-empty string, this means isel +/// should lower Intrinsic::trap to a call to the specified function name +/// instead of an ISD::TRAP node. +StringRef TargetOptions::getTrapFunctionName() const { + return TrapFuncName; +} + diff --git a/lib/CodeGen/TwoAddressInstructionPass.cpp b/lib/CodeGen/TwoAddressInstructionPass.cpp index a2e8134..6a63335 100644 --- a/lib/CodeGen/TwoAddressInstructionPass.cpp +++ b/lib/CodeGen/TwoAddressInstructionPass.cpp @@ -242,7 +242,7 @@ bool TwoAddressInstructionPass::Sink3AddrInstruction(MachineBasicBlock *MBB, // appropriate location, we can try to sink the current instruction // past it. if (!KillMI || KillMI->getParent() != MBB || KillMI == MI || - KillMI->getDesc().isTerminator()) + KillMI->isTerminator()) return false; // If any of the definitions are used by another instruction between the @@ -498,8 +498,7 @@ MachineInstr *findLocalKill(unsigned Reg, MachineBasicBlock *MBB, MachineInstr *UseMI = &*UI; if (UseMI == MI || UseMI->getParent() != MBB) continue; - DenseMap<MachineInstr*, unsigned>::iterator DI = DistanceMap.find(UseMI); - if (DI != DistanceMap.end()) + if (DistanceMap.count(UseMI)) continue; if (!UI.getOperand().isKill()) return 0; @@ -817,10 +816,9 @@ void TwoAddressInstructionPass::ProcessCopy(MachineInstr *MI, static bool isSafeToDelete(MachineInstr *MI, const TargetInstrInfo *TII, SmallVector<unsigned, 4> &Kills) { - const MCInstrDesc &MCID = MI->getDesc(); - if (MCID.mayStore() || MCID.isCall()) + if (MI->mayStore() || MI->isCall()) return false; - if (MCID.isTerminator() || MI->hasUnmodeledSideEffects()) + if (MI->isTerminator() || MI->hasUnmodeledSideEffects()) return false; for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { @@ -918,9 +916,8 @@ TwoAddressInstructionPass::RescheduleMIBelowKill(MachineBasicBlock *MBB, // Don't mess with copies, they may be coalesced later. return false; - const MCInstrDesc &MCID = KillMI->getDesc(); - if (MCID.hasUnmodeledSideEffects() || MCID.isCall() || MCID.isBranch() || - MCID.isTerminator()) + if (KillMI->hasUnmodeledSideEffects() || KillMI->isCall() || + KillMI->isBranch() || KillMI->isTerminator()) // Don't move pass calls, etc. return false; @@ -975,9 +972,8 @@ TwoAddressInstructionPass::RescheduleMIBelowKill(MachineBasicBlock *MBB, if (NumVisited > 10) // FIXME: Arbitrary limit to reduce compile time cost. return false; ++NumVisited; - const MCInstrDesc &OMCID = OtherMI->getDesc(); - if (OMCID.hasUnmodeledSideEffects() || OMCID.isCall() || OMCID.isBranch() || - OMCID.isTerminator()) + if (OtherMI->hasUnmodeledSideEffects() || OtherMI->isCall() || + OtherMI->isBranch() || OtherMI->isTerminator()) // Don't move pass calls, etc. return false; for (unsigned i = 0, e = OtherMI->getNumOperands(); i != e; ++i) { @@ -1119,9 +1115,8 @@ TwoAddressInstructionPass::RescheduleKillAboveMI(MachineBasicBlock *MBB, if (NumVisited > 10) // FIXME: Arbitrary limit to reduce compile time cost. return false; ++NumVisited; - const MCInstrDesc &MCID = OtherMI->getDesc(); - if (MCID.hasUnmodeledSideEffects() || MCID.isCall() || MCID.isBranch() || - MCID.isTerminator()) + if (OtherMI->hasUnmodeledSideEffects() || OtherMI->isCall() || + OtherMI->isBranch() || OtherMI->isTerminator()) // Don't move pass calls, etc. return false; SmallVector<unsigned, 2> OtherDefs; @@ -1201,7 +1196,6 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi, return false; MachineInstr &MI = *mi; - const MCInstrDesc &MCID = MI.getDesc(); unsigned regA = MI.getOperand(DstIdx).getReg(); unsigned regB = MI.getOperand(SrcIdx).getReg(); @@ -1223,7 +1217,7 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi, unsigned regCIdx = ~0U; bool TryCommute = false; bool AggressiveCommute = false; - if (MCID.isCommutable() && MI.getNumOperands() >= 3 && + if (MI.isCommutable() && MI.getNumOperands() >= 3 && TII->findCommutedOpIndices(&MI, SrcOp1, SrcOp2)) { if (SrcIdx == SrcOp1) regCIdx = SrcOp2; @@ -1261,7 +1255,7 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi, if (TargetRegisterInfo::isVirtualRegister(regA)) ScanUses(regA, &*mbbi, Processed); - if (MCID.isConvertibleTo3Addr()) { + if (MI.isConvertibleTo3Addr()) { // This instruction is potentially convertible to a true // three-address instruction. Check if it is profitable. if (!regBKilled || isProfitableToConv3Addr(regA, regB)) { @@ -1288,7 +1282,7 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi, // movq (%rax), %rcx // addq %rdx, %rcx // because it's preferable to schedule a load than a register copy. - if (MCID.mayLoad() && !regBKilled) { + if (MI.mayLoad() && !regBKilled) { // Determine if a load can be unfolded. unsigned LoadRegIndex; unsigned NewOpc = @@ -1531,7 +1525,7 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &MF) { // If it's safe and profitable, remat the definition instead of // copying it. if (DefMI && - DefMI->getDesc().isAsCheapAsAMove() && + DefMI->isAsCheapAsAMove() && DefMI->isSafeToReMat(TII, AA, regB) && isProfitableToReMat(regB, rc, mi, DefMI, mbbi, Dist)){ DEBUG(dbgs() << "2addr: REMATTING : " << *DefMI << "\n"); |